diff --git a/.vscode/settings.json b/.vscode/settings.json index dc24bfef..ed4d949e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -137,6 +137,7 @@ "noprogress", "onefile", "oneline", + "onupdate", "parsel", "pathex", "pickleable", @@ -164,11 +165,13 @@ "RPAREN", "rtime", "rvfc", + "sessionmaker", "SIGUSR", "smhd", "socketio", "softprops", "solverr", + "SQLA", "sstr", "startswith", "SUPPRESSHELP", @@ -199,6 +202,7 @@ "vconvert", "Vitest", "writedescription", + "WSEP", "xerror", "youtu", "youtubepot", diff --git a/API.md b/API.md index 9fac609e..078a42fc 100644 --- a/API.md +++ b/API.md @@ -2,10 +2,9 @@ This document describes the available endpoints and their usage. All endpoints return JSON responses (unless otherwise specified) and may require certain parameters (query, body, or path). Some endpoints serve static or streaming content (e.g., `.ts`, `.m3u8`, `.vtt` files). -> **Note**: If Basic Authentication is configured (via `auth_username` and `auth_password` in your configuration), you must include an `Authorization: Basic ` header or use `?apikey=` query parameter (fallback) in every request. +> **Note**: If Basic Authentication is configured, you must include an `Authorization: Basic ` header or use `?apikey=` query parameter (fallback) in every request. - All responses use standard HTTP status codes to indicate success or error conditions. -- Endpoints support an `OPTIONS` request for CORS. --- @@ -26,23 +25,31 @@ This document describes the available endpoints and their usage. All endpoints r - [POST /api/history/{id}](#post-apihistoryid) - [GET /api/history/{id}](#get-apihistoryid) - [GET /api/history](#get-apihistory) + - [GET /api/history/live](#get-apihistorylive) + - [POST /api/history/start](#post-apihistorystart) + - [POST /api/history/pause](#post-apihistorypause) + - [POST /api/history/cancel](#post-apihistorycancel) - [DELETE /api/history/{id}/archive](#delete-apihistoryidarchive) - [POST /api/history/{id}/archive](#post-apihistoryidarchive) - [GET /api/archiver](#get-apiarchiver) - [POST /api/archiver](#post-apiarchiver) - [DELETE /api/archiver](#delete-apiarchiver) - [GET /api/tasks](#get-apitasks) - - [PUT /api/tasks](#put-apitasks) + - [POST /api/tasks](#post-apitasks) + - [GET /api/tasks/{id}](#get-apitasksid) + - [DELETE /api/tasks/{id}](#delete-apitasksid) + - [PATCH /api/tasks/{id}](#patch-apitasksid) + - [PUT /api/tasks/{id}](#put-apitasksid) - [POST /api/tasks/inspect](#post-apitasksinspect) - [POST /api/tasks/{id}/mark](#post-apitasksidmark) - [DELETE /api/tasks/{id}/mark](#delete-apitasksidmark) - [POST /api/tasks/{id}/metadata](#post-apitasksidmetadata) - - [PATCH /api/tasks/{id}](#patch-apitasksid) - - [GET /api/task\_definitions/](#get-apitask_definitions) - - [GET /api/task\_definitions/{identifier}](#get-apitask_definitionsidentifier) - - [POST /api/task\_definitions/](#post-apitask_definitions) - - [PUT /api/task\_definitions/{identifier}](#put-apitask_definitionsidentifier) - - [DELETE /api/task\_definitions/{identifier}](#delete-apitask_definitionsidentifier) + - [GET /api/tasks/definitions/](#get-apitasksdefinitions) + - [GET /api/tasks/definitions/{id}](#get-apitasksdefinitionsid) + - [POST /api/tasks/definitions/](#post-apitasksdefinitions) + - [PUT /api/tasks/definitions/{id}](#put-apitasksdefinitionsid) + - [PATCH /api/tasks/definitions/{id}](#patch-apitasksdefinitionsid) + - [DELETE /api/tasks/definitions/{id}](#delete-apitasksdefinitionsid) - [GET /api/player/playlist/{file:.\*}.m3u8](#get-apiplayerplaylistfilem3u8) - [GET /api/player/m3u8/{mode}/{file:.\*}.m3u8](#get-apiplayerm3u8modefilem3u8) - [GET /api/player/segments/{segment}/{file:.\*}.ts](#get-apiplayersegmentssegmentfilets) @@ -57,21 +64,42 @@ This document describes the available endpoints and their usage. All endpoints r - [GET /api/download/{filename}](#get-apidownloadfilename) - [GET /api/random/background](#get-apirandombackground) - [GET /api/presets](#get-apipresets) - - [GET /api/dl\_fields](#get-apidl_fields) - - [PUT /api/dl\_fields](#put-apidl_fields) - - [PUT /api/presets](#put-apipresets) - - [GET /api/conditions](#get-apiconditions) - - [PUT /api/conditions](#put-apiconditions) + - [GET /api/presets/{id}](#get-apipresetsid) + - [POST /api/presets](#post-apipresets) + - [PATCH /api/presets/{id}](#patch-apipresetsid) + - [PUT /api/presets/{id}](#put-apipresetsid) + - [DELETE /api/presets/{id}](#delete-apipresetsid) + - [GET /api/dl\_fields/](#get-apidl_fields) + - [POST /api/dl\_fields/](#post-apidl_fields) + - [GET /api/dl\_fields/{id}](#get-apidl_fieldsid) + - [PATCH /api/dl\_fields/{id}](#patch-apidl_fieldsid) + - [PUT /api/dl\_fields/{id}](#put-apidl_fieldsid) + - [DELETE /api/dl\_fields/{id}](#delete-apidl_fieldsid) + - [GET /api/conditions/](#get-apiconditions) + - [POST /api/conditions/](#post-apiconditions) - [POST /api/conditions/test](#post-apiconditionstest) + - [GET /api/conditions/{id}](#get-apiconditionsid) + - [PATCH /api/conditions/{id}](#patch-apiconditionsid) + - [PUT /api/conditions/{id}](#put-apiconditionsid) + - [DELETE /api/conditions/{id}](#delete-apiconditionsid) - [GET /api/logs](#get-apilogs) - - [GET /api/notifications](#get-apinotifications) - - [PUT /api/notifications](#put-apinotifications) + - [GET /api/logs/stream](#get-apilogsstream) + - [GET /api/notifications/](#get-apinotifications) + - [GET /api/notifications/events/](#get-apinotificationsevents) + - [POST /api/notifications/](#post-apinotifications) + - [GET /api/notifications/{id}](#get-apinotificationsid) + - [PATCH /api/notifications/{id}](#patch-apinotificationsid) + - [PUT /api/notifications/{id}](#put-apinotificationsid) + - [DELETE /api/notifications/{id}](#delete-apinotificationsid) - [POST /api/yt-dlp/archive\_id/](#post-apiyt-dlparchive_id) - [POST /api/notifications/test](#post-apinotificationstest) - [GET /api/yt-dlp/options](#get-apiyt-dlpoptions) + - [GET /api/system/configuration](#get-apisystemconfiguration) + - [POST /api/system/terminal](#post-apisystemterminal) - [POST /api/system/pause](#post-apisystempause) - [POST /api/system/resume](#post-apisystemresume) - [POST /api/system/shutdown](#post-apisystemshutdown) + - [POST /api/system/check-updates](#post-apisystemcheck-updates) - [GET /api/dev/loop](#get-apidevloop) - [GET /api/dev/pip](#get-apidevpip) - [GET /api/docs/{file}](#get-apidocsfile) @@ -79,32 +107,32 @@ This document describes the available endpoints and their usage. All endpoints r - [Connection](#connection) - [Authentication](#authentication-1) - [Message Format](#message-format) - - [Core Events](#core-events) + - [Client Events (Client → Server)](#client-events-client--server) + - [`add_url`](#add_url) + - [`item_cancel`](#item_cancel) + - [`item_delete`](#item_delete) + - [`item_start`](#item_start) + - [`item_pause`](#item_pause) + - [Server Events (Server → Client)](#server-events-server--client) - [Connection Events](#connection-events) - - [`connect` (Built-in)](#connect-built-in) - - [`disconnect` (Built-in)](#disconnect-built-in) - - [`configuration` (Server → Client)](#configuration-server--client) - - [`connected` (Server → Client)](#connected-server--client) + - [`config_update`](#config_update) + - [`connected`](#connected) + - [`active_queue`](#active_queue) - [Logging Events](#logging-events) - - [`log_info` (Server → Client)](#log_info-server--client) - - [`log_success` (Server → Client)](#log_success-server--client) - - [`log_warning` (Server → Client)](#log_warning-server--client) - - [`log_error` (Server → Client)](#log_error-server--client) - - [`log_lines` (Server → Client)](#log_lines-server--client) - - [Download Queue Events](#download-queue-events) - - [`item_added` (Server → Client)](#item_added-server--client) - - [`item_updated` (Server → Client)](#item_updated-server--client) - - [`item_completed` (Server → Client)](#item_completed-server--client) - - [`item_cancelled` (Server → Client)](#item_cancelled-server--client) - - [`item_deleted` (Server → Client)](#item_deleted-server--client) - - [`item_moved` (Server → Client)](#item_moved-server--client) - - [Terminal/CLI Events](#terminalcli-events) - - [`cli_post` (Client → Server)](#cli_post-client--server) - - [`cli_output` (Server → Client)](#cli_output-server--client) - - [`cli_close` (Server → Client)](#cli_close-server--client) - - [Configuration Events](#configuration-events) - - [`presets_update` (Server → Client)](#presets_update-server--client) - - [`dlfields_update` (Server → Client)](#dlfields_update-server--client) + - [`log_info`](#log_info) + - [`log_success`](#log_success) + - [`log_warning`](#log_warning) + - [`log_error`](#log_error) + - [Download Queue Events](#download-queue-events) + - [`item_added`](#item_added) + - [`item_updated`](#item_updated) + - [`item_cancelled`](#item_cancelled) + - [`item_deleted`](#item_deleted) + - [`item_moved`](#item_moved) + - [`item_status`](#item_status) + - [`paused`](#paused) + - [`resumed`](#resumed) + - [Error Responses](#error-responses) --- @@ -161,6 +189,9 @@ If you fail to provide valid credentials, a `401 Unauthorized` response is retur } ``` +**Notes**: +- IDs are integer values generated by the database. + --- ### POST /api/yt-dlp/convert @@ -582,6 +613,124 @@ GET /api/history?type=queue&status=pending&order=ASC --- +### GET /api/history/live +**Purpose**: Get live queue data with real-time download progress. + +This endpoint returns the current state of active downloads from memory. + +**Response**: +```json +{ + "history_count": 0, // total number of completed items in history + "queue":{ + "id": "abc123", + "url": "https://example.com/video", + "title": "Video Title", + "status": "downloading", + "progress": 45.6, + "speed": "2.5 MiB/s", + "eta": "00:05:23", + ... + }, + ... +} +``` + +--- + +### POST /api/history/start +**Purpose**: Start one or more downloads in the queue. + +**Body**: +```json +{ + "ids": ["", "", ...] +} +``` + +**Response**: +```json +{ + "": "started", + "": "started", + ... +} +``` + +**Error Responses**: +- `400 Bad Request` if `ids` is missing or not an array: + ```json + { "error": "ids is required and must be an array." } + ``` + +**Notes**: +- Items must exist in the queue +- Sets `auto_start: true` for the specified items + +--- + +### POST /api/history/pause +**Purpose**: Pause one or more downloads in the queue. + +**Body**: +```json +{ + "ids": ["", "", ...] +} +``` + +**Response**: +```json +{ + "": "paused", + "": "paused", + ... +} +``` + +**Error Responses**: +- `400 Bad Request` if `ids` is missing or not an array: + ```json + { "error": "ids is required and must be an array." } + ``` + +**Notes**: +- Items must exist in the queue +- Sets `auto_start: false` for the specified items + +--- + +### POST /api/history/cancel +**Purpose**: Cancel one or more downloads and move them to history. + +**Body**: +```json +{ + "ids": ["", "", ...] +} +``` + +**Response**: +```json +{ + "": "ok", + "": "ok", + ... +} +``` + +**Error Responses**: +- `400 Bad Request` if `ids` is missing or not an array: + ```json + { "error": "ids is required and must be an array." } + ``` + +**Notes**: +- Items must exist in the queue +- Stops active downloads if they are currently running + +--- + ### DELETE /api/history/{id}/archive **Purpose**: Remove an item from archive file, allowing it to be re-downloaded. @@ -698,81 +847,206 @@ Notes: --- ### GET /api/tasks -**Purpose**: Retrieves the scheduled tasks from the internal `Tasks` manager. +**Purpose**: Retrieves the scheduled tasks. + +**Query Parameters**: +- `page` (optional): Page number (1-indexed). Default: `1`. +- `per_page` (optional): Items per page. Default: `config.default_pagination`. **Response**: ```json -[ - { - "id": "", - "name": "...", - "url": "...", - "folder": "...", - "preset": "...", - "timer": "", - "template": "...", - "cookies": "...", - "config": { ... }, - }, - ... -] +{ + "items": [ + { + "id": 1, + "name": "My Task", + "url": "https://youtube.com/...", + "timer": "5 */2 * * *", + "cookies": "", + "config": {}, + "template": "...", + "folder": "...", + "preset": "...", + "auto_start": true, + "handler_enabled": true, + "enabled": true + } + ], + "pagination": { + "page": 1, + "per_page": 50, + "total": 1, + "total_pages": 1, + "has_next": false, + "has_prev": false + } +} ``` --- -### PUT /api/tasks -**Purpose**: Overwrites the entire scheduled tasks list (Cron tasks). +### POST /api/tasks +**Purpose**: Create a new scheduled task. -**Body**: An array of task objects. Example: +**Body**: Task object. Example: ```json -[ - { - "id": "a2ae3f18-4428-4e32-9d4c-0cc45af8bb48", - "name": "My Task", - "url": "https://youtube.com/...", - "timer": "5 */2 * * *", - "cookies": "", - "config": {}, - "template": "...", - "folder": "...", - "auto_start": true, - "handler_enabled": true, - "enabled": true - }, - { - "url": "https://youtube.com/...", - "timer": "*/15 * * * *" - } -] +{ + "name": "My Task", + "url": "https://youtube.com/...", + "timer": "5 */2 * * *", + "cookies": "", + "config": {}, + "template": "...", + "folder": "...", + "preset": "...", + "auto_start": true, + "handler_enabled": true, + "enabled": true +} ``` -If `id` or other fields are missing, they may be auto-generated or defaulted (e.g., a random ID, a default cron, etc.). **Response**: ```json -[ - { - "id": "", - "name": "...", - "url": "...", - "timer": "...", - "cookies": "...", - "config": { ... }, - "template": "...", - "folder": "...", - "auto_start": true, - "handler_enabled": true, - "enabled": true - } - ... -] -``` -or on error -```json { - "error": "text" + "id": 1, + "name": "My Task", + "url": "https://youtube.com/...", + "timer": "5 */2 * * *", + "cookies": "", + "config": {}, + "template": "...", + "folder": "...", + "preset": "...", + "auto_start": true, + "handler_enabled": true, + "enabled": true } ``` +**Error Responses**: +- `400 Bad Request` - Invalid request body or validation error +- `409 Conflict` - Task with the same name already exists + +--- + +### GET /api/tasks/{id} +**Purpose**: Retrieve a specific task by ID. + +**Path Parameter**: +- `id`: Task ID. + +**Response**: +```json +{ + "id": 1, + "name": "My Task", + "url": "https://youtube.com/...", + "timer": "5 */2 * * *", + "cookies": "", + "config": {}, + "template": "...", + "folder": "...", + "preset": "...", + "auto_start": true, + "handler_enabled": true, + "enabled": true +} +``` + +**Error Responses**: +- `400 Bad Request` - ID is missing +- `404 Not Found` - Task does not exist + +--- + +### DELETE /api/tasks/{id} +**Purpose**: Delete a scheduled task by ID. + +**Path Parameter**: +- `id`: Task ID. + +**Response**: +```json +{ + "id": 1, + "name": "Deleted Task", + "url": "https://youtube.com/...", + "timer": "5 */2 * * *", + "cookies": "", + "config": {}, + "template": "...", + "folder": "...", + "preset": "...", + "auto_start": true, + "handler_enabled": true, + "enabled": true +} +``` + +**Error Responses**: +- `400 Bad Request` - ID is missing +- `404 Not Found` - Task does not exist + +--- + +### PATCH /api/tasks/{id} +**Purpose**: Partially update a scheduled task. + +**Path Parameter**: +- `id`: Task ID. + +**Request Body**: +JSON object with fields to update: +```json +{ + "enabled": false, + "timer": "0 */6 * * *" +} +``` + +**Notes**: +- Only include fields you want to update +- All fields are optional + +**Response**: Returns the updated task object. + +**Error Responses**: +- `400 Bad Request` - ID is missing, invalid JSON, or validation error +- `404 Not Found` - Task does not exist +- `409 Conflict` - Task name conflict + +--- + +### PUT /api/tasks/{id} +**Purpose**: Replace an existing scheduled task. + +**Path Parameter**: +- `id`: Task ID. + +**Request Body**: Full task object: +```json +{ + "name": "Updated Task", + "url": "https://youtube.com/...", + "timer": "0 */6 * * *", + "cookies": "", + "config": {}, + "template": "...", + "folder": "...", + "preset": "...", + "auto_start": true, + "handler_enabled": true, + "enabled": true +} +``` + +**Response**: Returns the updated task object. + +**Error Responses**: +- `400 Bad Request` - ID is missing, invalid JSON, or validation error +- `404 Not Found` - Task does not exist +- `409 Conflict` - Task name conflict + --- ### POST /api/tasks/inspect @@ -942,114 +1216,71 @@ or --- -### PATCH /api/tasks/{id} -**Purpose**: Update specific fields of a scheduled task. - -**Path Parameter**: -- `id`: Task ID. - -**Request Body**: -JSON object with fields to update: -```json -{ - "enabled": false, - "handler_enabled": true, - "timer": "0 */6 * * *", - "preset": "audio", - "folder": "downloads/music", - "template": "%(title)s.%(ext)s", - "cli": "--extract-audio", - "auto_start": true -} -``` - -**Notes**: -- Only include fields you want to update -- All fields are optional - -**Response**: -Returns the updated task -```json -{ - "id": "task_id", - "name": "Task Name", - "url": "https://example.com/playlist", - "preset": "audio", - "folder": "downloads/music", - "template": "%(title)s.%(ext)s", - "cli": "--extract-audio", - "timer": "0 */6 * * *", - "auto_start": true, - "handler_enabled": true, - "enabled": false -} -``` - -**Error Responses**: -```json -{ "error": "Task 'task_id' does not exist." } -{ "error": "Invalid JSON in request body.", "message": "..." } -{ "error": "Request body must be a JSON object." } -{ "error": "No valid fields to update." } -{ "error": "Validation failed: Invalid timer format." } -``` - -**Status Codes**: -- `200 OK` - Task updated successfully -- `400 Bad Request` - Missing task ID, invalid JSON, no valid fields, or validation error -- `404 Not Found` - Task does not exist -- `500 Internal Server Error` - Failed to update task - ---- - -### GET /api/task_definitions/ -**Purpose**: Retrieve all task definitions. +### GET /api/tasks/definitions/ +**Purpose**: Retrieve task definitions. **Query Parameters**: - `include=definition` (optional) - Include the full definition object in response. - -**Response**: -```json -[ - { - "id": "", - "name": "Task Definition Name", - "description": "...", - "enabled": true, - "definition": { ... } // only if include=definition - }, - ... -] -``` - ---- - -### GET /api/task_definitions/{identifier} -**Purpose**: Retrieve a specific task definition by ID or name. - -**Path Parameter**: -- `identifier`: Task definition ID or name. +- `page` (optional): Page number (1-indexed). Default: `1`. +- `per_page` (optional): Items per page. Default: `config.default_pagination`. **Response**: ```json { - "id": "", - "name": "Task Definition Name", - "description": "...", - "enabled": true, - "definition": { - "handler": "GenericTaskHandler", - "config": { ... } + "items": [ + { + "id": 1, + "name": "Task Definition Name", + "description": "...", + "enabled": true, + "definition": { ... } // only if include=definition + }, + ... + ], + "pagination": { + "page": 1, + "per_page": 50, + "total": 1, + "total_pages": 1, + "has_next": false, + "has_prev": false } } ``` -- `400 Bad Request` if identifier is missing. +--- + +### GET /api/tasks/definitions/{id} +**Purpose**: Retrieve a specific task definition by ID. + +**Path Parameter**: +- `id`: Task definition ID. + +**Response**: +```json +{ + "id": 1, + "name": "Task Definition Name", + "description": "...", + "enabled": true, + "definition": { + "parse": { + "url": { ... }, + "items": { ... } + }, + "engine": { ... }, + "request": { ... }, + "response": { ... } + } +} +``` + +- `400 Bad Request` if ID is missing. - `404 Not Found` if the task definition doesn't exist. --- -### POST /api/task_definitions/ +### POST /api/tasks/definitions/ **Purpose**: Create a new task definition. **Body**: @@ -1059,19 +1290,13 @@ Returns the updated task "description": "...", "enabled": true, "definition": { - "handler": "GenericTaskHandler", - "config": { ... } - } -} -``` - -Or wrap in a definition object: -```json -{ - "definition": { - "name": "My Task Definition", - "handler": "GenericTaskHandler", - ... + "parse": { + "url": { ... }, + "items": { ... } + }, + "engine": { ... }, + "request": { ... }, + "response": { ... } } } ``` @@ -1079,7 +1304,7 @@ Or wrap in a definition object: **Response**: ```json { - "id": "", + "id": 1, "name": "My Task Definition", "description": "...", "enabled": true, @@ -1092,11 +1317,11 @@ Or wrap in a definition object: --- -### PUT /api/task_definitions/{identifier} -**Purpose**: Update an existing task definition. +### PUT /api/tasks/definitions/{id} +**Purpose**: Replace an existing task definition. **Path Parameter**: -- `identifier`: Task definition ID or name. +- `id`: Task definition ID. **Body**: ```json @@ -1105,8 +1330,10 @@ Or wrap in a definition object: "description": "...", "enabled": false, "definition": { - "handler": "GenericTaskHandler", - "config": { ... } + "parse": { ... }, + "engine": { ... }, + "request": { ... }, + "response": { ... } } } ``` @@ -1114,7 +1341,7 @@ Or wrap in a definition object: **Response**: ```json { - "id": "", + "id": 1, "name": "Updated Name", "description": "...", "enabled": false, @@ -1123,23 +1350,53 @@ Or wrap in a definition object: ``` - `200 OK` if successful. -- `400 Bad Request` if identifier is missing or validation fails. +- `400 Bad Request` if ID is missing or validation fails. +- `404 Not Found` if the task definition doesn't exist. --- -### DELETE /api/task_definitions/{identifier} +### PATCH /api/tasks/definitions/{id} +**Purpose**: Partially update a task definition. + +**Path Parameter**: +- `id`: Task definition ID. + +**Body**: +```json +{ + "enabled": false, + "description": "Updated description" +} +``` + +**Response**: Updated task definition object. + +- `200 OK` if successful. +- `400 Bad Request` if ID is missing or validation fails. +- `404 Not Found` if the task definition doesn't exist. + +--- + +### DELETE /api/tasks/definitions/{id} **Purpose**: Delete a task definition. **Path Parameter**: -- `identifier`: Task definition ID or name. +- `id`: Task definition ID. **Response**: ```json -{ "status": "deleted" } +{ + "id": 1, + "name": "Deleted Definition", + "description": "...", + "enabled": false, + "definition": { ... } +} ``` - `200 OK` if successful. -- `400 Bad Request` if identifier is missing or task definition doesn't exist. +- `400 Bad Request` if ID is missing. +- `404 Not Found` if the task definition doesn't exist. --- @@ -1400,172 +1657,357 @@ Binary image data with appropriate headers --- ### GET /api/presets -**Purpose**: Retrieve all available download presets. +**Purpose**: Retrieve available presets. **Query Parameters**: -- `filter=` (optional) - Comma-separated list of fields to include in response. +- `page` (optional): Page number (1-indexed). Default: `1`. +- `per_page` (optional): Items per page. Default: `config.default_pagination`. **Response**: -```json5 -[ - { - "id": "", - "name": "preset_name", - "description": "...", - "folder": "my_channel/foo", - "template": "%(title)s.%(ext)s", - "cookies": "...", - "cli": "--write-subs --embed-subs", - "default": true|false, // optional, indicates if this is the default preset. - ... - }, - ... -] +```json +{ + "items": [ + { + "id": 1, + "name": "preset_name", + "description": "...", + "folder": "my_channel/foo", + "template": "%(title)s.%(ext)s", + "cookies": "...", + "cli": "--write-subs --embed-subs", + "default": true + } + ], + "pagination": { + "page": 1, + "per_page": 50, + "total": 1, + "total_pages": 1, + "has_next": false, + "has_prev": false + } +} +``` + +**Notes**: +- `default: true` indicates this is a system default preset (cannot be modified or deleted) + +--- + +### GET /api/presets/{id} +**Purpose**: Retrieve a specific preset by ID. + +**Path Parameter**: +- `id`: Preset ID. + +**Response**: +```json +{ + "id": 1, + "name": "preset_name", + "description": "...", + "folder": "my_channel/foo", + "template": "%(title)s.%(ext)s", + "cookies": "...", + "cli": "--write-subs --embed-subs", + "default": false +} +``` + +**Error Responses**: +- `400 Bad Request` - ID is missing +- `404 Not Found` - Preset not found + +--- + +### POST /api/presets +**Purpose**: Create a new download preset. + +**Body**: +```json +{ + "name": "My Preset", + "description": "...", + "folder": "my_channel/foo", + "template": "%(title)s.%(ext)s", + "cookies": "...", + "cli": "--write-subs --embed-subs" +} +``` + +**Response**: +```json +{ + "id": 1, + "name": "My Preset", + "description": "...", + "folder": "my_channel/foo", + "template": "%(title)s.%(ext)s", + "cookies": "...", + "cli": "--write-subs --embed-subs", + "default": false +} +``` + +**Error Responses**: +- `400 Bad Request` - Invalid request body or validation error + +--- + +### PATCH /api/presets/{id} +**Purpose**: Partially update a preset. + +**Path Parameter**: +- `id`: Preset ID. + +**Body**: +```json +{ + "description": "Updated description", + "cli": "--write-subs --embed-subs --format best" +} +``` + +**Notes**: +- Only include fields you want to update +- All fields are optional +- Default presets cannot be modified + +**Response**: Returns the updated preset object. + +**Error Responses**: +- `400 Bad Request` - ID is missing, invalid JSON, validation error, or attempting to modify default preset +- `404 Not Found` - Preset not found +- `409 Conflict` - Preset name conflict + +--- + +### PUT /api/presets/{id} +**Purpose**: Replace an existing preset. + +**Path Parameter**: +- `id`: Preset ID. + +**Body**: Full preset object: +```json +{ + "name": "Updated Preset", + "description": "...", + "folder": "my_channel/foo", + "template": "%(title)s.%(ext)s", + "cookies": "...", + "cli": "--write-subs --embed-subs" +} +``` + +**Notes**: +- Default presets cannot be modified + +**Response**: Returns the updated preset object. + +**Error Responses**: +- `400 Bad Request` - ID is missing, invalid JSON, validation error, or attempting to modify default preset +- `404 Not Found` - Preset not found +- `409 Conflict` - Preset name conflict + +--- + +### DELETE /api/presets/{id} +**Purpose**: Delete a preset by ID. + +**Path Parameter**: +- `id`: Preset ID. + +**Response**: Returns the deleted preset object. + +**Error Responses**: +- `400 Bad Request` - ID is missing or attempting to delete default preset +- `404 Not Found` - Preset not found + +--- + +### GET /api/dl_fields/ +**Purpose**: Retrieve download fields with pagination. + +**Query Parameters**: +- `page` (optional): Page number (1-indexed). Default: `1`. +- `per_page` (optional): Items per page. Default: `config.default_pagination`. + +**Response**: +```json +{ + "items": [ + { "id": 1, "name": "...", "description": "...", "field": "...", "kind": "text", "order": 0, "value": "", "extras": {} } + ], + "pagination": { + "page": 1, + "per_page": 50, + "total": 1, + "total_pages": 1, + "has_next": false, + "has_prev": false + } +} ``` --- -### GET /api/dl_fields -**Purpose**: Retrieve the list of configured download fields. +### POST /api/dl_fields/ +**Purpose**: Create a new download field. -**Query Parameters (optional)**: -- `filter`: Comma-separated list of field names to include in each object. +**Body**: +```json +{ + "name": "Title", + "description": "...", + "field": "title", + "kind": "text", + "order": 0, + "value": "", + "icon": "fa-solid fa-tag", + "extras": {} +} +``` **Response**: ```json -[ - { "id": "", "name": "...", ... }, - ... -] +{ "id": 1, "name": "Title", "description": "...", "field": "title", "kind": "text", "order": 0, "value": "", "icon": "fa-solid fa-tag", "extras": {} } ``` --- -### PUT /api/dl_fields -**Purpose**: Save the list of download fields. Replaces existing entries. +### GET /api/dl_fields/{id} +**Purpose**: Retrieve a single download field by ID. -**Body**: Array of objects. Required per-item fields: `name`. `id` is auto-generated if missing or invalid. -```json -[ - { "name": "...", "id": "", ... }, - { "name": "..." } -] -``` +**Path Parameter**: +- `id`: Download field ID. **Response**: ```json -[ - { "id": "", "name": "...", ... }, - ... -] -``` -or an error: -```json -{ "error": "text" } +{ "id": 1, "name": "Title", "description": "...", "field": "title", "kind": "text", "order": 0, "value": "", "icon": "fa-solid fa-tag", "extras": {} } ``` --- -### PUT /api/presets -**Purpose**: Save/update download presets. +### PATCH /api/dl_fields/{id} +**Purpose**: Partially update a download field. -**Body**: An array of preset objects: +**Path Parameter**: +- `id`: Download field ID. + +**Body**: ```json -[ - { - "name": "My Preset", // required, unique name for the preset - "id": "", // optional, will be generated if not provided - "description": "...", // optional, description of the preset - "folder": "my_channel/foo", // optional, relative to download_path - "template": "%(title)s.%(ext)s", // optional, filename template - "cookies": "...", // optional, Netscape HTTP Cookie format - "cli": "--write-subs --embed-subs", // optional, additional command options for yt-dlp - }, - ... -] +{ "description": "Updated description", "order": 1 } ``` +**Response**: Updated download field object. + +--- + +### PUT /api/dl_fields/{id} +**Purpose**: Replace a download field. + +**Path Parameter**: +- `id`: Download field ID. + +**Body**: +```json +{ + "name": "Title", + "description": "...", + "field": "title", + "kind": "text", + "order": 0, + "value": "", + "icon": "fa-solid fa-tag", + "extras": {} +} +``` + +**Response**: Updated download field object. + +--- + +### DELETE /api/dl_fields/{id} +**Purpose**: Delete a download field by ID. + +**Path Parameter**: +- `id`: Download field ID. + **Response**: ```json -[ - { - "id": "", - "name": "My Preset", - "description": "...", - }, - ... -] +{ + "id": 1, + "name": "Title", + "description": "...", + "field": "title", + "kind": "text", + "order": 0, + "value": "", + "icon": "fa-solid fa-tag", + "extras": {} +} ``` --- -### GET /api/conditions -**Purpose**: Retrieve all configured download conditions. +### GET /api/conditions/ +**Purpose**: Retrieve download conditions with pagination. + +**Query Parameters**: +- `page` (optional): Page number (1-indexed). Default: `1`. +- `per_page` (optional): Items per page. Default: `config.default_pagination`. **Response**: ```json -[ - { - "id": "", - "name": "condition_name", - "filter": "...", - "cli": "...", - "extras": {}, - "enabled": true, - "priority": 0, - "description": "What this condition does" - }, - ... -] +{ + "items": [ + { + "id": 1, + "name": "condition_name", + "filter": "...", + "cli": "...", + "extras": {}, + "enabled": true, + "priority": 0, + "description": "What this condition does" + } + ], + "pagination": { + "page": 1, + "per_page": 50, + "total": 1, + "total_pages": 1, + "has_next": false, + "has_prev": false + } +} ``` **Notes**: - Conditions are evaluated in priority order (higher priority first). - Priority defaults to 0 when not specified. +- IDs are integer values generated by the database. --- -### PUT /api/conditions -**Purpose**: Save/update download conditions. +### POST /api/conditions/ +**Purpose**: Create a new download condition. -**Body**: An array of condition objects: +**Body**: ```json -[ - { - "id": "", // optional, will be generated if not provided - "name": "Use proxy for region locked content", - "filter": "availability = 'needs_auth' & channel_id = 'channel_id'", - "cli": "--proxy http://myproxy.com:8080", - "extras": {}, - "enabled": true, - "priority": 10, - "description": "Apply proxy for region-locked videos" - }, - ... -] +{ + "name": "Use proxy for region locked content", + "filter": "availability = 'needs_auth' & channel_id = 'channel_id'", + "cli": "--proxy http://myproxy.com:8080", + "extras": {}, + "enabled": true, + "priority": 10, + "description": "Apply proxy for region-locked videos" +} ``` -**Response**: -```json -[ - { - "id": "", - "name": "Use proxy for region locked content", - "filter": "availability = 'needs_auth' & channel_id = 'channel_id'", - "cli": "--proxy http://myproxy.com:8080", - "extras": {}, - "enabled": true, - "priority": 10, - "description": "Apply proxy for region-locked videos" - }, - ... -] -``` - -**Notes**: -- Disabled conditions (`enabled: false`) will be stored but ignored during matching. -- All conditions are enabled by default when the `enabled` field is not provided. -- Priority determines check order. Higher priority conditions are checked first. +**Response**: Created condition object. --- @@ -1590,6 +2032,53 @@ or an error: --- +### GET /api/conditions/{id} +**Purpose**: Retrieve a condition by ID. + +**Path Parameter**: +- `id`: Condition ID. + +**Response**: Condition object. + +--- + +### PATCH /api/conditions/{id} +**Purpose**: Partially update a condition. + +**Path Parameter**: +- `id`: Condition ID. + +**Body**: +```json +{ "enabled": false, "priority": 5 } +``` + +**Response**: Updated condition object. + +--- + +### PUT /api/conditions/{id} +**Purpose**: Replace a condition. + +**Path Parameter**: +- `id`: Condition ID. + +**Body**: Full condition object. + +**Response**: Updated condition object. + +--- + +### DELETE /api/conditions/{id} +**Purpose**: Delete a condition by ID. + +**Path Parameter**: +- `id`: Condition ID. + +**Response**: Deleted condition object. + +--- + ### GET /api/logs **Purpose**: Retrieve recent application logs (if file logging is enabled). @@ -1619,85 +2108,150 @@ or an error: --- -### GET /api/notifications -**Purpose**: Retrieve the configured notification targets and which event types are allowed. +### GET /api/logs/stream +**Purpose**: Stream live log lines via Server-Sent Events (SSE). -**Response** (example): +**Response**: +- `Content-Type: text/event-stream` +- Emits `log_lines` events with a log line payload. + +**Event Payload**: ```json { - "notifications": [ - { - "id": "uuid", - "name":"...", - "on": ["completed", "error",...], // empty array means all events. - "request":{ - "type":"json|form", - "method":"POST|PUT", - "url":"https://...", - "headers":[ - {"key":"...", "value":"..."}, - ... - ] - } - } - } - ], - "allowedTypes": ["added", "completed", "error", "cancelled", "cleared", "log_info", "log_success", "log_warning", "log_error", "test"] + "id": "", + "line": "", + "datetime": "2024-01-01T12:00:00.000000+00:00" } ``` +- Returns `404 Not Found` if file logging is not enabled or the log file is missing. + --- -### PUT /api/notifications -**Purpose**: Overwrites the entire list of notification targets. +### GET /api/notifications/ +**Purpose**: Retrieve notification targets with pagination. -**Body**: An array of notification target configurations. Example: -```json -[ - { - "id": "uuid", - "name": "My Webhook", - "on": ["completed", "error"], - "request": { - "type": "json", - "method": "POST", - "url": "https://...", - "headers": [ - { "key": "Authorization", "value": "Bearer ..." } - ] - } - }, - { - "name": "Another Webhook", - "on": ["completed"], - "request": { - "type": "form", - "method": "PUT", - "url": "https://...", - "headers": [] - } - } - ... -] -``` -- If `id` is not provided or is not a valid UUIDv4, it will be auto-generated. -- If the payload list is empty, all existing notifications are removed. +**Query Parameters**: +- `page` (optional): Page number (1-indexed). Default: `1`. +- `per_page` (optional): Items per page. Default: `config.default_pagination`. **Response**: ```json { - "notifications": [ + "items": [ { - "id": "uuid", - "name": "...", - "on": ["completed", "error", ...], - "request": { ... } - }, - ... + "id": 1, + "name": "My Webhook", + "on": ["item_completed"], + "presets": ["default"], + "enabled": true, + "request": { + "type": "json", + "method": "POST", + "url": "https://example.com/webhook", + "data_key": "data", + "headers": [{ "key": "Authorization", "value": "Bearer ..." }] + } + } ], - "allowedTypes": ["added", "completed", "error", "cancelled", "cleared", "log_info", "log_success", ...] + "pagination": { + "page": 1, + "per_page": 50, + "total": 1, + "total_pages": 1, + "has_next": false, + "has_prev": false + } } ``` + +**Notes**: +- Empty `on` and `presets` arrays mean all events/presets. +- `request.method` supports `POST` and `PUT`. +- `request.type` supports `json` and `form`. +- IDs are integer values generated by the database. + +--- + +### GET /api/notifications/events/ +**Purpose**: Retrieve allowed notification event names. + +**Response**: +```json +{ "events": ["item_added", "item_completed", "log_error", "test"] } +``` + +--- + +### POST /api/notifications/ +**Purpose**: Create a notification target. + +**Body**: +```json +{ + "name": "My Webhook", + "on": ["item_completed"], + "presets": ["default"], + "enabled": true, + "request": { + "type": "json", + "method": "POST", + "url": "https://example.com/webhook", + "data_key": "data", + "headers": [{ "key": "Authorization", "value": "Bearer ..." }] + } +} +``` + +**Response**: Created notification target. + +--- + +### GET /api/notifications/{id} +**Purpose**: Retrieve a notification target by ID. + +**Path Parameter**: +- `id`: Notification target ID. + +**Response**: Notification target object. + +--- + +### PATCH /api/notifications/{id} +**Purpose**: Partially update a notification target. + +**Path Parameter**: +- `id`: Notification target ID. + +**Body**: +```json +{ "enabled": false } +``` + +**Response**: Updated notification target. + +--- + +### PUT /api/notifications/{id} +**Purpose**: Replace a notification target. + +**Path Parameter**: +- `id`: Notification target ID. + +**Body**: Full notification target object. + +**Response**: Updated notification target. + +--- + +### DELETE /api/notifications/{id} +**Purpose**: Delete a notification target. + +**Path Parameter**: +- `id`: Notification target ID. + +**Response**: Deleted notification target. + --- ### POST /api/yt-dlp/archive_id/ @@ -1741,10 +2295,7 @@ or an error: **Response**: ```json -{ - "type": "test", - "message": "This is a test notification." -} +{} ``` --- @@ -1752,19 +2303,88 @@ or an error: ### GET /api/yt-dlp/options **Purpose**: Get the current yt-dlp CLI options as a JSON object. +**Response**: JSON object with yt-dlp options and metadata. + +--- + +### GET /api/system/configuration +**Purpose**: Retrieve comprehensive system configuration including app settings, presets, download fields, queue status, and folder structure. + **Response**: ```json -[ - { - "description": "Description of the option", - "flags":[ "--option", "-o" ], - "group": "Option Group", - "ignored": false, // true if this option is ignored by ytptube. +{ + "app": { + "version": "...", + "download_path": "/path/to/downloads", + "base_path": "/", + ... }, - ... -] + "presets": [ + { + "id": 1, + "name": "default", + "description": "...", + ... + } + ], + "dl_fields": [ + { + "id": 1, + "name": "Title", + "field": "title", + "kind": "text", + ... + } + ], + "paused": false, + "folders": [ + {"name": "folder1", "path": "folder1"}, + {"name": "folder2", "path": "folder2"} + ], + "history_count": 150, + "queue": [ + { + "id": "abc123", + "url": "https://example.com/video", + "status": "pending", + ... + } + ] +} ``` +**Notes**: +- This endpoint combines multiple data sources into a single response for efficient initialization +- The `folders` array includes available download folders up to the configured depth limit +- The `queue` array contains active download items + +--- + +### POST /api/system/terminal +**Purpose**: Stream yt-dlp CLI output via Server-Sent Events (SSE). Requires `YTP_CONSOLE_ENABLED=true`. + +**Body**: +```json +{ + "command": "--help" +} +``` + +**Response**: +- `Content-Type: text/event-stream` +- Emits `output` events for stdout/stderr and a final `close` event when the process exits. + +**Event Payloads**: +```json +{ "type": "stdout", "line": "" } +``` +```json +{ "exitcode": 0 } +``` + +- `403 Forbidden` if console is disabled. +- `400 Bad Request` if the request body is invalid. + --- ### POST /api/system/pause @@ -1876,228 +2496,494 @@ or an error: ## WebSocket API -The WebSocket API provides real-time bidirectional communication between the client and server using Socket.IO protocol. It enables live updates for downloads, queue status, notifications, and terminal access. - -> ![IMPORTANT] -> The WebSocket API is unstable and many events will be moved to REST endpoints in future releases. -> Please do not rely on the WebSocket API for the time being. - ### Connection -**URL**: `ws://localhost:8081/socket.io/` (development) or `https://yourdomain.com/socket.io/` (production) +**Endpoint**: `/ws` (or `{base_path}/ws` if base_path is configured) -The client automatically connects to the WebSocket server and receives a `connected` event with initial state. The connection uses automatic reconnection with exponential backoff (default: up to 30 attempts, 5s delay). +The WebSocket API provides real-time bidirectional communication between the client and server for download queue management and status updates. -**Connection Options**: +**Connection Details**: +- Protocol: WebSocket (ws:// or wss://) +- Heartbeat: 10-second interval +- Auto-reconnect: Client should implement reconnection logic + +**Example Connection**: ```javascript -{ - transports: ['websocket', 'polling'], // Fallback to long-polling if WebSocket unavailable - withCredentials: true, // Include cookies for authentication - reconnection: true, // Enable automatic reconnection - reconnectionAttempts: 30, // Max reconnection attempts - reconnectionDelay: 5000 // Initial reconnection delay in ms -} +const ws = new WebSocket('ws://localhost:8080/ws'); +ws.onopen = () => console.log('Connected'); +ws.onmessage = (event) => { + const message = JSON.parse(event.data); + console.log('Event:', message.event, 'Data:', message.data); +}; ``` +--- + ### Authentication -If Basic Authentication is configured, include credentials when establishing the WebSocket connection: +WebSocket connections use the same authentication as HTTP endpoints. If `YTP_AUTH_USERNAME` and `YTP_AUTH_PASSWORD` are set, authentication can be provided via: -1. **Via HTTP Headers** (automatic in browsers): +1. **Query Parameter** (recommended for WebSocket): + ``` + ws://localhost:8080/ws?apikey= + ``` + +2. **HTTP Basic Auth header** (during WebSocket handshake): ``` Authorization: Basic base64(":") ``` -2. **Via Query Parameter**: - ``` - ws://localhost:8081/socket.io/?apikey=:")> - ``` +--- ### Message Format -All WebSocket messages are JSON-encoded and follow a consistent structure: +All WebSocket messages use JSON format with `event` and `data` fields: -**Server-to-Client (Event)** - Emitted by server, received by client: +**Client → Server**: ```json { - "id": "unique-event-id", - "created_at": "2024-01-15T10:30:00.000000+00:00", - "event": "item_added", - "title": "Item Queued", - "message": "Video added to download queue", - "data": {...} + "event": "event_name", + "data": { /* payload */ } } ``` -### Core Events +**Server → Client**: +```json +{ + "event": "event_name", + "data": { /* Event object */ } +} +``` + +--- + +### Client Events (Client → Server) + +These events can be sent by the client to control downloads and queue operations. + +#### `add_url` + +Add a new URL to the download queue. + +**Request**: +```json +{ + "event": "add_url", + "data": { + "url": "https://youtube.com/watch?v=...", + "preset": "default", + "folder": "my_channel/foo", + "template": "%(title)s.%(ext)s", + "cookies": "...", + "cli": "--write-subs --embed-subs", + "auto_start": true + } +} +``` + +**Required Fields**: +- `url` - The video URL to download + +**Optional Fields**: +- `preset` - Preset name to use +- `folder` - Output folder relative to download_path +- `template` - Filename template +- `cookies` - Authentication cookies (Netscape format) +- `cli` - Additional yt-dlp CLI arguments +- `auto_start` - Whether to auto-start the download (default: true) + +**Response Events**: +- `item_status` - Item added successfully +- `log_error` - Error adding URL + +--- + +#### `item_cancel` + +Cancel an active or pending download. + +**Request**: +```json +{ + "event": "item_cancel", + "data": "item_id" +} +``` + +**Required**: Item ID (string) + +**Response Events**: +- `item_cancelled` - Item cancelled successfully +- `log_error` - Error cancelling item + +--- + +#### `item_delete` + +Delete a download item and optionally remove its files. + +**Request**: +```json +{ + "event": "item_delete", + "data": { + "id": "item_id", + "remove_file": true + } +} +``` + +**Required Fields**: +- `id` - The item ID to delete + +**Optional Fields**: +- `remove_file` - Whether to delete downloaded files (default: false) + +**Response Events**: +- `item_deleted` - Item deleted successfully +- `log_error` - Error deleting item + +--- + +#### `item_start` + +Start one or more paused download items. + +**Request** (single item): +```json +{ + "event": "item_start", + "data": "item_id" +} +``` + +**Request** (multiple items): +```json +{ + "event": "item_start", + "data": ["item_id1", "item_id2", "item_id3"] +} +``` + +**Required**: Item ID(s) - string or array of strings + +**Response Events**: +- `item_updated` - Items started successfully +- `log_error` - Error starting items + +--- + +#### `item_pause` + +Pause one or more active download items. + +**Request** (single item): +```json +{ + "event": "item_pause", + "data": "item_id" +} +``` + +**Request** (multiple items): +```json +{ + "event": "item_pause", + "data": ["item_id1", "item_id2", "item_id3"] +} +``` + +**Required**: Item ID(s) - string or array of strings + +**Response Events**: +- `item_updated` - Items paused successfully +- `log_error` - Error pausing items + +--- + +### Server Events (Server → Client) + +These events are emitted by the server and sent to connected WebSocket clients. #### Connection Events -##### `connect` (Built-in) -Fired when WebSocket connection is established. No data payload. +##### `config_update` -```typescript -socket.on('connect', () => console.log('WebSocket connected')); +Emitted when system configuration changes (presets, tasks, conditions, etc.). + +**Event**: +```json +{ + "event": "config_update", + "data": { + "feature": "presets|tasks|conditions|notifications|dl_fields", + "action": "create|update|delete", + "data": { /* Updated object */ } + } +} ``` -##### `disconnect` (Built-in) -Fired when WebSocket connection is closed. No data payload. +--- -```typescript -socket.on('disconnect', (reason: string) => console.log('WebSocket disconnected:', reason)); +##### `connected` + +Emitted when a client successfully connects to the WebSocket. + +**Event**: +```json +{ + "event": "connected", + "data": { + "sid": "session_id", + "timestamp": 1234567890.123 + } +} ``` -##### `configuration` (Server → Client) -Sends the current application configuration. +--- -**Data Fields**: -- `config`: Global configuration object -- `presets`: Available download presets -- `dl_fields`: Available download fields -- `paused`: Queue pause status (boolean) +##### `active_queue` -##### `connected` (Server → Client) -When a client connects, this events sends the folder and current queue. +Emitted periodically with the current active queue status. -**Data Fields**: -- `queue`: Current download queue (array of items) -- `folders`: Directory structure for downloads +**Event**: +```json +{ + "event": "active_queue", + "data": { + "queue": [ + { + "id": "abc123", + "status": "downloading", + "progress": 45.6, + ... + } + ] + } +} +``` + +--- #### Logging Events -All logging events follow the same structure with JSON-encoded message: +##### `log_info` -##### `log_info` (Server → Client) -General informational message. +Informational log message. -##### `log_success` (Server → Client) -Success notification message. - -##### `log_warning` (Server → Client) -Warning notification message. - -##### `log_error` (Server → Client) -Error notification message. - -##### `log_lines` (Server → Client) -Continuous application log lines (requires subscription event). - -**Data Fields**: -- `line`: Log line content -- `timestamp`: Log timestamp - -### Download Queue Events - -#### `item_added` (Server → Client) -Emitted when a new item is successfully added to the queue. The response payload is the complete item object. - -```typescript -socket.on('item_added', (stream: string) => { - const json = JSON.parse(stream); - const item = json.data; - console.log(`Added: ${item.title} [${item.url}]`); -}); +**Event**: +```json +{ + "event": "log_info", + "data": { + "title": "Info Title", + "message": "Informational message", + "timestamp": 1234567890.123 + } +} ``` -#### `item_updated` (Server → Client) -Emitted when an item's status or progress changes **(high-frequency event)**. +--- -**Data Fields**: Same as `item_added` +##### `log_success` -```typescript -socket.on('item_updated', (stream: string) => { - const json = JSON.parse(stream); - const item = json.data; - console.log(`Progress: ${item.title} - ${item.progress}%`); -}); +Success log message. + +**Event**: +```json +{ + "event": "log_success", + "data": { + "title": "Success Title", + "message": "Operation completed successfully", + "timestamp": 1234567890.123 + } +} ``` -#### `item_completed` (Server → Client) -Emitted when a download completes. Item moves from queue to history. +--- -**Data Fields**: Complete item object with final status +##### `log_warning` -```typescript -socket.on('item_completed', (stream: string) => { - const json = JSON.parse(stream); - const item = json.data; - console.log(`✓ Completed: ${item.title}`); - console.log(`Saved to: ${item.output_path}`); -}); +Warning log message. + +**Event**: +```json +{ + "event": "log_warning", + "data": { + "title": "Warning Title", + "message": "Warning message", + "timestamp": 1234567890.123 + } +} ``` -#### `item_cancelled` (Server → Client) -Emitted when a download is cancelled by user. +--- -**Data Fields**: Item object with status `cancelled` +##### `log_error` -```typescript -socket.on('item_cancelled', (stream: string) => { - const json = JSON.parse(stream); - const item = json.data; - console.log(`✗ Cancelled: ${item.title}`); -}); +Error log message. + +**Event**: +```json +{ + "event": "log_error", + "data": { + "title": "Error Title", + "message": "Error details", + "timestamp": 1234567890.123 + } +} ``` -#### `item_deleted` (Server → Client) -Emitted when an item is deleted from history. +--- -**Data Fields**: Item object +#### Download Queue Events -```typescript -socket.on('item_deleted', (stream: string) => { - const json = JSON.parse(stream); - console.log(`Deleted from history: ${json.data._id}`); -}); +##### `item_added` + +Emitted when a new item is added to the download queue. + +**Event**: +```json +{ + "event": "item_added", + "data": { + "id": "abc123", + "url": "https://example.com/video", + "title": "Video Title", + "status": "pending", + "preset": "default", + ... + } +} ``` -#### `item_moved` (Server → Client) -Emitted when an item moves between queue and history. +--- -**Data Fields**: -- `to`: Destination location (`queue` or `history`) -- `item`: Complete item object +##### `item_updated` -```typescript -socket.on('item_moved', (stream: string) => { - const json = JSON.parse(stream); - console.log(`Item moved to: ${json.data.to}`); -}); +Emitted when a download item's status or progress updates (high-frequency event). + +**Event**: +```json +{ + "event": "item_updated", + "data": { + "id": "abc123", + "status": "downloading", + "progress": 45.6, + "speed": "2.5 MiB/s", + "eta": "00:05:23", + ... + } +} ``` -### Terminal/CLI Events +--- -The terminal feature requires `YTP_CONSOLE_ENABLED=true`. +##### `item_cancelled` -#### `cli_post` (Client → Server) -Execute a command via yt-dlp CLI. +Emitted when a download is cancelled. -**Data**: Command arguments as string (without `yt-dlp` prefix) +**Event**: +```json +{ + "event": "item_cancelled", + "data": { + "id": "abc123", + "status": "cancelled", + ... + } +} +``` -#### `cli_output` (Server → Client) -Output line from the CLI command execution. +--- -**Data Fields**: -- `type`: Output type (`stdout` or `stderr`) -- `line`: Output line content +##### `item_deleted` -#### `cli_close` (Server → Client) -Emitted when CLI command execution completes. +Emitted when a download item is deleted from the queue or history. -**Data Fields**: -- `exitcode`: Command exit code (0 = success, non-zero = error) +**Event**: +```json +{ + "event": "item_deleted", + "data": { + "id": "abc123" + } +} +``` -### Configuration Events +--- -#### `presets_update` (Server → Client) -Emitted when download presets are updated or created. +##### `item_moved` -**Data**: Array of preset objects +Emitted when a download item is moved between queue and history. -#### `dlfields_update` (Server → Client) -Emitted when download fields configuration is updated. +**Event**: +```json +{ + "event": "item_moved", + "data": { + "id": "abc123", + "from": "queue", + "to": "done" + } +} +``` -**Data**: Array of field objects +--- + +##### `item_status` + +Emitted with status updates for specific operations. + +**Event**: +```json +{ + "event": "item_status", + "data": { + "id": "abc123", + "status": "queued", + "message": "Item added to queue", + ... + } +} +``` + +--- + +##### `paused` + +Emitted when the download queue is paused. + +**Event**: +```json +{ + "event": "paused", + "data": { + "paused": true, + "at": 1234567890.123 + } +} +``` + +--- + +##### `resumed` + +Emitted when the download queue is resumed. + +**Event**: +```json +{ + "event": "resumed", + "data": { + "paused": false, + "at": 1234567890.123 + } +} ``` --- diff --git a/FAQ.md b/FAQ.md index fd034fc6..f1332746 100644 --- a/FAQ.md +++ b/FAQ.md @@ -171,25 +171,25 @@ Then restart the container to apply the changes. # How can I monitor sites without RSS feeds? -YTPTube includes a **generic task handler** that turns JSON definition files into site-specific scrapers. You can use it +YTPTube includes a **generic task handler** that turns JSON definitions into site-specific scrapers. You can use it to watch pages that do not expose RSS or public APIs and automatically enqueue new links into the download queue. -1. Create definition files under `/config/tasks/*.json` (for Docker this is the mounted `config/tasks/` folder). -2. Keep your scheduled task in `tasks.json` pointing at the page you want to monitor and make sure it uses a preset that - enables a download archive (`--download-archive`). -3. When the task runs, the handler scans the JSON files, picks the first definition whose `match` rule covers the task - URL, fetches the page, extracts items, and queues the unseen ones. +1. Create definition via the WebUI > tasks > Definitions. +2. Create task that reference same url click on inspect to see the results. Make sure it uses a preset that enables + a download archive (`--download-archive`). +3. When the task handler run, the handler scans the definitions, picks the first definition whose `match` rule covers + the task URL, fetches the page, extracts items, and queues the unseen ones. ### Definition schema -Each file must contain a single JSON object with the following keys: +Each definition must contain a single JSON object with the following keys: ```json5 { "name": "example", // Friendly identifier shown in logs - "match": [ - "https://example.com/articles/*", // Glob strings, or objects with {"regex": "..."} or {"glob": "..."} - { "regex": "https://example.com/post/[0-9]+" } + "match_url": [ + "https://example.com/articles/*", // Glob strings + "https://example.com/post/[0-9]+" // Regex strings ], "engine": { // Optional, defaults to HTTPX "type": "httpx", // "httpx" (default) or "selenium" @@ -207,7 +207,7 @@ Each file must contain a single JSON object with the following keys: "headers": { "User-Agent": "MyAgent/1.0" }, "params": { "page": 1 }, "data": null, - "json": null, + "json_data": null, "timeout": 30 }, "response": { // Optional: how to interpret the body @@ -247,6 +247,7 @@ For JSON endpoints, switch the response format and use `jsonpath` selectors: ```json5 { + ... "response": { "type": "json" }, "parse": { "items": { @@ -282,9 +283,6 @@ For JSON endpoints, switch the response format and use `jsonpath` selectors: the moment. Optional keys: `arguments` (list or string), `wait_for` (type `css`/`xpath` + `expression`), `wait_timeout`, and `page_load_timeout`. -Definitions are reloaded automatically when files change, so you can tweak them without restarting YTPTube. Check -`var/config/tasks/01-*.json` for sample files. - > [!NOTE] > A machine-readable schema is available at `app/schema/task_definition.json` if you want to validate your JSON with editors or CI tools. diff --git a/app/features/conditions/__init__.py b/app/features/conditions/__init__.py new file mode 100644 index 00000000..c9f5d96b --- /dev/null +++ b/app/features/conditions/__init__.py @@ -0,0 +1 @@ +"""Conditions Feature""" diff --git a/app/features/conditions/deps.py b/app/features/conditions/deps.py new file mode 100644 index 00000000..81f5b1f2 --- /dev/null +++ b/app/features/conditions/deps.py @@ -0,0 +1,5 @@ +from app.features.conditions.repository import ConditionsRepository + + +def get_conditions_repo() -> ConditionsRepository: + return ConditionsRepository.get_instance() diff --git a/app/features/conditions/migration.py b/app/features/conditions/migration.py new file mode 100644 index 00000000..8b973486 --- /dev/null +++ b/app/features/conditions/migration.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +from app.features.conditions.models import ConditionModel +from app.features.core.migration import Migration as FeatureMigration +from app.library.config import Config + +if TYPE_CHECKING: + from app.features.conditions.repository import ConditionsRepository + +LOG: logging.Logger = logging.getLogger(__name__) + + +class Migration(FeatureMigration): + name: str = "conditions" + + def __init__(self, repo: ConditionsRepository, config: Config | None = None): + self._config: Config = config or Config.get_instance() + super().__init__(config=self._config) + self._repo: ConditionsRepository = repo + self._source_file: Path = Path(self._config.config_path) / "conditions.json" + + async def should_run(self) -> bool: + return self._source_file.exists() + + async def migrate(self) -> None: + if await self._repo.count() > 0: + LOG.warning("Conditions already exist in the database; skipping migration.") + await self._move_file(self._source_file) + return + + try: + items: list[dict] | None = json.loads(self._source_file.read_text()) + except Exception as exc: + LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc) + await self._move_file(self._source_file) + return + + if items is None: + LOG.warning("No conditions found in %s; skipping migration.", self._source_file) + await self._move_file(self._source_file) + return + + inserted = 0 + seen_names: dict[str, int] = {} + for index, item in enumerate(items): + if not (normalized := await self._normalize(item, index, seen_names)): + continue + try: + await self._repo.create(normalized) + inserted += 1 + except Exception as exc: + LOG.exception("Failed to insert condition '%s': %s", normalized.name, exc) + + LOG.info("Migrated %s condition(s) from %s.", inserted, self._source_file) + await self._move_file(self._source_file) + + async def _normalize(self, item: Any, index: int, seen_names: dict[str, int]) -> ConditionModel | None: + if not isinstance(item, dict): + LOG.warning("Skipping condition at index %s due to invalid type.", index) + return None + + assert isinstance(item, dict) + name: str | None = item.get("name") + if not name or not isinstance(name, str): + LOG.warning("Skipping condition at index %s due to missing name.", index) + return None + + normalized_name = name.strip() + if not normalized_name: + LOG.warning("Skipping condition at index %s due to empty name.", index) + return None + + name = self._unique_name(normalized_name, seen_names) + + extras = item.get("extras") if isinstance(item.get("extras"), dict) else {} + extras = cast("dict[str, Any]", extras) + filter_value: str | None = item.get("filter") + if (not filter_value or not isinstance(filter_value, str)) and len(extras) == 0: + LOG.warning("Skipping condition '%s' due to missing filter.", name) + return None + + cli: str | None = item.get("cli") + if not isinstance(cli, str): + cli = "" + + enabled_value = item.get("enabled") + enabled: bool = enabled_value if isinstance(enabled_value, bool) else True + priority_value = item.get("priority") + priority: int = priority_value if isinstance(priority_value, int) else 0 + if isinstance(priority, bool) or 0 > priority: + priority = 0 + + description_value = item.get("description") + description: str = description_value if isinstance(description_value, str) else "" + + return ConditionModel( + id=index, + name=name, + filter=filter_value, + cli=cli, + extras=extras, + enabled=bool(enabled), + priority=priority, + description=description, + ) diff --git a/app/features/conditions/models.py b/app/features/conditions/models.py new file mode 100644 index 00000000..088cb0b2 --- /dev/null +++ b/app/features/conditions/models.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from datetime import datetime # noqa: TC003 + +from sqlalchemy import JSON, Boolean, Index, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.features.core.models import Base, UTCDateTime, utcnow + + +class ConditionModel(Base): + __tablename__: str = "conditions" + __table_args__: tuple[Index, ...] = ( + Index("ix_conditions_name", "name"), + Index("ix_conditions_enabled", "enabled"), + Index("ix_conditions_priority", "priority"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + filter: Mapped[str] = mapped_column(Text, nullable=False) + cli: Mapped[str] = mapped_column(Text, nullable=False, default="") + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + description: Mapped[str] = mapped_column(Text, nullable=False, default="") + extras: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False) + updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False) diff --git a/app/features/conditions/repository.py b/app/features/conditions/repository.py new file mode 100644 index 00000000..069987cf --- /dev/null +++ b/app/features/conditions/repository.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from app.features.conditions.migration import Migration +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Iterable + + from sqlalchemy.engine.result import Result + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + from sqlalchemy.sql.selectable import Select + +from sqlalchemy import delete, func, or_, select + +from app.features.conditions.models import ConditionModel +from app.features.core.deps import get_session + +LOG: logging.Logger = logging.getLogger(__name__) + + +class ConditionsRepository(metaclass=Singleton): + def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + self._migrated = False + self.session: AsyncGenerator[AsyncSession] = session or get_session + + async def run_migrations(self) -> None: + if self._migrated: + return + + self._migrated = True + await Migration(repo=self, config=None).run() + + @staticmethod + def get_instance() -> ConditionsRepository: + return ConditionsRepository() + + async def list(self) -> list[ConditionModel]: + async with self.session() as session: + result: Result[tuple[ConditionModel]] = await session.execute( + select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc()) + ) + return list(result.scalars().all()) + + async def list_paginated(self, page: int, per_page: int) -> tuple[list[ConditionModel], int, int, int]: + async with self.session() as session: + total: int = await self.count() + total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1 + + if page > total_pages and total > 0: + page = total_pages + + query: Select[tuple[ConditionModel]] = ( + select(ConditionModel) + .order_by(ConditionModel.priority.desc(), ConditionModel.name.asc()) + .limit(per_page) + .offset((page - 1) * per_page) + ) + result: Result[tuple[ConditionModel]] = await session.execute(query) + return list(result.scalars().all()), total, page, total_pages + + async def count(self) -> int: + async with self.session() as session: + result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(ConditionModel)) + return int(result.scalar_one()) + + async def get(self, identifier: int | str) -> ConditionModel | None: + async with self.session() as session: + if not identifier: + return None + + if isinstance(identifier, int): + clause: ColumnElement[bool] = ConditionModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause: ColumnElement[bool] = or_( + ConditionModel.id == int(identifier), ConditionModel.name == identifier + ) + else: + clause: ColumnElement[bool] = ConditionModel.name == identifier + + result: Result[tuple[ConditionModel]] = await session.execute(select(ConditionModel).where(clause).limit(1)) + return result.scalar_one_or_none() + + async def get_by_name(self, name: str, exclude_id: int | None = None) -> ConditionModel | None: + async with self.session() as session: + query: Select[tuple[ConditionModel]] = select(ConditionModel).where(ConditionModel.name == name) + if exclude_id is not None: + query = query.where(ConditionModel.id != exclude_id) + + result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1)) + return result.scalar_one_or_none() + + async def create(self, payload: ConditionModel | dict) -> ConditionModel: + async with self.session() as session: + model: ConditionModel = ConditionModel(**payload) if isinstance(payload, dict) else payload + if model.id is not None: + model.id = None + + if await self.get_by_name(name=model.name) is not None: + msg: str = f"Condition with name '{model.name}' already exists." + raise ValueError(msg) + + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def update(self, identifier: int | str, payload: dict[str, Any]) -> ConditionModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = ConditionModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause: ColumnElement[bool] = or_( + ConditionModel.id == int(identifier), ConditionModel.name == identifier + ) + else: + clause: ColumnElement[bool] = ConditionModel.name == identifier + + result: Result[tuple[ConditionModel]] = await session.execute(select(ConditionModel).where(clause).limit(1)) + if not (model := result.scalar_one_or_none()): + msg: str = f"Condition '{identifier}' not found." + raise KeyError(msg) + + payload.pop("id", None) + payload.pop("created_at", None) + payload.pop("updated_at", None) + + if "name" in payload and await self.get_by_name(name=payload["name"], exclude_id=model.id) is not None: + msg: str = f"Condition with name '{payload['name']}' already exists." + raise ValueError(msg) + + for key, value in payload.items(): + if hasattr(model, key): + setattr(model, key, value) + + await session.commit() + await session.refresh(model) + return model + + async def delete(self, identifier: int | str) -> ConditionModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = ConditionModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause: ColumnElement[bool] = or_( + ConditionModel.id == int(identifier), ConditionModel.name == identifier + ) + else: + clause: ColumnElement[bool] = ConditionModel.name == identifier + + result: Result[tuple[ConditionModel]] = await session.execute(select(ConditionModel).where(clause).limit(1)) + if not (model := result.scalar_one_or_none()): + msg: str = f"Condition '{identifier}' not found." + raise KeyError(msg) + + await session.delete(model) + await session.commit() + return model + + async def replace_all(self, items: Iterable[dict | ConditionModel]) -> list[ConditionModel]: + async with self.session() as session: + try: + await session.execute(delete(ConditionModel)) + models: list[ConditionModel] = [ + ConditionModel(**item) if isinstance(item, dict) else item for item in items + ] + session.add_all(models) + await session.commit() + except Exception: + await session.rollback() + raise + return models diff --git a/app/features/conditions/router.py b/app/features/conditions/router.py new file mode 100644 index 00000000..b432c82f --- /dev/null +++ b/app/features/conditions/router.py @@ -0,0 +1,328 @@ +import logging +from collections import OrderedDict +from typing import Any + +from aiohttp import web +from aiohttp.web import Request, Response +from pydantic import ValidationError + +from app.features.conditions.schemas import Condition, ConditionList, ConditionPatch +from app.features.conditions.service import Conditions +from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination +from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination +from app.library.cache import Cache +from app.library.config import Config +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events +from app.library.router import route + +LOG: logging.Logger = logging.getLogger(__name__) + + +def _model(model: Any) -> Condition: + return Condition.model_validate(model) + + +def _serialize(model: Any) -> dict: + return _model(model).model_dump() + + +@route("GET", "api/conditions/", name="conditions_list") +async def conditions_list(request: Request, encoder: Encoder) -> Response: + """ + Get the conditions + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + repo = Conditions.get_instance()._repo + + page, per_page = normalize_pagination(request) + items, total, current_page, total_pages = await repo.list_paginated(page, per_page) + return web.json_response( + data=ConditionList( + items=[_model(model) for model in items], + pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)), + ), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", "api/conditions/test/", name="condition_test") +async def conditions_test(request: Request, encoder: Encoder, cache: Cache, config: Config) -> Response: + """ + Test condition against URL. + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + cache (Cache): The cache instance. + config (Config): The configuration instance. + + Returns: + Response: The response object + + """ + params = await request.json() + + if not isinstance(params, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + if not (url := params.get("url")): + return web.json_response({"error": "url is required."}, status=web.HTTPBadRequest.status_code) + + if not (cond := params.get("condition")): + return web.json_response({"error": "condition is required."}, status=web.HTTPBadRequest.status_code) + + try: + preset: str = params.get("preset", config.default_preset) + key: str = cache.hash(url + str(preset)) + if not cache.has(key): + from app.library.Utils import fetch_info + from app.library.YTDLPOpts import YTDLPOpts + + data: dict | None = await fetch_info( + config=YTDLPOpts.get_instance().preset(name=preset).get_all(), + url=url, + debug=False, + no_archive=True, + follow_redirect=True, + sanitize_info=True, + ) + + if not data: + return web.json_response( + data={"error": f"Failed to extract info from '{url!s}'."}, + status=web.HTTPBadRequest.status_code, + ) + cache.set(key=key, value=data, ttl=600) + else: + data = cache.get(key) + except Exception as e: + LOG.exception(e) + return web.json_response( + data={"error": f"Failed to extract video info. '{e!s}'"}, + status=web.HTTPInternalServerError.status_code, + ) + + try: + from app.library.mini_filter import match_str + + status: bool = match_str(cond, data) + except Exception as e: + LOG.exception(e) + return web.json_response( + data={"error": str(e)}, + status=web.HTTPBadRequest.status_code, + ) + + return web.json_response( + data={ + "status": status, + "condition": cond, + "data": OrderedDict(sorted(data.items(), key=lambda item: len(str(item[1])))), + }, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", "api/conditions/", name="condition_add") +async def conditions_add(request: Request, encoder: Encoder, notify: EventBus) -> Response: + """ + Add Condition. + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + notify (EventBus): The event bus instance. + + Returns: + Response: The response object + + """ + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + item: Condition = Condition.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate condition.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + try: + saved = _serialize(await Conditions.get_instance().save(item=item.model_dump())) + except ValueError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code) + + notify.emit( + Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.CONDITIONS, action=CEAction.CREATE, data=saved) + ) + return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("GET", r"api/conditions/{id:\d+}", name="condition_get") +async def conditions_get(request: Request, encoder: Encoder) -> Response: + """ + Get the conditions + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + if not (id := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await Conditions.get_instance().get(id)): + return web.json_response({"error": "Condition not found"}, status=web.HTTPNotFound.status_code) + + return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("DELETE", r"api/conditions/{id:\d+}", name="condition_delete") +async def conditions_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response: + """ + Delete Condition. + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + notify (EventBus): The event bus instance. + + Returns: + Response: The response object. + + """ + if not (id := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + try: + deleted = _serialize(await Conditions.get_instance()._repo.delete(id)) + notify.emit( + Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.CONDITIONS, action=CEAction.DELETE, data=deleted) + ) + return web.json_response(data=deleted, status=web.HTTPOk.status_code, dumps=encoder.encode) + except KeyError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code) + + +@route("PATCH", r"api/conditions/{id:\d+}", name="condition_patch") +async def conditions_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response: + """ + Patch Condition. + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + notify (EventBus): The event bus instance. + + Returns: + Response: The response object. + + """ + if not (id := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await Conditions.get_instance().get(id)): + return web.json_response({"error": "Condition not found"}, status=web.HTTPNotFound.status_code) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + service = Conditions.get_instance() + + try: + validated = ConditionPatch.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate condition.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if validated.name and await service._repo.get_by_name(validated.name, exclude_id=model.id): + return web.json_response( + data={"error": f"Condition with name '{validated.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + updated = _serialize(await service._repo.update(model.id, validated.model_dump(exclude_unset=True))) + notify.emit( + Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.CONDITIONS, action=CEAction.UPDATE, data=updated) + ) + return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("PUT", r"api/conditions/{id:\d+}", name="condition_update") +async def conditions_update(request: Request, encoder: Encoder, notify: EventBus) -> Response: + """ + Update Condition. + + Args: + request (Request): The request object. + encoder (Encoder): The encoder instance. + notify (EventBus): The event bus instance. + + Returns: + Response: The response object. + + """ + if not (id := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await Conditions.get_instance().get(id)): + return web.json_response({"error": "Condition not found"}, status=web.HTTPNotFound.status_code) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting list with dicts."}, + status=web.HTTPBadRequest.status_code, + ) + + service = Conditions.get_instance() + + try: + validated = Condition.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate condition.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if validated.name and await service._repo.get_by_name(validated.name, exclude_id=model.id): + return web.json_response( + data={"error": f"Condition with name '{validated.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + updated = _serialize(await service._repo.update(model.id, validated.model_dump(exclude_unset=True))) + notify.emit( + Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.CONDITIONS, action=CEAction.UPDATE, data=updated) + ) + return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode) diff --git a/app/features/conditions/schemas.py b/app/features/conditions/schemas.py new file mode 100644 index 00000000..b2abdfff --- /dev/null +++ b/app/features/conditions/schemas.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from app.features.core.schemas import Pagination +from app.features.core.utils import parse_int +from app.library.mini_filter import match_str +from app.library.Utils import arg_converter + + +class Condition(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True) + + id: int | None = None + name: str = Field(min_length=1) + filter: str = "" + cli: str = "" + extras: dict[str, Any] = Field(default_factory=dict) + enabled: bool = True + priority: int = 0 + description: str = "" + + @field_validator("filter") + @classmethod + def _validate_filter(cls, value: str) -> str: + try: + match_str(value, {}) + except Exception as exc: + msg: str = f"Invalid filter. '{exc!s}'." + raise ValueError(msg) from exc + return value + + @field_validator("cli") + @classmethod + def _validate_cli(cls, value: str) -> str: + if not value: + return "" + try: + arg_converter(args=value) + except ModuleNotFoundError: + return value + except Exception as exc: + msg: str = f"Invalid command options for yt-dlp. '{exc!s}'." + raise ValueError(msg) from exc + return value + + @field_validator("priority", mode="before") + @classmethod + def _normalize_priority(cls, value: Any) -> int: + return parse_int(value, field="Priority", minimum=0) + + @field_validator("enabled", mode="before") + @classmethod + def _validate_enabled(cls, value: Any) -> bool: + if not isinstance(value, bool): + msg: str = "Enabled must be a boolean." + raise ValueError(msg) + return value + + @model_validator(mode="after") + def _validate_cli_or_extras(self) -> Condition: + if not self.cli and not self.extras: + msg: str = "Either cli or extras must be set." + raise ValueError(msg) + return self + + +class ConditionPatch(Condition): + """ + Model for patching Condition fields. All fields are optional. + """ + + name: str | None = None + filter: str | None = None + cli: str | None = None + extras: dict[str, Any] | None = None + enabled: bool | None = None + priority: int | None = None + description: str | None = None + + @model_validator(mode="after") + def _validate_cli_or_extras(self) -> ConditionPatch: + return self + + +class ConditionList(BaseModel): + model_config = ConfigDict(from_attributes=True) + + items: list[Condition] = Field(default_factory=list) + pagination: Pagination diff --git a/app/features/conditions/service.py b/app/features/conditions/service.py new file mode 100644 index 00000000..1ede3af8 --- /dev/null +++ b/app/features/conditions/service.py @@ -0,0 +1,147 @@ +import logging + +from aiohttp import web + +from app.features.conditions.models import ConditionModel +from app.features.conditions.repository import ConditionsRepository +from app.library.Events import EventBus, Events +from app.library.mini_filter import match_str +from app.library.Singleton import Singleton + +LOG: logging.Logger = logging.getLogger("feature.conditions") + + +class Conditions(metaclass=Singleton): + def __init__(self): + self._repo: ConditionsRepository = ConditionsRepository.get_instance() + + @staticmethod + def get_instance() -> "Conditions": + return Conditions() + + async def on_shutdown(self, _: web.Application): + pass + + def attach(self, _: web.Application): + async def handle_event(_, __): + await self._repo.run_migrations() + + EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations") + + async def get_all(self) -> list[ConditionModel]: + return await self._repo.list() + + async def save(self, item: ConditionModel | dict) -> ConditionModel: + """ + Save the item. + + Args: + item (Condition|dict): The items to save. + + Returns: + ConditionModel: The current instance. + + """ + try: + if not isinstance(item, ConditionModel): + item: ConditionModel = ConditionModel(**item) + except Exception as exc: + msg: str = f"Failed to parse item. '{exc!s}'" + raise ValueError(msg) from exc + + try: + repo = self._repo + if item.id is None or 0 == item.id: + model = await repo.create(item) + else: + model = await repo.update(item.id, item.serialize()) + except KeyError as exc: + raise ValueError(str(exc)) from exc + + return model + + async def has(self, identifier: str) -> bool: + """ + Check if the item exists by id or name. + + Args: + identifier (str): The id or name of the preset. + + Returns: + bool: True if the item exists, False otherwise. + + """ + return await self.get(identifier) is not None + + async def get(self, identifier: str) -> ConditionModel | None: + """ + Get the item by id or name. + + Args: + identifier (str): The id or name of the preset. + + Returns: + ConditionModel|None: The item if found, None otherwise. + + """ + repo = self._repo + return await repo.get(identifier) + + async def match(self, info: dict) -> ConditionModel | None: + """ + Check if any condition matches the info dict. + + Args: + info (dict): The info dict to check. + + Returns: + Condition|None: The condition if found, None otherwise. + + """ + if not info or not isinstance(info, dict) or len(info) < 1: + return None + + repo = self._repo + items: list[ConditionModel] = await repo.list() + if len(items) < 1: + return None + + for item in sorted(items, key=lambda x: x.priority, reverse=True): + if not item.enabled: + continue + + if not item.filter: + LOG.error(f"Filter is empty for '{item.name}'.") + continue + + try: + if not match_str(item.filter, info): + continue + + LOG.debug(f"Matched '{item.id}: {item.name}' with filter '{item.filter}'.") + return item + except Exception as e: + LOG.error(f"Failed to evaluate '{item.id}: {item.name}'. '{e!s}'.") + continue + + return None + + async def single_match(self, identifier: int | str, info: dict) -> ConditionModel | None: + """ + Check if condition matches the info dict. + + Args: + identifier (int|str): The id or name of the item. + info (dict): The info dict to check. + + Returns: + ConditionModel|None: The condition if found, None otherwise. + + """ + if not info or not isinstance(info, dict) or len(info) < 1: + return None + + if not (item := await self.get(identifier)) or not item.enabled or not item.filter: + return None + + return item if match_str(item.filter, info) else None diff --git a/app/features/conditions/tests/test_conditions_repository.py b/app/features/conditions/tests/test_conditions_repository.py new file mode 100644 index 00000000..0b030ab9 --- /dev/null +++ b/app/features/conditions/tests/test_conditions_repository.py @@ -0,0 +1,226 @@ +"""Tests for ConditionsRepository.""" + +from __future__ import annotations + +import pytest +import pytest_asyncio + +from app.features.conditions.models import ConditionModel +from app.features.conditions.repository import ConditionsRepository +from app.library.sqlite_store import SqliteStore + + +@pytest_asyncio.fixture +async def repo(): + ConditionsRepository._reset_singleton() + SqliteStore._reset_singleton() + + store = SqliteStore(db_path=":memory:") + await store.get_connection() + + # Create repository + repository = ConditionsRepository.get_instance() + + yield repository + + # Cleanup - close connections properly + if store._conn: + await store._conn.close() + if store._engine: + await store._engine.dispose() + + # Reset singletons + ConditionsRepository._reset_singleton() + SqliteStore._reset_singleton() + + +class TestConditionsRepository: + """Test suite for ConditionsRepository database operations.""" + + @pytest.mark.asyncio + async def test_repository_singleton(self, repo): + """Verify repository follows singleton pattern.""" + instance1 = ConditionsRepository.get_instance() + instance2 = ConditionsRepository.get_instance() + assert instance1 is instance2, "Should return same singleton instance" + + @pytest.mark.asyncio + async def test_list_empty(self, repo): + """List returns empty when no conditions exist.""" + conditions = await repo.list() + assert conditions == [], "Should return empty list when no conditions" + + @pytest.mark.asyncio + async def test_count_empty(self, repo): + """Count returns 0 when no conditions exist.""" + count = await repo.count() + assert count == 0, "Should return 0 when no conditions exist" + + @pytest.mark.asyncio + async def test_create_condition(self, repo): + """Create condition with valid data.""" + data = { + "name": "Test Condition", + "filter": "duration > 60", + "cli": "--format best", + "enabled": True, + "priority": 10, + "description": "Test description", + "extras": {"key": "value"}, + } + + model = await repo.create(data) + + assert model.id is not None, "Should generate ID for new condition" + assert model.name == "Test Condition", "Should store name correctly" + assert model.filter == "duration > 60", "Should store filter correctly" + assert model.cli == "--format best", "Should store CLI correctly" + assert model.enabled is True, "Should store enabled flag correctly" + assert model.priority == 10, "Should store priority correctly" + assert model.description == "Test description", "Should store description correctly" + assert model.extras == {"key": "value"}, "Should store extras as dict" + + @pytest.mark.asyncio + async def test_create_with_defaults(self, repo): + """Create condition with minimal data uses defaults.""" + data = { + "name": "Minimal", + "filter": "duration > 30", + } + + model = await repo.create(data) + + assert model.cli == "", "Should default CLI to empty string" + assert model.enabled is True, "Should default enabled to True" + assert model.priority == 0, "Should default priority to 0" + assert model.description == "", "Should default description to empty" + assert model.extras == {}, "Should default extras to empty dict" + + @pytest.mark.asyncio + async def test_get_by_id(self, repo): + """Get condition by integer ID.""" + created = await repo.create({"name": "Get Test", "filter": "duration > 40"}) + + retrieved = await repo.get(created.id) + + assert retrieved is not None, "Should retrieve created condition" + assert retrieved.id == created.id, "Should retrieve correct condition by ID" + assert retrieved.name == "Get Test", "Should match created condition name" + + @pytest.mark.asyncio + async def test_get_by_name(self, repo): + """Get condition by string name.""" + await repo.create({"name": "Named Test", "filter": "duration > 50"}) + + retrieved = await repo.get("Named Test") + + assert retrieved is not None, "Should retrieve by name" + assert retrieved.name == "Named Test", "Should match condition name" + + @pytest.mark.asyncio + async def test_get_nonexistent(self, repo): + """Get nonexistent condition returns None.""" + result = await repo.get(99999) + assert result is None, "Should return None for nonexistent ID" + + result = await repo.get("nonexistent") + assert result is None, "Should return None for nonexistent name" + + @pytest.mark.asyncio + async def test_update_condition(self, repo): + """Update existing condition.""" + created = await repo.create({"name": "Update Test", "filter": "duration > 60"}) + + updated = await repo.update( + created.id, + { + "name": "Updated Name", + "priority": 5, + "extras": {"updated": True}, + }, + ) + + assert updated.name == "Updated Name", "Should update name" + assert updated.priority == 5, "Should update priority" + assert updated.extras == {"updated": True}, "Should update extras" + assert updated.filter == "duration > 60", "Should preserve unchanged filter" + + @pytest.mark.asyncio + async def test_update_nonexistent_raises(self, repo): + """Update nonexistent condition raises KeyError.""" + with pytest.raises(KeyError, match="not found"): + await repo.update(99999, {"name": "Should Fail"}) + + @pytest.mark.asyncio + async def test_delete_condition(self, repo): + """Delete existing condition.""" + created = await repo.create({"name": "Delete Test", "filter": "duration > 70"}) + + deleted = await repo.delete(created.id) + + assert deleted.id == created.id, "Should return deleted condition" + + result = await repo.get(created.id) + assert result is None, "Deleted condition should not be retrievable" + + @pytest.mark.asyncio + async def test_delete_nonexistent_raises(self, repo): + """Delete nonexistent condition raises KeyError.""" + with pytest.raises(KeyError, match="not found"): + await repo.delete(99999) + + @pytest.mark.asyncio + async def test_list_paginated(self, repo): + """List paginated returns correct subset.""" + for i in range(5): + await repo.create({"name": f"Item {i}", "filter": "duration > 10", "priority": i}) + + items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2) + + assert len(items) == 2, "Should return 2 items per page" + assert total == 5, "Should report total count of 5" + assert page == 1, "Should be on page 1" + assert total_pages == 3, "Should have 3 pages total" + + @pytest.mark.asyncio + async def test_list_ordering(self, repo): + """List orders by priority desc then name asc.""" + await repo.create({"name": "B", "filter": "test", "priority": 1}) + await repo.create({"name": "A", "filter": "test", "priority": 1}) + await repo.create({"name": "C", "filter": "test", "priority": 2}) + + items = await repo.list() + + assert items[0].name == "C", "Highest priority should be first" + assert items[1].name == "A", "Same priority sorted alphabetically" + assert items[2].name == "B", "Same priority sorted alphabetically" + + @pytest.mark.asyncio + async def test_get_by_name_excludes_id(self, repo): + """Get by name can exclude specific ID.""" + first = await repo.create({"name": "Duplicate", "filter": "test"}) + + result = await repo.get_by_name("Duplicate", exclude_id=first.id) + assert result is None, "Should not find when excluding only match" + + result = await repo.get_by_name("Duplicate", exclude_id=None) + assert result is not None, "Should find without exclusion" + + @pytest.mark.asyncio + async def test_replace_all(self, repo): + """Replace all conditions atomically.""" + await repo.create({"name": "Old 1", "filter": "test"}) + await repo.create({"name": "Old 2", "filter": "test"}) + + new_items = [ + {"name": "New 1", "filter": "duration > 10"}, + {"name": "New 2", "filter": "duration > 20"}, + ] + + result = await repo.replace_all(new_items) + + assert len(result) == 2, "Should create 2 new conditions" + + all_items = await repo.list() + assert len(all_items) == 2, "Should only have new conditions" + assert all_items[0].name in ["New 1", "New 2"], "Should only have new items" diff --git a/app/features/core/deps.py b/app/features/core/deps.py new file mode 100644 index 00000000..6e468dbe --- /dev/null +++ b/app/features/core/deps.py @@ -0,0 +1,12 @@ +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.library.sqlite_store import SqliteStore + + +@asynccontextmanager +async def get_session() -> AsyncGenerator[AsyncSession]: + async with SqliteStore.get_instance().sessionmaker()() as session: + yield session diff --git a/app/features/core/migration.py b/app/features/core/migration.py new file mode 100644 index 00000000..a323e21a --- /dev/null +++ b/app/features/core/migration.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import abc +import logging +import time +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from app.library.config import Config + +LOG: logging.Logger = logging.getLogger(__name__) + + +class Migration(abc.ABC): + name: str = "" + + def __init__(self, config: Config): + self._migrated_dir: Path = Path(config.config_path) / "migrated" + + async def run(self) -> bool: + if not await self.should_run(): + return False + + self._migrated_dir.mkdir(parents=True, exist_ok=True) + try: + await self.migrate() + except Exception as exc: + LOG.exception("Feature migration '%s' failed: %s", self.name, exc) + return False + + return True + + @abc.abstractmethod + async def should_run(self) -> bool: + raise NotImplementedError + + @abc.abstractmethod + async def migrate(self) -> None: + raise NotImplementedError + + async def _move_file(self, source: Path) -> Path: + destination: Path = self._migrated_dir / source.name + if destination.exists(): + timestamp = int(time.time()) + destination: Path = self._migrated_dir / f"{source.stem}_{timestamp}{source.suffix}" + + source.rename(destination) + return destination + + def _unique_name(self, name: str, seen_names: dict[str, int]) -> str: + base = name + count = seen_names.get(base, 0) + if count == 0: + seen_names[base] = 1 + return base + + suffix = count + 1 + new_name = f"{base} ({suffix})" + while new_name in seen_names: + suffix += 1 + new_name = f"{base} ({suffix})" + + seen_names[base] = suffix + seen_names[new_name] = 1 + return new_name diff --git a/app/features/core/models.py b/app/features/core/models.py new file mode 100644 index 00000000..a09d8160 --- /dev/null +++ b/app/features/core/models.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from datetime import UTC, datetime + +from sqlalchemy import DateTime as SQLADateTime +from sqlalchemy import TypeDecorator +from sqlalchemy.orm import DeclarativeBase + + +def utcnow() -> datetime: + """ + Return current UTC time as timezone-aware datetime. + + This is the canonical way to get current UTC time for database fields. + """ + return datetime.now(tz=UTC) + + +class UTCDateTime(TypeDecorator): + impl = SQLADateTime + cache_ok = True + + def process_bind_param(self, value: datetime | None, _dialect) -> datetime | None: + """Convert datetime to UTC before storing.""" + if value is not None: + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC).replace(tzinfo=None) + return value + + def process_result_value(self, value: datetime | None, _dialect) -> datetime | None: + """Ensure datetime is timezone-aware (UTC) when loading.""" + if value is not None and value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value + + +class Base(DeclarativeBase): + pass diff --git a/app/features/core/schemas.py b/app/features/core/schemas.py new file mode 100644 index 00000000..6da489f5 --- /dev/null +++ b/app/features/core/schemas.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field + + +class Pagination(BaseModel): + page: int + per_page: int + total: int + total_pages: int + has_next: bool + has_prev: bool + + +class CEFeature(str, Enum): + PRESETS = "presets" + DL_FIELDS = "dl_fields" + CONDITIONS = "conditions" + NOTIFICATIONS = "notifications" + TASKS = "tasks" + TASKS_DEFINITIONS = "tasks_definitions" + + def __str__(self) -> str: + return self.value + + +class CEAction(str, Enum): + CREATE = "create" + UPDATE = "update" + DELETE = "delete" + REPLACE = "replace" + + def __str__(self) -> str: + return self.value + + +class ConfigEvent(BaseModel): + feature: CEFeature + action: CEAction + data: dict[str, Any] | list[dict[str, Any]] + extras: dict[str, Any] = Field(default_factory=dict) diff --git a/app/features/core/utils.py b/app/features/core/utils.py new file mode 100644 index 00000000..6e2faa58 --- /dev/null +++ b/app/features/core/utils.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from aiohttp.web import Request + from pydantic import ValidationError + + +def parse_int(value: Any, *, field: str, minimum: int | None = None) -> int: + try: + parsed = int(value) + except Exception as exc: + msg = f"{field} must be an integer." + raise ValueError(msg) from exc + + if minimum is not None and parsed < minimum: + msg = f"{field} must be >= {minimum}." + raise ValueError(msg) + + return parsed + + +def normalize_pagination(request: Request, page: int | None = None, per_page: int | None = None) -> tuple[int, int]: + if page is None: + page = int(request.query.get("page", 1)) + + if per_page is None: + per_page = int(request.query.get("per_page", 0)) + + if 0 == per_page: + from app.library.config import Config + + items_per_page: int = int(Config.get_instance().default_pagination) + per_page = items_per_page // 2 if items_per_page > 50 else items_per_page + + if page < 1: + msg = "page must be >= 1." + raise ValueError(msg) + + if per_page < 1 or per_page > 1000: + msg = "per_page must be between 1 and 1000." + raise ValueError(msg) + + return int(page), int(per_page) + + +def build_pagination(total: int, page: int, per_page: int, total_pages: int) -> dict: + return { + "page": page, + "per_page": per_page, + "total": total, + "total_pages": total_pages, + "has_next": page < total_pages, + "has_prev": page > 1, + } + + +def format_validation_errors(exc: ValidationError) -> list[dict[str, Any]]: + """ + Format Pydantic ValidationError. + + Args: + exc: The ValidationError from Pydantic + + Returns: + List of dicts with loc, msg, and type + + """ + return [ + { + "loc": list(error.get("loc", [])), + "msg": error.get("msg", "Validation error"), + "type": error.get("type", "value_error"), + } + for error in exc.errors() + ] + + +def gen_random(length: int = 16) -> str: + import secrets + + return "".join(secrets.token_urlsafe(length)[:length]) diff --git a/app/features/dl_fields/__init__.py b/app/features/dl_fields/__init__.py new file mode 100644 index 00000000..3a0532ca --- /dev/null +++ b/app/features/dl_fields/__init__.py @@ -0,0 +1 @@ +"""DL Fields Feature""" diff --git a/app/features/dl_fields/deps.py b/app/features/dl_fields/deps.py new file mode 100644 index 00000000..3cb9af30 --- /dev/null +++ b/app/features/dl_fields/deps.py @@ -0,0 +1,5 @@ +from app.features.dl_fields.repository import DLFieldsRepository + + +def get_dl_fields_repo() -> DLFieldsRepository: + return DLFieldsRepository.get_instance() diff --git a/app/features/dl_fields/migration.py b/app/features/dl_fields/migration.py new file mode 100644 index 00000000..2673c0de --- /dev/null +++ b/app/features/dl_fields/migration.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +from app.features.core.migration import Migration as FeatureMigration +from app.features.dl_fields.schemas import DLField +from app.library.config import Config + +if TYPE_CHECKING: + from app.features.dl_fields.repository import DLFieldsRepository + +LOG: logging.Logger = logging.getLogger(__name__) + + +class Migration(FeatureMigration): + name: str = "dl_fields" + + def __init__(self, repo: DLFieldsRepository, config: Config | None = None): + self._config: Config = config or Config.get_instance() + super().__init__(config=self._config) + self._repo: DLFieldsRepository = repo + self._source_file: Path = Path(self._config.config_path) / "dl_fields.json" + + async def should_run(self) -> bool: + return self._source_file.exists() + + async def migrate(self) -> None: + if await self._repo.count() > 0: + LOG.warning("DL fields already exist in the database; skipping migration.") + await self._move_file(self._source_file) + return + + try: + items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text()) + except Exception as exc: + LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc) + await self._move_file(self._source_file) + return + + if items is None: + LOG.warning("No dl fields found in %s; skipping migration.", self._source_file) + await self._move_file(self._source_file) + return + + inserted = 0 + seen_names: dict[str, int] = {} + for index, item in enumerate(items): + if not (normalized := self._normalize(item, index, seen_names)): + continue + try: + await self._repo.create(normalized) + inserted += 1 + except Exception as exc: + LOG.exception("Failed to insert dl field '%s': %s", normalized["name"], exc) + + LOG.info("Migrated %s dl field(s) from %s.", inserted, self._source_file) + await self._move_file(self._source_file) + + def _normalize(self, item: Any, index: int, seen_names: dict[str, int]) -> dict[str, Any] | None: + if not isinstance(item, dict): + LOG.warning("Skipping dl field at index %s due to invalid type.", index) + return None + + assert isinstance(item, dict) + + name: str | None = item.get("name") + if not name or not isinstance(name, str): + LOG.warning("Skipping dl field at index %s due to missing name.", index) + return None + + normalized_name = name.strip() + if not normalized_name: + LOG.warning("Skipping dl field at index %s due to empty name.", index) + return None + + name = self._unique_name(normalized_name, seen_names) + description: str = item.get("description") if isinstance(item.get("description"), str) else "" + field_value: str | None = item.get("field") + if not field_value or not isinstance(field_value, str): + LOG.warning("Skipping dl field '%s' due to missing field value.", name) + return None + + kind: str = item.get("kind") if isinstance(item.get("kind"), str) else "text" + icon: str = item.get("icon") if isinstance(item.get("icon"), str) else "" + value: str = item.get("value") if isinstance(item.get("value"), str) else "" + extras = item.get("extras") if isinstance(item.get("extras"), dict) else {} + extras = cast("dict[str, Any]", extras) + + order_value: int = 0 + if isinstance(item.get("order"), int) and not isinstance(item.get("order"), bool): + order_value = item.get("order", 0) + + payload = { + "name": name, + "description": description, + "field": field_value, + "kind": kind, + "icon": icon, + "order": order_value, + "value": value, + "extras": extras, + } + + try: + DLField.model_validate(payload) + except Exception as exc: + LOG.warning("Skipping dl field '%s' due to validation error: %s", name, exc) + return None + + return payload diff --git a/app/features/dl_fields/models.py b/app/features/dl_fields/models.py new file mode 100644 index 00000000..1c831903 --- /dev/null +++ b/app/features/dl_fields/models.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from datetime import datetime # noqa: TC003 + +from sqlalchemy import JSON, Index, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.features.core.models import Base, UTCDateTime, utcnow + + +class DLFieldModel(Base): + __tablename__: str = "dl_fields" + __table_args__: tuple[Index, ...] = ( + Index("ix_dl_fields_name", "name"), + Index("ix_dl_fields_order", "order"), + Index("ix_dl_fields_kind", "kind"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + description: Mapped[str] = mapped_column(Text, nullable=False, default="") + field: Mapped[str] = mapped_column(String(255), nullable=False) + kind: Mapped[str] = mapped_column(String(50), nullable=False, default="text") + icon: Mapped[str] = mapped_column(String(255), nullable=False, default="") + order: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + value: Mapped[str] = mapped_column(Text, nullable=False, default="") + extras: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False) + updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False) diff --git a/app/features/dl_fields/repository.py b/app/features/dl_fields/repository.py new file mode 100644 index 00000000..96502c6e --- /dev/null +++ b/app/features/dl_fields/repository.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from sqlalchemy import delete, func, or_, select + +from app.features.core.deps import get_session +from app.features.dl_fields.migration import Migration +from app.features.dl_fields.models import DLFieldModel +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Iterable + + from sqlalchemy.engine.result import Result + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + from sqlalchemy.sql.selectable import Select + +LOG: logging.Logger = logging.getLogger(__name__) + + +class DLFieldsRepository(metaclass=Singleton): + def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + self._migrated = False + self.session: AsyncGenerator[AsyncSession] = session or get_session + + async def run_migrations(self) -> None: + if self._migrated: + return + + self._migrated = True + await Migration(repo=self).run() + + @staticmethod + def get_instance() -> DLFieldsRepository: + return DLFieldsRepository() + + async def list(self) -> list[DLFieldModel]: + async with self.session() as session: + result: Result[tuple[DLFieldModel]] = await session.execute( + select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc()) + ) + return list(result.scalars().all()) + + async def list_paginated(self, page: int, per_page: int) -> tuple[list[DLFieldModel], int, int, int]: + async with self.session() as session: + total: int = await self.count() + total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1 + + if page > total_pages and total > 0: + page = total_pages + + query: Select[tuple[DLFieldModel]] = ( + select(DLFieldModel) + .order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc()) + .limit(per_page) + .offset((page - 1) * per_page) + ) + result: Result[tuple[DLFieldModel]] = await session.execute(query) + return list(result.scalars().all()), total, page, total_pages + + async def count(self) -> int: + async with self.session() as session: + result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(DLFieldModel)) + return int(result.scalar_one()) + + async def get(self, identifier: int | str) -> DLFieldModel | None: + async with self.session() as session: + if not identifier: + return None + + if isinstance(identifier, int): + clause: ColumnElement[bool] = DLFieldModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(DLFieldModel.id == int(identifier), DLFieldModel.name == identifier) + else: + clause = DLFieldModel.name == identifier + + result: Result[tuple[DLFieldModel]] = await session.execute(select(DLFieldModel).where(clause).limit(1)) + return result.scalar_one_or_none() + + async def get_by_name(self, name: str, exclude_id: int | None = None) -> DLFieldModel | None: + async with self.session() as session: + query: Select[tuple[DLFieldModel]] = select(DLFieldModel).where(DLFieldModel.name == name) + if exclude_id is not None: + query = query.where(DLFieldModel.id != exclude_id) + + result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1)) + return result.scalar_one_or_none() + + async def create(self, payload: DLFieldModel | dict) -> DLFieldModel: + async with self.session() as session: + model: DLFieldModel = DLFieldModel(**payload) if isinstance(payload, dict) else payload + + if await self.get_by_name(name=model.name) is not None: + msg: str = f"DL field with name '{model.name}' already exists." + raise ValueError(msg) + + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def update(self, identifier: int | str, payload: dict[str, Any]) -> DLFieldModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = DLFieldModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(DLFieldModel.id == int(identifier), DLFieldModel.name == identifier) + else: + clause = DLFieldModel.name == identifier + + result: Result[tuple[DLFieldModel]] = await session.execute(select(DLFieldModel).where(clause).limit(1)) + model: DLFieldModel | None = result.scalar_one_or_none() + + if not model: + msg: str = f"DL field '{identifier}' not found." + raise KeyError(msg) + + payload.pop("id", None) + + if "name" in payload and await self.get_by_name(name=payload["name"], exclude_id=model.id) is not None: + msg = f"DL field with name '{payload['name']}' already exists." + raise ValueError(msg) + + for key, value in payload.items(): + if hasattr(model, key): + setattr(model, key, value) + + await session.commit() + await session.refresh(model) + return model + + async def delete(self, identifier: int | str) -> DLFieldModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = DLFieldModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(DLFieldModel.id == int(identifier), DLFieldModel.name == identifier) + else: + clause = DLFieldModel.name == identifier + + result: Result[tuple[DLFieldModel]] = await session.execute(select(DLFieldModel).where(clause).limit(1)) + model = result.scalar_one_or_none() + + if not model: + msg: str = f"DL field '{identifier}' not found." + raise KeyError(msg) + + await session.delete(model) + await session.commit() + return model + + async def replace_all(self, items: Iterable[dict | DLFieldModel]) -> list[DLFieldModel]: + async with self.session() as session: + try: + await session.execute(delete(DLFieldModel)) + models: list[DLFieldModel] = [ + DLFieldModel(**item) if isinstance(item, dict) else item for item in items + ] + session.add_all(models) + await session.commit() + except Exception: + await session.rollback() + raise + return models diff --git a/app/features/dl_fields/router.py b/app/features/dl_fields/router.py new file mode 100644 index 00000000..330ea91c --- /dev/null +++ b/app/features/dl_fields/router.py @@ -0,0 +1,176 @@ +import logging +from typing import Any + +from aiohttp import web +from aiohttp.web import Request, Response +from pydantic import ValidationError + +from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination +from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination +from app.features.dl_fields.schemas import DLField, DLFieldList, DLFieldPatch +from app.features.dl_fields.service import DLFields +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events +from app.library.router import route + +LOG: logging.Logger = logging.getLogger(__name__) + + +def _model(model: Any) -> DLField: + return DLField.model_validate(model) + + +def _serialize(model: Any) -> dict: + return _model(model).model_dump() + + +@route("GET", "api/dl_fields/", "dl_fields") +async def dl_fields_list(request: Request, encoder: Encoder) -> Response: + repo = DLFields.get_instance()._repo + page, per_page = normalize_pagination(request) + items, total, current_page, total_pages = await repo.list_paginated(page, per_page) + return web.json_response( + data=DLFieldList( + items=[_model(model) for model in items], + pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)), + ), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", "api/dl_fields/", "dl_fields_add") +async def dl_fields_add(request: Request, encoder: Encoder, notify: EventBus) -> Response: + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + item: DLField = DLField.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate dl field.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + try: + saved = _serialize(await DLFields.get_instance().save(item=item.model_dump())) + except ValueError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code) + + notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.DL_FIELDS, action=CEAction.CREATE, data=saved)) + + return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("GET", r"api/dl_fields/{id:\d+}", "dl_fields_get") +async def dl_fields_get(request: Request, encoder: Encoder) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await DLFields.get_instance().get(identifier)): + return web.json_response({"error": "DL field not found"}, status=web.HTTPNotFound.status_code) + + return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("DELETE", r"api/dl_fields/{id:\d+}", "dl_fields_delete") +async def dl_fields_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + try: + deleted = _serialize(await DLFields.get_instance()._repo.delete(identifier)) + notify.emit( + Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.DL_FIELDS, action=CEAction.DELETE, data=deleted) + ) + return web.json_response( + data=deleted, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + except KeyError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code) + + +@route("PATCH", r"api/dl_fields/{id:\d+}", "dl_fields_patch") +async def dl_fields_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await DLFields.get_instance().get(identifier)): + return web.json_response({"error": "DL field not found"}, status=web.HTTPNotFound.status_code) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + validated = DLFieldPatch.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate dl field.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if validated.name and await DLFields.get_instance()._repo.get_by_name(validated.name, exclude_id=model.id): + return web.json_response( + data={"error": f"DL field with name '{validated.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + updated = _serialize(await DLFields.get_instance()._repo.update(model.id, validated.model_dump(exclude_unset=True))) + notify.emit( + Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.DL_FIELDS, action=CEAction.UPDATE, data=updated) + ) + return web.json_response( + data=updated, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("PUT", r"api/dl_fields/{id:\d+}", "dl_fields_update") +async def dl_fields_update(request: Request, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await DLFields.get_instance().get(identifier)): + return web.json_response({"error": "DL field not found"}, status=web.HTTPNotFound.status_code) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + validated = DLField.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate dl field.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if validated.name and await DLFields.get_instance()._repo.get_by_name(validated.name, exclude_id=model.id): + return web.json_response( + data={"error": f"DL field with name '{validated.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + updated = _serialize(await DLFields.get_instance()._repo.update(model.id, validated.model_dump(exclude_unset=True))) + notify.emit( + Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.DL_FIELDS, action=CEAction.UPDATE, data=updated) + ) + + return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode) diff --git a/app/features/dl_fields/schemas.py b/app/features/dl_fields/schemas.py new file mode 100644 index 00000000..666237a4 --- /dev/null +++ b/app/features/dl_fields/schemas.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from app.features.core.schemas import Pagination +from app.features.core.utils import parse_int + + +class FieldType(str, Enum): + STRING = "string" + TEXT = "text" + BOOL = "bool" + + +class DLField(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True) + + id: int | None = None + name: str = Field(min_length=1) + description: str = "" + field: str = Field(min_length=1) + kind: FieldType = FieldType.TEXT + icon: str = "" + order: int = 0 + value: str = "" + extras: dict[str, Any] = Field(default_factory=dict) + + @field_validator("field") + @classmethod + def _validate_field(cls, value: str) -> str: + if not value.startswith("--"): + msg: str = "Field must start with '--'." + raise ValueError(msg) + if " " in value: + msg = "Field must not contain spaces." + raise ValueError(msg) + return value + + @field_validator("order", mode="before") + @classmethod + def _normalize_order(cls, value: Any) -> int: + return parse_int(value, field="Order", minimum=0) + + @model_validator(mode="after") + def _normalize_extras(self) -> DLField: + if not isinstance(self.extras, dict): + msg: str = "Extras must be a dictionary." + raise ValueError(msg) + return self + + +class DLFieldPatch(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True) + + name: str | None = None + description: str | None = None + field: str | None = None + kind: FieldType | None = None + icon: str | None = None + order: int | None = None + value: str | None = None + extras: dict[str, Any] | None = None + + @field_validator("field") + @classmethod + def _validate_field(cls, value: str | None) -> str | None: + if value is None: + return value + if not value.startswith("--"): + msg: str = "Field must start with '--'." + raise ValueError(msg) + if " " in value: + msg = "Field must not contain spaces." + raise ValueError(msg) + return value + + @field_validator("order", mode="before") + @classmethod + def _normalize_order(cls, value: Any) -> int | None: + if value is None: + return None + return parse_int(value, field="Order", minimum=0) + + @model_validator(mode="after") + def _normalize_extras(self) -> DLFieldPatch: + if self.extras is None: + return self + if not isinstance(self.extras, dict): + msg: str = "Extras must be a dictionary." + raise ValueError(msg) + return self + + +class DLFieldList(BaseModel): + model_config = ConfigDict(from_attributes=True) + + items: list[DLField] = Field(default_factory=list) + pagination: Pagination diff --git a/app/features/dl_fields/service.py b/app/features/dl_fields/service.py new file mode 100644 index 00000000..77e2b261 --- /dev/null +++ b/app/features/dl_fields/service.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from app.features.dl_fields.models import DLFieldModel +from app.features.dl_fields.repository import DLFieldsRepository +from app.features.dl_fields.schemas import DLField +from app.library.Events import EventBus, Events +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from aiohttp import web + +LOG: logging.Logger = logging.getLogger("feature.dl_fields") + + +class DLFields(metaclass=Singleton): + def __init__(self): + self._repo: DLFieldsRepository = DLFieldsRepository.get_instance() + + @staticmethod + def get_instance() -> DLFields: + return DLFields() + + async def on_shutdown(self, _: web.Application) -> None: + pass + + def attach(self, _: web.Application) -> None: + async def handle_event(_, __): + await self._repo.run_migrations() + + EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations") + + async def get_all(self) -> list[DLFieldModel]: + return await self._repo.list() + + async def get_all_serialized(self) -> list[dict[str, Any]]: + items = await self._repo.list() + return [DLField.model_validate(item).model_dump() for item in items] + + async def save(self, item: DLField | dict) -> DLFieldModel: + try: + if not isinstance(item, DLField): + item = DLField.model_validate(item) + except Exception as exc: + msg: str = f"Failed to parse item. '{exc!s}'" + raise ValueError(msg) from exc + + try: + repo = self._repo + if item.id is None or 0 == item.id: + model = await repo.create(item.model_dump(exclude_unset=True)) + else: + model = await repo.update(item.id, item.model_dump(exclude_unset=True)) + except KeyError as exc: + raise ValueError(str(exc)) from exc + + return model + + async def get(self, identifier: int | str) -> DLFieldModel | None: + return await self._repo.get(identifier) diff --git a/app/features/dl_fields/tests/test_dl_fields.py b/app/features/dl_fields/tests/test_dl_fields.py new file mode 100644 index 00000000..9f7c408e --- /dev/null +++ b/app/features/dl_fields/tests/test_dl_fields.py @@ -0,0 +1,76 @@ +"""Tests for dl_fields feature service.""" + +from __future__ import annotations + +import pytest +import pytest_asyncio + +from app.features.dl_fields.repository import DLFieldsRepository +from app.features.dl_fields.service import DLFields +from app.library.sqlite_store import SqliteStore + + +@pytest_asyncio.fixture +async def repo(tmp_path): + """Provide a fresh repository instance with initialized database for each test.""" + DLFieldsRepository._reset_singleton() + DLFields._reset_singleton() + SqliteStore._reset_singleton() + + store = SqliteStore(db_path=":memory:") + await store.get_connection() + + repository = DLFieldsRepository.get_instance() + + yield repository + + if store._conn: + await store._conn.close() + if store._engine: + await store._engine.dispose() + + DLFieldsRepository._reset_singleton() + DLFields._reset_singleton() + SqliteStore._reset_singleton() + + +class TestDLFieldsService: + """Test suite for DLFields service methods.""" + + @pytest.mark.asyncio + async def test_save_creates_field(self, repo): + """Save should create a new dl field when ID is missing.""" + service = DLFields.get_instance() + payload = { + "name": "quality", + "description": "Video quality", + "field": "--format", + "kind": "string", + "order": 1, + "extras": {"options": ["best", "worst"]}, + } + + model = await service.save(payload) + + assert model.id is not None, "Should create new dl field" + assert model.name == "quality", "Should store name correctly" + + @pytest.mark.asyncio + async def test_get_all_serialized(self, repo): + """Get all serialized returns list of dictionaries.""" + service = DLFields.get_instance() + await service.save( + { + "name": "audio_only", + "description": "Audio", + "field": "--extract-audio", + "kind": "bool", + "order": 2, + } + ) + + items = await service.get_all_serialized() + + assert len(items) == 1, "Should return one dl field" + assert items[0]["name"] == "audio_only", "Should serialize name" + assert isinstance(items[0]["id"], int), "Should serialize integer ID" diff --git a/app/features/dl_fields/tests/test_dl_fields_repository.py b/app/features/dl_fields/tests/test_dl_fields_repository.py new file mode 100644 index 00000000..031b1d83 --- /dev/null +++ b/app/features/dl_fields/tests/test_dl_fields_repository.py @@ -0,0 +1,270 @@ +"""Tests for DLFieldsRepository.""" + +from __future__ import annotations + +import pytest +import pytest_asyncio + +from app.features.dl_fields.models import DLFieldModel +from app.features.dl_fields.repository import DLFieldsRepository +from app.library.sqlite_store import SqliteStore + + +@pytest_asyncio.fixture +async def repo(tmp_path): + """Provide a fresh repository instance with initialized database for each test.""" + DLFieldsRepository._reset_singleton() + SqliteStore._reset_singleton() + + store = SqliteStore(db_path=str(":memory:")) + await store.get_connection() + + repository = DLFieldsRepository.get_instance() + + yield repository + + if store._conn: + await store._conn.close() + if store._engine: + await store._engine.dispose() + + DLFieldsRepository._reset_singleton() + SqliteStore._reset_singleton() + + +class TestDLFieldsRepository: + """Test suite for DLFieldsRepository database operations.""" + + @pytest.mark.asyncio + async def test_repository_singleton(self, repo): + """Verify repository follows singleton pattern.""" + instance1 = DLFieldsRepository.get_instance() + instance2 = DLFieldsRepository.get_instance() + assert instance1 is instance2, "Should return same singleton instance" + + @pytest.mark.asyncio + async def test_list_empty(self, repo): + """List returns empty when no fields exist.""" + fields = await repo.list() + assert fields == [], "Should return empty list when no fields" + + @pytest.mark.asyncio + async def test_count_empty(self, repo): + """Count returns 0 when no fields exist.""" + count = await repo.count() + assert count == 0, "Should return 0 when no fields exist" + + @pytest.mark.asyncio + async def test_create_field(self, repo): + """Create field with valid data.""" + data = { + "name": "quality", + "description": "Video quality setting", + "field": "--format", + "kind": "string", + "icon": "fa-video", + "order": 1, + "value": "best", + "extras": {"options": ["best", "worst"]}, + } + + model = await repo.create(data) + + assert model.id is not None, "Should generate ID for new field" + assert model.name == "quality", "Should store name correctly" + assert model.description == "Video quality setting", "Should store description correctly" + assert model.field == "--format", "Should store field correctly" + assert model.kind == "string", "Should store kind correctly" + assert model.icon == "fa-video", "Should store icon correctly" + assert model.order == 1, "Should store order correctly" + assert model.value == "best", "Should store value correctly" + assert model.extras == {"options": ["best", "worst"]}, "Should store extras as dict" + + @pytest.mark.asyncio + async def test_create_with_defaults(self, repo): + """Create field with minimal data uses defaults.""" + data = { + "name": "minimal", + "description": "Minimal", + "field": "--minimal", + "kind": "text", + } + + model = await repo.create(data) + + assert model.icon == "", "Should default icon to empty string" + assert model.order == 0, "Should default order to 0" + assert model.value == "", "Should default value to empty string" + assert model.extras == {}, "Should default extras to empty dict" + + @pytest.mark.asyncio + async def test_get_by_id(self, repo): + """Get field by integer ID.""" + created = await repo.create( + { + "name": "get_test", + "description": "Get test", + "field": "--get-test", + "kind": "text", + } + ) + + retrieved = await repo.get(created.id) + + assert retrieved is not None, "Should retrieve created field" + assert retrieved.id == created.id, "Should retrieve correct field by ID" + assert retrieved.name == "get_test", "Should match created field name" + + @pytest.mark.asyncio + async def test_get_by_name(self, repo): + """Get field by string name.""" + await repo.create( + { + "name": "named_test", + "description": "Named test", + "field": "--named-test", + "kind": "text", + } + ) + + retrieved = await repo.get("named_test") + + assert retrieved is not None, "Should retrieve by name" + assert retrieved.name == "named_test", "Should match field name" + + @pytest.mark.asyncio + async def test_get_nonexistent(self, repo): + """Get nonexistent field returns None.""" + result = await repo.get(99999) + assert result is None, "Should return None for nonexistent ID" + + result = await repo.get("nonexistent") + assert result is None, "Should return None for nonexistent name" + + @pytest.mark.asyncio + async def test_update_field(self, repo): + """Update existing field.""" + created = await repo.create( + { + "name": "update_test", + "description": "Update test", + "field": "--update-test", + "kind": "text", + } + ) + + updated = await repo.update( + created.id, + { + "name": "updated_name", + "order": 5, + "extras": {"updated": True}, + }, + ) + + assert updated.name == "updated_name", "Should update name" + assert updated.order == 5, "Should update order" + assert updated.extras == {"updated": True}, "Should update extras" + assert updated.field == "--update-test", "Should preserve unchanged field" + + @pytest.mark.asyncio + async def test_update_nonexistent_raises(self, repo): + """Update nonexistent field raises KeyError.""" + with pytest.raises(KeyError, match="not found"): + await repo.update(99999, {"name": "should_fail"}) + + @pytest.mark.asyncio + async def test_delete_field(self, repo): + """Delete existing field.""" + created = await repo.create( + { + "name": "delete_test", + "description": "Delete test", + "field": "--delete-test", + "kind": "text", + } + ) + + deleted = await repo.delete(created.id) + + assert deleted.id == created.id, "Should return deleted field" + + result = await repo.get(created.id) + assert result is None, "Deleted field should not be retrievable" + + @pytest.mark.asyncio + async def test_delete_nonexistent_raises(self, repo): + """Delete nonexistent field raises KeyError.""" + with pytest.raises(KeyError, match="not found"): + await repo.delete(99999) + + @pytest.mark.asyncio + async def test_list_paginated(self, repo): + """List paginated returns correct subset.""" + for i in range(5): + await repo.create( + { + "name": f"item_{i}", + "description": "desc", + "field": f"--item-{i}", + "kind": "text", + "order": i, + } + ) + + items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2) + + assert len(items) == 2, "Should return 2 items per page" + assert total == 5, "Should report total count of 5" + assert page == 1, "Should be on page 1" + assert total_pages == 3, "Should have 3 pages total" + + @pytest.mark.asyncio + async def test_list_ordering(self, repo): + """List orders by order asc then name asc.""" + await repo.create({"name": "b", "description": "b", "field": "--b", "kind": "text", "order": 1}) + await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1}) + await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0}) + + items = await repo.list() + + assert items[0].name == "c", "Lowest order should be first" + assert items[1].name == "a", "Same order sorted alphabetically" + assert items[2].name == "b", "Same order sorted alphabetically" + + @pytest.mark.asyncio + async def test_get_by_name_excludes_id(self, repo): + """Get by name can exclude specific ID.""" + first = await repo.create( + { + "name": "duplicate", + "description": "duplicate", + "field": "--duplicate", + "kind": "text", + } + ) + + result = await repo.get_by_name("duplicate", exclude_id=first.id) + assert result is None, "Should not find when excluding only match" + + result = await repo.get_by_name("duplicate", exclude_id=None) + assert result is not None, "Should find without exclusion" + + @pytest.mark.asyncio + async def test_replace_all(self, repo): + """Replace all fields atomically.""" + await repo.create({"name": "old_1", "description": "old", "field": "--old-1", "kind": "text"}) + await repo.create({"name": "old_2", "description": "old", "field": "--old-2", "kind": "text"}) + + new_items = [ + {"name": "new_1", "description": "new", "field": "--new-1", "kind": "text"}, + {"name": "new_2", "description": "new", "field": "--new-2", "kind": "text"}, + ] + + result = await repo.replace_all(new_items) + + assert len(result) == 2, "Should create 2 new fields" + + all_items = await repo.list() + assert len(all_items) == 2, "Should only have new fields" + assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items" diff --git a/app/features/notifications/deps.py b/app/features/notifications/deps.py new file mode 100644 index 00000000..ae270442 --- /dev/null +++ b/app/features/notifications/deps.py @@ -0,0 +1,5 @@ +from app.features.notifications.repository import NotificationsRepository + + +def get_notifications_repo() -> NotificationsRepository: + return NotificationsRepository.get_instance() diff --git a/app/features/notifications/migration.py b/app/features/notifications/migration.py new file mode 100644 index 00000000..6262cdee --- /dev/null +++ b/app/features/notifications/migration.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +from app.features.core.migration import Migration as FeatureMigration +from app.features.notifications.schemas import NotificationEvents +from app.features.presets.service import Presets +from app.library.config import Config + +if TYPE_CHECKING: + from app.features.notifications.repository import NotificationsRepository + +LOG: logging.Logger = logging.getLogger(__name__) + + +class Migration(FeatureMigration): + name: str = "notifications" + + def __init__(self, repo: NotificationsRepository, config: Config | None = None): + self._config: Config = config or Config.get_instance() + super().__init__(config=self._config) + self._repo: NotificationsRepository = repo + self._source_file: Path = Path(self._config.config_path) / "notifications.json" + + async def should_run(self) -> bool: + return self._source_file.exists() + + async def migrate(self) -> None: + if await self._repo.count() > 0: + LOG.warning("Notification targets already exist in the database; skipping migration.") + await self._move_file(self._source_file) + return + + try: + items: list[dict] | None = json.loads(self._source_file.read_text()) + except Exception as exc: + LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc) + await self._move_file(self._source_file) + return + + if items is None: + LOG.warning("No notification targets found in %s; skipping migration.", self._source_file) + await self._move_file(self._source_file) + return + + inserted = 0 + seen_names: dict[str, int] = {} + for index, item in enumerate(items): + normalized = self._normalize(item, index, seen_names) + if not normalized: + continue + try: + await self._repo.create(normalized) + inserted += 1 + except Exception as exc: + LOG.exception("Failed to insert notification '%s': %s", normalized.get("name"), exc) + + LOG.info("Migrated %s notification target(s) from %s.", inserted, self._source_file) + await self._move_file(self._source_file) + + def _normalize(self, item: Any, index: int, seen_names: dict[str, int]) -> dict[str, Any] | None: + if not isinstance(item, dict): + LOG.warning("Skipping notification at index %s due to invalid type.", index) + return None + + assert isinstance(item, dict) + + name: str | None = item.get("name") + if not name or not isinstance(name, str): + LOG.warning("Skipping notification at index %s due to missing name.", index) + return None + + normalized_name = name.strip() + if not normalized_name: + LOG.warning("Skipping notification at index %s due to empty name.", index) + return None + + name = self._unique_name(normalized_name, seen_names) + request = item.get("request") if isinstance(item.get("request"), dict) else {} + request = cast("dict[str, Any]", request) + url: str | None = request.get("url") + if not url or not isinstance(url, str): + LOG.warning("Skipping notification '%s' due to missing request url.", name) + return None + + raw_method = request.get("method") + method = raw_method.upper() if isinstance(raw_method, str) else "POST" + if method not in {"POST", "PUT"}: + LOG.warning("Skipping notification '%s' due to invalid method '%s'.", name, method) + return None + + raw_type = request.get("type") + req_type = raw_type.lower() if isinstance(raw_type, str) else "json" + if req_type not in {"json", "form"}: + LOG.warning("Skipping notification '%s' due to invalid type '%s'.", name, req_type) + return None + + data_key = request.get("data_key") if isinstance(request.get("data_key"), str) else "data" + if not data_key: + data_key = "data" + + headers = self._normalize_headers(request.get("headers")) + + enabled_value = item.get("enabled") + enabled: bool = enabled_value if isinstance(enabled_value, bool) else True + events = self._normalize_events(item.get("on")) + if events is None: + LOG.warning("Skipping notification '%s' due to invalid events.", name) + return None + + presets = self._normalize_presets(item.get("presets")) + if presets is None: + LOG.warning("Skipping notification '%s' due to invalid presets.", name) + return None + + return { + "name": name, + "on": events, + "presets": presets, + "enabled": enabled, + "request_url": url, + "request_method": method, + "request_type": req_type, + "request_data_key": data_key, + "request_headers": headers, + } + + def _normalize_events(self, events: Any) -> list[str] | None: + if events is None: + return [] + if not isinstance(events, list): + return [] + + allowed = set(NotificationEvents.get_events().values()) + valid = [event for event in events if event in allowed] + invalid = [event for event in events if event not in allowed] + + if len(invalid) > 0 and len(valid) < 1: + return None + + if len(invalid) > 0: + LOG.warning("Dropping invalid notification events: %s", ", ".join(invalid)) + + return valid + + def _normalize_presets(self, presets: Any) -> list[str] | None: + if presets is None: + return [] + if not isinstance(presets, list): + return [] + + allowed = {preset.name for preset in Presets.get_instance().get_all()} + valid = [preset for preset in presets if preset in allowed] + invalid = [preset for preset in presets if preset not in allowed] + + if len(invalid) > 0 and len(valid) < 1: + return None + + if len(invalid) > 0: + LOG.warning("Dropping invalid notification presets: %s", ", ".join(invalid)) + + return valid + + def _normalize_headers(self, headers: Any) -> list[dict[str, str]]: + if not isinstance(headers, list): + return [] + + normalized: list[dict[str, str]] = [] + for header in headers: + if not isinstance(header, dict): + continue + key = str(header.get("key", "")).strip() + value = str(header.get("value", "")).strip() + if key and value: + normalized.append({"key": key, "value": value}) + return normalized diff --git a/app/features/notifications/models.py b/app/features/notifications/models.py new file mode 100644 index 00000000..38c698a5 --- /dev/null +++ b/app/features/notifications/models.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from datetime import datetime # noqa: TC003 + +from sqlalchemy import JSON, Boolean, Index, Integer, String, Text +from sqlalchemy import Enum as SQLAEnum +from sqlalchemy.orm import Mapped, mapped_column + +from app.features.core.models import Base, UTCDateTime, utcnow +from app.features.notifications.schemas import NotificationRequestMethod, NotificationRequestType + + +class NotificationModel(Base): + __tablename__: str = "notifications" + __table_args__: tuple[Index, ...] = ( + Index("ix_notifications_name", "name"), + Index("ix_notifications_enabled", "enabled"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + on: Mapped[list[str]] = mapped_column(JSON, nullable=False, default=list) + presets: Mapped[list[str]] = mapped_column(JSON, nullable=False, default=list) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + request_url: Mapped[str] = mapped_column(Text, nullable=False) + request_method: Mapped[NotificationRequestMethod] = mapped_column( + SQLAEnum(NotificationRequestMethod, name="notification_request_method", native_enum=False), + nullable=False, + default=NotificationRequestMethod.POST, + ) + request_type: Mapped[NotificationRequestType] = mapped_column( + SQLAEnum(NotificationRequestType, name="notification_request_type", native_enum=False), + nullable=False, + default=NotificationRequestType.JSON, + ) + request_data_key: Mapped[str] = mapped_column(String(255), nullable=False, default="data") + request_headers: Mapped[list[dict]] = mapped_column(JSON, nullable=False, default=list) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False) + updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False) diff --git a/app/features/notifications/repository.py b/app/features/notifications/repository.py new file mode 100644 index 00000000..073fd355 --- /dev/null +++ b/app/features/notifications/repository.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from sqlalchemy import func, or_, select + +from app.features.core.deps import get_session +from app.features.notifications.migration import Migration +from app.features.notifications.models import NotificationModel +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from sqlalchemy.engine.result import Result + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + from sqlalchemy.sql.selectable import Select + +LOG: logging.Logger = logging.getLogger(__name__) + + +class NotificationsRepository(metaclass=Singleton): + def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + self._migrated = False + self.session: AsyncGenerator[AsyncSession] = session or get_session + + async def run_migrations(self) -> None: + if self._migrated: + return + + self._migrated = True + await Migration(repo=self, config=None).run() + + @staticmethod + def get_instance() -> NotificationsRepository: + return NotificationsRepository() + + async def list(self) -> list[NotificationModel]: + async with self.session() as session: + result: Result[tuple[NotificationModel]] = await session.execute( + select(NotificationModel).order_by(NotificationModel.name.asc()) + ) + return list(result.scalars().all()) + + async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]: + async with self.session() as session: + total: int = await self.count() + total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1 + + if page > total_pages and total > 0: + page = total_pages + + query: Select[tuple[NotificationModel]] = ( + select(NotificationModel) + .order_by(NotificationModel.name.asc()) + .limit(per_page) + .offset((page - 1) * per_page) + ) + result: Result[tuple[NotificationModel]] = await session.execute(query) + return list(result.scalars().all()), total, page, total_pages + + async def count(self) -> int: + async with self.session() as session: + result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(NotificationModel)) + return int(result.scalar_one()) + + async def get(self, identifier: int | str) -> NotificationModel | None: + async with self.session() as session: + if not identifier: + return None + + if isinstance(identifier, int): + clause: ColumnElement[bool] = NotificationModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(NotificationModel.id == int(identifier), NotificationModel.name == identifier) + else: + clause = NotificationModel.name == identifier + + result: Result[tuple[NotificationModel]] = await session.execute( + select(NotificationModel).where(clause).limit(1) + ) + return result.scalar_one_or_none() + + async def get_by_name(self, name: str, exclude_id: int | None = None) -> NotificationModel | None: + async with self.session() as session: + query: Select[tuple[NotificationModel]] = select(NotificationModel).where(NotificationModel.name == name) + if exclude_id is not None: + query = query.where(NotificationModel.id != exclude_id) + + result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1)) + return result.scalar_one_or_none() + + async def create(self, payload: NotificationModel | dict) -> NotificationModel: + async with self.session() as session: + model: NotificationModel = NotificationModel(**payload) if isinstance(payload, dict) else payload + if model.id is not None: + model.id = None + + if await self.get_by_name(name=model.name) is not None: + msg: str = f"Notification target with name '{model.name}' already exists." + raise ValueError(msg) + + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def update(self, identifier: int | str, payload: dict[str, Any]) -> NotificationModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = NotificationModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(NotificationModel.id == int(identifier), NotificationModel.name == identifier) + else: + clause = NotificationModel.name == identifier + + result: Result[tuple[NotificationModel]] = await session.execute( + select(NotificationModel).where(clause).limit(1) + ) + model: NotificationModel | None = result.scalar_one_or_none() + + if not model: + msg: str = f"Notification target '{identifier}' not found." + raise KeyError(msg) + + payload.pop("id", None) + + if "name" in payload and await self.get_by_name(name=payload["name"], exclude_id=model.id) is not None: + msg = f"Notification target with name '{payload['name']}' already exists." + raise ValueError(msg) + + for key, value in payload.items(): + if hasattr(model, key): + setattr(model, key, value) + + await session.commit() + await session.refresh(model) + return model + + async def delete(self, identifier: int | str) -> NotificationModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = NotificationModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(NotificationModel.id == int(identifier), NotificationModel.name == identifier) + else: + clause = NotificationModel.name == identifier + + result: Result[tuple[NotificationModel]] = await session.execute( + select(NotificationModel).where(clause).limit(1) + ) + model = result.scalar_one_or_none() + + if not model: + msg: str = f"Notification target '{identifier}' not found." + raise KeyError(msg) + + await session.delete(model) + await session.commit() + return model diff --git a/app/features/notifications/router.py b/app/features/notifications/router.py new file mode 100644 index 00000000..bb5fdbf5 --- /dev/null +++ b/app/features/notifications/router.py @@ -0,0 +1,218 @@ +import logging +from typing import Any + +from aiohttp import web +from aiohttp.web import Request, Response +from pydantic import ValidationError + +from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination +from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination +from app.features.notifications.schemas import Notification, NotificationEvents, NotificationList, NotificationPatch +from app.features.notifications.service import Notifications +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events +from app.library.router import route + +LOG: logging.Logger = logging.getLogger(__name__) + + +def _model(model: Any) -> Notification: + return Notifications.get_instance().model_to_schema(model) + + +def _serialize(model: Any) -> dict: + return _model(model).model_dump() + + +@route("GET", "api/notifications/", "notifications_list") +async def notifications_list(request: Request, encoder: Encoder) -> Response: + page, per_page = normalize_pagination(request) + items, total, current_page, total_pages = await Notifications.get_instance().list_paginated(page, per_page) + return web.json_response( + data=NotificationList( + items=[_model(model) for model in items], + pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)), + ), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("GET", "api/notifications/events/", "notifications_events") +async def notifications_events(encoder: Encoder) -> Response: + return web.json_response( + data={"events": list(NotificationEvents.get_events().values())}, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", "api/notifications/test/", "notification_test") +async def notification_test(encoder: Encoder, notify: EventBus) -> Response: + notify.emit(Events.TEST, title="Test Notification", message="This is a test notification.") + return web.json_response(data={}, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("POST", "api/notifications/", "notification_add") +async def notifications_add(request: Request, encoder: Encoder, notify: EventBus) -> Response: + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + item: Notification = Notification.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate notification.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + try: + saved = _serialize(await Notifications.get_instance().create(item)) + except ValueError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code) + + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.NOTIFICATIONS, action=CEAction.CREATE, data=saved), + ) + return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("GET", r"api/notifications/{id:\d+}", "notification_get") +async def notifications_get(request: Request, encoder: Encoder) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await Notifications.get_instance().get(identifier)): + return web.json_response({"error": "Notification not found"}, status=web.HTTPNotFound.status_code) + + return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("DELETE", r"api/notifications/{id:\d+}", "notification_delete") +async def notifications_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + try: + deleted = _serialize(await Notifications.get_instance().delete(identifier)) + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.NOTIFICATIONS, action=CEAction.DELETE, data=deleted), + ) + return web.json_response( + data=deleted, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + except KeyError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code) + + +@route("PATCH", r"api/notifications/{id:\d+}", "notification_patch") +async def notifications_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await Notifications.get_instance().get(identifier)): + return web.json_response({"error": "Notification not found"}, status=web.HTTPNotFound.status_code) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + validated = NotificationPatch.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate notification.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + current = _model(model).model_dump() + if validated.name is not None: + current["name"] = validated.name + if validated.on is not None: + current["on"] = validated.on + if validated.presets is not None: + current["presets"] = validated.presets + if validated.enabled is not None: + current["enabled"] = validated.enabled + + if validated.request is not None: + request_payload = current.get("request", {}) + if validated.request.url is not None: + request_payload["url"] = validated.request.url + if validated.request.method is not None: + request_payload["method"] = validated.request.method + if validated.request.type is not None: + request_payload["type"] = validated.request.type + if validated.request.headers is not None: + request_payload["headers"] = [header.model_dump() for header in validated.request.headers] + if validated.request.data_key is not None: + request_payload["data_key"] = validated.request.data_key + current["request"] = request_payload + + try: + updated = _serialize(await Notifications.get_instance().update(model.id, current)) + except ValueError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code) + except KeyError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code) + + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.NOTIFICATIONS, action=CEAction.UPDATE, data=updated), + ) + return web.json_response( + data=updated, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("PUT", r"api/notifications/{id:\d+}", "notification_update") +async def notifications_update(request: Request, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await Notifications.get_instance().get(identifier)): + return web.json_response({"error": "Notification not found"}, status=web.HTTPNotFound.status_code) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + validated = Notification.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate notification.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + try: + updated = _serialize(await Notifications.get_instance().update(model.id, validated)) + except ValueError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code) + except KeyError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code) + + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.NOTIFICATIONS, action=CEAction.UPDATE, data=updated), + ) + return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode) diff --git a/app/features/notifications/schemas.py b/app/features/notifications/schemas.py new file mode 100644 index 00000000..d2eb024f --- /dev/null +++ b/app/features/notifications/schemas.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from app.features.core.schemas import Pagination +from app.library.Events import Events + + +class NotificationEvents: + TEST: str = Events.TEST + + ITEM_ADDED: str = Events.ITEM_ADDED + ITEM_COMPLETED: str = Events.ITEM_COMPLETED + ITEM_CANCELLED: str = Events.ITEM_CANCELLED + ITEM_DELETED: str = Events.ITEM_DELETED + ITEM_PAUSED: str = Events.ITEM_PAUSED + ITEM_RESUMED: str = Events.ITEM_RESUMED + ITEM_MOVED: str = Events.ITEM_MOVED + + PAUSED: str = Events.PAUSED + RESUMED: str = Events.RESUMED + + LOG_INFO: str = Events.LOG_INFO + LOG_SUCCESS: str = Events.LOG_SUCCESS + LOG_WARNING: str = Events.LOG_WARNING + LOG_ERROR: str = Events.LOG_ERROR + + TASK_DISPATCHED: str = Events.TASK_DISPATCHED + + @staticmethod + def get_events() -> dict[str, str]: + return {k: v for k, v in vars(NotificationEvents).items() if not k.startswith("__") and not callable(v)} + + @staticmethod + def events() -> list[str]: + return [ + getattr(NotificationEvents, ev) + for ev in dir(NotificationEvents) + if not ev.startswith("_") and not callable(getattr(NotificationEvents, ev)) + ] + + @staticmethod + def is_valid(event: str) -> bool: + return event in NotificationEvents.get_events().values() + + +class NotificationRequestType(str, Enum): + JSON = "json" + FORM = "form" + + def __str__(self) -> str: + return self.value + + +class NotificationRequestMethod(str, Enum): + POST = "POST" + PUT = "PUT" + + def __str__(self) -> str: + return self.value + + +class NotificationRequestHeader(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True) + + key: str = Field(min_length=1) + value: str = Field(min_length=1) + + +class NotificationRequest(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True) + + type: NotificationRequestType = NotificationRequestType.JSON + method: NotificationRequestMethod = NotificationRequestMethod.POST + url: str = Field(min_length=1) + headers: list[NotificationRequestHeader] = Field(default_factory=list) + data_key: str = Field(default="data", min_length=1) + + @field_validator("method", mode="before") + @classmethod + def _normalize_method(cls, value: Any) -> Any: + if value is None: + return value + if isinstance(value, NotificationRequestMethod): + return value + return str(value).upper() + + @field_validator("type", mode="before") + @classmethod + def _normalize_type(cls, value: Any) -> Any: + if value is None: + return value + if isinstance(value, NotificationRequestType): + return value + return str(value).lower() + + +class NotificationRequestPatch(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True) + + type: NotificationRequestType | None = None + method: NotificationRequestMethod | None = None + url: str | None = None + headers: list[NotificationRequestHeader] | None = None + data_key: str | None = None + + +class Notification(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True) + + id: int | None = None + name: str = Field(min_length=1) + on: list[str] = Field(default_factory=list) + presets: list[str] = Field(default_factory=list) + enabled: bool = True + request: NotificationRequest + + @field_validator("on", "presets", mode="before") + @classmethod + def _normalize_list(cls, value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + return [str(value).strip()] + + +class NotificationPatch(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True) + + name: str | None = None + on: list[str] | None = None + presets: list[str] | None = None + enabled: bool | None = None + request: NotificationRequestPatch | None = None + + +class NotificationList(BaseModel): + model_config = ConfigDict(from_attributes=True) + + items: list[Notification] = Field(default_factory=list) + pagination: Pagination diff --git a/app/features/notifications/service.py b/app/features/notifications/service.py new file mode 100644 index 00000000..bdef57b7 --- /dev/null +++ b/app/features/notifications/service.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +import asyncio +import logging +import traceback +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from app.features.notifications.models import NotificationModel +from app.features.notifications.repository import NotificationsRepository +from app.features.notifications.schemas import ( + Notification, + NotificationEvents, + NotificationRequest, + NotificationRequestHeader, + NotificationRequestType, +) +from app.features.presets.schemas import Preset +from app.features.presets.service import Presets +from app.library.BackgroundWorker import BackgroundWorker +from app.library.config import Config +from app.library.encoder import Encoder +from app.library.Events import Event, EventBus, Events +from app.library.httpx_client import async_client +from app.library.ItemDTO import Item, ItemDTO +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from collections.abc import Awaitable + + import httpx + from aiohttp import web + +LOG: logging.Logger = logging.getLogger("feature.notifications") + + +class Notifications(metaclass=Singleton): + def __init__( + self, + repo: NotificationsRepository | None = None, + client: httpx.AsyncClient | None = None, + encoder: Encoder | None = None, + config: Config | None = None, + background_worker: BackgroundWorker | None = None, + ) -> None: + self._repo: NotificationsRepository = repo or NotificationsRepository.get_instance() + + config = config or Config.get_instance() + + self._debug: bool = config.debug + self._version: str = config.app_version + self._client: httpx.AsyncClient = client or async_client() + self._encoder: Encoder = encoder or Encoder() + self._offload: BackgroundWorker = background_worker or BackgroundWorker.get_instance() + + @staticmethod + def get_instance() -> Notifications: + return Notifications() + + async def on_shutdown(self, _: web.Application) -> None: + pass + + def attach(self, _: web.Application) -> None: + async def handle_event(_, __): + await self._repo.run_migrations() + + EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations") + EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{__class__.__name__}.emit") + + async def list(self) -> list[NotificationModel]: + return await self._repo.list() + + async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]: + return await self._repo.list_paginated(page, per_page) + + async def get(self, identifier: int | str) -> NotificationModel | None: + return await self._repo.get(identifier) + + async def create(self, item: Notification | dict) -> NotificationModel: + if not isinstance(item, Notification): + item = Notification.model_validate(item) + normalized = self._normalize(item) + payload = self._payload_from_schema(normalized) + return await self._repo.create(payload) + + async def update(self, identifier: int | str, payload: Notification | dict) -> NotificationModel: + if not isinstance(payload, Notification): + payload = Notification.model_validate(payload) + normalized = self._normalize(payload) + update_payload = self._payload_from_schema(normalized) + return await self._repo.update(identifier, update_payload) + + async def delete(self, identifier: int | str) -> NotificationModel: + return await self._repo.delete(identifier) + + async def send(self, ev: Event, wait: bool = True) -> list[dict] | list[Awaitable[dict]]: + targets = await self._repo.list() + if len(targets) < 1: + return [] + + if not isinstance(ev.data, (ItemDTO, Item, dict)): + LOG.debug("Received invalid item type '%s' with event '%s'.", type(ev.data), ev.event) + return [] + + tasks: list[Awaitable[dict]] = [] + apprise_targets: list[Notification] = [] + + for target_model in targets: + target = self.model_to_schema(target_model) + if not target.enabled: + continue + + if len(target.on) > 0 and ev.event not in target.on and NotificationEvents.TEST != ev.event: + continue + + if NotificationEvents.TEST != ev.event and not self._check_preset(target, ev): + continue + + if not target.request.url.startswith("http"): + apprise_targets.append(target) + else: + tasks.append(self._send(target, ev)) + + if len(apprise_targets) > 0: + tasks.append(self._apprise(apprise_targets, ev)) + + if wait: + return await asyncio.gather(*tasks) + + return tasks + + def emit(self, e: Event, _, **__) -> None: + if not NotificationEvents.is_valid(e.event): + return + + self._offload.submit(self.send, e) + + def _normalize(self, item: Notification) -> Notification: + if item.enabled is not None and not isinstance(item.enabled, bool): + msg: str = "Enabled must be a boolean." + raise ValueError(msg) + + item.on = self._filter_events(item.on) + item.presets = self._filter_presets(item.presets) + item.request.headers = [ + NotificationRequestHeader(key=header.key, value=header.value) + for header in item.request.headers + if header.key and header.value + ] + return item + + def _filter_events(self, events: list[str]) -> list[str]: + if not events: + return [] + + allowed = set(NotificationEvents.get_events().values()) + valid = [event for event in events if event in allowed] + invalid = [event for event in events if event not in allowed] + + if len(invalid) > 0 and len(valid) < 1: + msg: str = f"Invalid notification target. Invalid events '{', '.join(invalid)}' found." + raise ValueError(msg) + + if len(invalid) > 0: + LOG.warning("Dropping invalid notification events: %s", ", ".join(invalid)) + + return valid + + def _filter_presets(self, presets: list[str]) -> list[str]: + if not presets: + return [] + + all_presets: list[Preset] = Presets.get_instance().get_all() + allowed = {preset.name for preset in all_presets} + valid = [preset for preset in presets if preset in allowed] + invalid = [preset for preset in presets if preset not in allowed] + + if len(invalid) > 0 and len(valid) < 1: + msg: str = f"Invalid notification target. Invalid presets '{', '.join(invalid)}' found." + raise ValueError(msg) + + if len(invalid) > 0: + LOG.warning("Dropping invalid notification presets: %s", ", ".join(invalid)) + + return valid + + def model_to_schema(self, model: NotificationModel) -> Notification: + headers: list[NotificationRequestHeader] = [] + if isinstance(model.request_headers, list): + for header in model.request_headers: + if not isinstance(header, dict): + continue + key = str(header.get("key", "")).strip() + value = str(header.get("value", "")).strip() + if key and value: + headers.append(NotificationRequestHeader(key=key, value=value)) + + return Notification( + id=model.id, + name=model.name, + on=list(model.on or []), + presets=list(model.presets or []), + enabled=model.enabled, + request=NotificationRequest.model_validate( + { + "type": model.request_type, + "method": model.request_method, + "url": model.request_url, + "headers": headers, + "data_key": model.request_data_key, + } + ), + ) + + def _payload_from_schema(self, item: Notification) -> dict[str, Any]: + return { + "name": item.name, + "on": list(item.on), + "presets": list(item.presets), + "enabled": item.enabled, + "request_url": item.request.url, + "request_method": str(item.request.method), + "request_type": str(item.request.type), + "request_data_key": item.request.data_key, + "request_headers": [header.model_dump() for header in item.request.headers], + } + + def _check_preset(self, target: Notification, ev: Event) -> bool: + if len(target.presets) < 1: + return True + + if not isinstance(ev.data, (Item, ItemDTO, dict)): + return False + + preset_name: str | None = None + + if isinstance(ev.data, Item): + preset_name = ev.data.preset + + if isinstance(ev.data, ItemDTO): + preset_name = ev.data.preset + + if isinstance(ev.data, dict): + preset_name = ev.data.get("preset", None) + + if not preset_name: + return False + + return preset_name in target.presets + + async def _apprise(self, targets: list[Notification], ev: Event) -> dict: + if not targets: + return {} + + import apprise + + try: + notify = apprise.Apprise(debug=self._debug) + apr_config = Path(Config.get_instance().apprise_config) + if apr_config.exists(): + apprise_file = apprise.AppriseConfig() + apprise_file.add(str(apr_config)) + notify.add(apprise_file) + + for target in targets: + notify.add(target.request.url) + + status = await notify.async_notify( + body=ev.message or self._encoder.encode(ev.serialize()), + title=ev.title or f"YTPTube Event: {ev.event}", + ) + + if not status: + msg = "Apprise failed to send notification." + raise RuntimeError(msg) # noqa: TRY301 + except Exception as exc: + LOG.exception(exc) + LOG.error("Error sending Apprise notification: %s", exc) + return {"error": str(exc), "event": ev.event, "id": ev.id, "targets": [t.name for t in targets]} + + return {} + + async def _send(self, target: Notification, ev: Event) -> dict: + try: + LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name) + + headers: dict[str, str] = { + "User-Agent": f"YTPTube/{self._version}", + "X-Event-Id": ev.id, + "X-Event": ev.event, + "Content-Type": "application/json" + if NotificationRequestType.JSON == target.request.type + else "application/x-www-form-urlencoded", + } + + if len(target.request.headers) > 0: + headers.update({h.key: h.value for h in target.request.headers if h.key and h.value}) + + payload_data: dict[str, Any] = ev.serialize() + + if "data" != target.request.data_key: + payload_data[target.request.data_key] = payload_data["data"] + payload_data.pop("data", None) + + if NotificationRequestType.FORM == target.request.type: + payload_data[target.request.data_key] = self._encoder.encode(payload_data[target.request.data_key]) + data_payload: dict[str, Any] | None = payload_data + content_payload: str | None = None + else: + data_payload = None + content_payload = self._encoder.encode(payload_data) + + response = await self._client.request( + method=str(target.request.method).upper(), + url=target.request.url, + headers=headers, + data=data_payload, + content=content_payload, + ) + + resp_data: dict[str, Any] = { + "url": target.request.url, + "status": response.status_code, + "text": response.text, + } + + msg: str = ( + f"Notification target '{target.name}' Responded to event '{ev.event}: {ev.id}' " + f"with status '{response.status_code}'." + ) + if self._debug and resp_data.get("text"): + msg += f" body '{resp_data.get('text', '??')}'." + + LOG.info(msg) + + return resp_data + except Exception as exc: + err_msg = str(exc) or type(exc).__name__ + tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) + LOG.error( + "Error sending notification event '%s: %s' to '%s'. '%s'. %s", + ev.event, + ev.id, + target.name, + err_msg, + tb, + ) + return {"url": target.request.url, "status": 500, "text": str(ev)} diff --git a/app/features/notifications/tests/test_notifications_repository.py b/app/features/notifications/tests/test_notifications_repository.py new file mode 100644 index 00000000..51ef1260 --- /dev/null +++ b/app/features/notifications/tests/test_notifications_repository.py @@ -0,0 +1,154 @@ +"""Tests for NotificationsRepository.""" + +from __future__ import annotations + +import pytest +import pytest_asyncio + +from app.features.notifications.repository import NotificationsRepository +from app.library.sqlite_store import SqliteStore + + +@pytest_asyncio.fixture +async def repo(tmp_path): + """Provide a fresh repository instance with initialized database for each test.""" + NotificationsRepository._reset_singleton() + SqliteStore._reset_singleton() + + store = SqliteStore(db_path=":memory:") + await store.get_connection() + + repository = NotificationsRepository.get_instance() + + yield repository + + if store._conn: + await store._conn.close() + if store._engine: + await store._engine.dispose() + + NotificationsRepository._reset_singleton() + SqliteStore._reset_singleton() + + +class TestNotificationsRepository: + """Test suite for NotificationsRepository database operations.""" + + @pytest.mark.asyncio + async def test_list_empty(self, repo): + """List returns empty when no notifications exist.""" + notifications = await repo.list() + assert notifications == [], "Should return empty list when no notifications exist" + + @pytest.mark.asyncio + async def test_create_notification(self, repo): + """Create notification with valid data.""" + payload = { + "name": "Webhook", + "on": ["item_completed"], + "presets": [], + "enabled": True, + "request_url": "https://example.com/webhook", + "request_method": "POST", + "request_type": "json", + "request_data_key": "data", + "request_headers": [{"key": "Authorization", "value": "Bearer token"}], + } + + model = await repo.create(payload) + + assert model.id is not None, "Should generate ID for new notification" + assert model.name == "Webhook", "Should store name correctly" + assert model.request_url == "https://example.com/webhook", "Should store request url" + assert model.request_method == "POST", "Should store request method" + + @pytest.mark.asyncio + async def test_get_by_id(self, repo): + """Get notification by integer ID.""" + created = await repo.create( + { + "name": "Get Test", + "on": [], + "presets": [], + "enabled": True, + "request_url": "https://example.com", + "request_method": "POST", + "request_type": "json", + "request_data_key": "data", + "request_headers": [], + } + ) + + retrieved = await repo.get(created.id) + + assert retrieved is not None, "Should retrieve created notification" + assert retrieved.id == created.id, "Should match created notification id" + + @pytest.mark.asyncio + async def test_update_notification(self, repo): + """Update existing notification.""" + created = await repo.create( + { + "name": "Update Test", + "on": [], + "presets": [], + "enabled": True, + "request_url": "https://example.com", + "request_method": "POST", + "request_type": "json", + "request_data_key": "data", + "request_headers": [], + } + ) + + updated = await repo.update(created.id, {"name": "Updated Name", "enabled": False}) + + assert updated.name == "Updated Name", "Should update name" + assert updated.enabled is False, "Should update enabled flag" + + @pytest.mark.asyncio + async def test_delete_notification(self, repo): + """Delete existing notification.""" + created = await repo.create( + { + "name": "Delete Test", + "on": [], + "presets": [], + "enabled": True, + "request_url": "https://example.com", + "request_method": "POST", + "request_type": "json", + "request_data_key": "data", + "request_headers": [], + } + ) + + deleted = await repo.delete(created.id) + + assert deleted.id == created.id, "Should return deleted notification" + assert await repo.get(created.id) is None, "Deleted notification should not be retrievable" + + @pytest.mark.asyncio + async def test_list_paginated(self, repo): + """List paginated returns correct subset.""" + for i in range(4): + await repo.create( + { + "name": f"Item {i}", + "on": [], + "presets": [], + "enabled": True, + "request_url": "https://example.com", + "request_method": "POST", + "request_type": "json", + "request_data_key": "data", + "request_headers": [], + } + ) + + items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2) + + assert len(items) == 2, "Should return 2 items per page" + assert total == 4, "Should report total count of 4" + assert page == 1, "Should be on page 1" + assert total_pages == 2, "Should have 2 pages total" diff --git a/app/features/presets/__init__.py b/app/features/presets/__init__.py new file mode 100644 index 00000000..33e0f1db --- /dev/null +++ b/app/features/presets/__init__.py @@ -0,0 +1 @@ +"""Presets feature module.""" diff --git a/app/features/presets/defaults.py b/app/features/presets/defaults.py new file mode 100644 index 00000000..850a73fc --- /dev/null +++ b/app/features/presets/defaults.py @@ -0,0 +1,94 @@ +from datetime import UTC, datetime + +DEFAULT_PRESET_UPDATED_AT: datetime = datetime(2026, 1, 25, tzinfo=UTC) + +DEFAULT_PRESETS: list[dict[str, object]] = [ + { + "name": "default", + "default": True, + "cli": "--socket-timeout 30 --download-archive %(archive_file)s", + "description": "Default preset for yt-dlp. It will download whatever yt-dlp decides is the best quality for the video and audio.", + "folder": "", + "template": "", + "cookies": "", + "priority": 0, + "updated_at": DEFAULT_PRESET_UPDATED_AT, + }, + { + "name": "mobile", + "cli": '--socket-timeout 30 --download-archive %(archive_file)s\n-t mp4 --merge-output-format mp4 --add-chapters --remux-video mp4 \n--embed-metadata --embed-thumbnail \n--postprocessor-args "-movflags +faststart"', + "default": True, + "description": "This preset is designed for mobile devices. It will download the best quality video and audio in mp4 format, merge them, and add chapters, metadata, and thumbnail.", + "folder": "", + "template": "", + "cookies": "", + "priority": 0, + "updated_at": DEFAULT_PRESET_UPDATED_AT, + }, + { + "name": "1080p", + "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n-S vcodec:h264 --format 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'", + "default": True, + "description": "Download the best quality video and audio in mp4 format for 1080p resolution.", + "folder": "", + "template": "", + "cookies": "", + "priority": 0, + "updated_at": DEFAULT_PRESET_UPDATED_AT, + }, + { + "name": "720p", + "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n-S vcodec:h264 --format 'bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'", + "default": True, + "description": "Download the best quality video and audio in mp4 format for 720p resolution.", + "folder": "", + "template": "", + "cookies": "", + "priority": 0, + "updated_at": DEFAULT_PRESET_UPDATED_AT, + }, + { + "name": "audio_only", + "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n--extract-audio --add-chapters --embed-metadata --embed-thumbnail --format 'bestaudio/best'", + "default": True, + "description": "This preset is designed to download only the audio of the video. It will extract the audio, add chapters, metadata, and thumbnail.", + "folder": "", + "template": "", + "cookies": "", + "priority": 0, + "updated_at": DEFAULT_PRESET_UPDATED_AT, + }, + { + "name": "info_reader_plugin", + "description": 'This preset generate specific filename format and metadata to work with yt-dlp info reader plugins for jellyfin/emby and plex and to make play state sync work for WatchState.\n\nThere is one more step you need to do via Other > Terminal if you have it enabled or directly from container shell\n\nyt-dlp -I0 --write-info-json --write-thumbnail --convert-thumbnails jpg --paths /downloads/youtube -o "%(channel|Unknown_title)s [%(channel_id|Unknown_id)s]/%(title).180B [%(channel_id|Unknown_id)s].%(ext)s" -- https://www.youtube.com/channel/UCClfFsWcT3N2I7VTXXyt84A\n\nChange the url to the channel you want to download.\n\nFor more information please visit \nhttps://github.com/arabcoders/watchstate/blob/master/FAQ.md#how-to-get-watchstate-working-with-youtube-contentlibrary', + "folder": "", + "template": "%(channel)s [%(channel_id|Unknown_id)s]/Season %(release_date>%Y,upload_date>%Y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(id)s].%(ext)s", + "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n--windows-filenames --write-info-json --embed-info-json \n--convert-thumbnails jpg --write-thumbnail --embed-metadata", + "cookies": "", + "default": True, + "priority": 0, + "updated_at": DEFAULT_PRESET_UPDATED_AT, + }, + { + "name": "nfo_maker_tv", + "description": "This preset generate specific filename format and metadata to work with media servers like jellyfin/emby and plex for episodes.", + "folder": "", + "template": "%(channel)s [%(channel_id|Unknown_id)s]/Season %(release_date>%Y,upload_date>%Y|Unknown)s/S%(release_date>%Y,upload_date>%Y)sE%(release_date>%m%d,upload_date>%m%d)s - %(title).100B [%(id)s].%(ext)s", + "cli": '--socket-timeout 30 --download-archive %(archive_file)s\n--use-postprocessor "NFOMakerPP:when=after_move" \n--windows-filenames --convert-thumbnails jpg --write-thumbnail', + "cookies": "", + "default": True, + "priority": 0, + "updated_at": DEFAULT_PRESET_UPDATED_AT, + }, + { + "name": "nfo_maker_movie", + "description": "This preset generate specific filename format and metadata to work with media servers like jellyfin/emby and plex for movies.", + "folder": "", + "template": "%(channel)s [%(channel_id|Unknown_id)s]/Season %(release_date>%Y,upload_date>%Y|Unknown)s/S%(release_date>%Y,upload_date>%Y)sE%(release_date>%m%d,upload_date>%m%d)s - %(title).100B [%(id)s].%(ext)s", + "cli": '--socket-timeout 30 --download-archive %(archive_file)s\n--use-postprocessor "NFOMakerPP:when=after_move;mode=movie" \n--windows-filenames --convert-thumbnails jpg --write-thumbnail', + "cookies": "", + "default": True, + "priority": 0, + "updated_at": DEFAULT_PRESET_UPDATED_AT, + }, +] diff --git a/app/features/presets/deps.py b/app/features/presets/deps.py new file mode 100644 index 00000000..88bfcf53 --- /dev/null +++ b/app/features/presets/deps.py @@ -0,0 +1,10 @@ +from app.features.presets.repository import PresetsRepository +from app.features.presets.service import Presets + + +def get_presets_repo() -> PresetsRepository: + return PresetsRepository.get_instance() + + +def get_presets_service() -> Presets: + return Presets.get_instance() diff --git a/app/features/presets/migration.py b/app/features/presets/migration.py new file mode 100644 index 00000000..f44c78e3 --- /dev/null +++ b/app/features/presets/migration.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from app.features.core.migration import Migration as FeatureMigration +from app.features.presets.schemas import Preset +from app.features.presets.utils import preset_name +from app.library.config import Config +from app.library.Utils import arg_converter + +if TYPE_CHECKING: + from app.features.presets.repository import PresetsRepository + +LOG: logging.Logger = logging.getLogger(__name__) + + +class Migration(FeatureMigration): + name: str = "presets" + + def __init__(self, repo: PresetsRepository, config: Config | None = None) -> None: + self._config: Config = config or Config.get_instance() + super().__init__(config=self._config) + self._repo: PresetsRepository = repo + self._source_file: Path = Path(self._config.config_path) / "presets.json" + + async def should_run(self) -> bool: + return self._source_file.exists() + + async def migrate(self) -> None: + try: + items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text()) + except Exception as exc: + LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc) + await self._move_file(self._source_file) + return + + if not items: + LOG.warning("No presets found in %s; skipping migration.", self._source_file) + await self._move_file(self._source_file) + return + + inserted = 0 + seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.list()} + + for index, item in enumerate(items): + if not (normalized := self._normalize(item, index, seen_names)): + continue + try: + await self._repo.create(normalized) + inserted += 1 + except Exception as exc: + LOG.exception("Failed to insert preset '%s': %s", normalized["name"], exc) + + LOG.info("Migrated %s preset(s) from %s.", inserted, self._source_file) + await self._move_file(self._source_file) + + def _normalize(self, item: Any, index: int, seen_names: dict[str, int]) -> dict[str, Any] | None: + if not isinstance(item, dict): + LOG.warning("Skipping preset at index %s due to invalid type.", index) + return None + + name: str | None = item.get("name") + if not name or not isinstance(name, str): + LOG.warning("Skipping preset at index %s due to missing name.", index) + return None + + if not (name := preset_name(name)): + LOG.warning("Skipping preset at index %s due to empty name.", index) + return None + + name = self._unique_name(name, seen_names) + + description: str = item.get("description") if isinstance(item.get("description"), str) else "" + folder: str = item.get("folder") if isinstance(item.get("folder"), str) else "" + template: str = item.get("template") if isinstance(item.get("template"), str) else "" + cookies: str = item.get("cookies") if isinstance(item.get("cookies"), str) else "" + cli: str = item.get("cli") if isinstance(item.get("cli"), str) else "" + priority = 0 + if isinstance(item.get("priority"), int) and not isinstance(item.get("priority"), bool): + priority = int(item.get("priority")) + + if not cli and isinstance(item.get("format"), str): + cli = f"--format {item.get('format')}" + + if cli: + try: + arg_converter(args=cli, level=True) + except Exception as exc: + LOG.warning("Skipping preset '%s' due to invalid CLI: %s", name, exc) + return None + + payload = { + "name": name, + "description": description, + "folder": folder, + "template": template, + "cookies": cookies, + "cli": cli, + "default": False, + "priority": priority, + } + + try: + Preset.model_validate(payload) + except Exception as exc: + LOG.warning("Skipping preset '%s' due to validation error: %s", name, exc) + return None + + return payload diff --git a/app/features/presets/models.py b/app/features/presets/models.py new file mode 100644 index 00000000..2ec5b227 --- /dev/null +++ b/app/features/presets/models.py @@ -0,0 +1,27 @@ +from datetime import datetime + +from sqlalchemy import Boolean, Index, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.features.core.models import Base, UTCDateTime, utcnow + + +class PresetModel(Base): + __tablename__: str = "presets" + __table_args__: tuple[Index, ...] = ( + Index("ix_presets_name", "name"), + Index("ix_presets_is_default", "is_default"), + Index("ix_presets_priority", "priority"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + description: Mapped[str] = mapped_column(Text, nullable=False, default="") + folder: Mapped[str] = mapped_column(Text, nullable=False, default="") + template: Mapped[str] = mapped_column(Text, nullable=False, default="") + cookies: Mapped[str] = mapped_column(Text, nullable=False, default="") + cli: Mapped[str] = mapped_column(Text, nullable=False, default="") + default: Mapped[bool] = mapped_column("is_default", Boolean, nullable=False, default=False) + priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False) + updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False) diff --git a/app/features/presets/repository.py b/app/features/presets/repository.py new file mode 100644 index 00000000..c8adf9b3 --- /dev/null +++ b/app/features/presets/repository.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from sqlalchemy import func, or_, select + +from app.features.core.deps import get_session +from app.features.core.schemas import CEFeature, ConfigEvent +from app.features.presets.migration import Migration +from app.features.presets.models import PresetModel +from app.features.presets.utils import preset_name, seed_defaults +from app.library.config import Config +from app.library.Events import Event, EventBus, Events +from app.library.Services import Services +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from aiohttp import web + from sqlalchemy.engine.result import Result + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + from sqlalchemy.sql.selectable import Select + +LOG: logging.Logger = logging.getLogger(__name__) + + +class PresetsRepository(metaclass=Singleton): + def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + self._migrated = False + self.session: AsyncGenerator[AsyncSession] = session or get_session + + async def run_migrations(self) -> None: + if self._migrated: + return + + self._migrated = True + await Migration(repo=self).run() + + def attach(self, _: web.Application) -> None: + async def handle_event(_, __): + await seed_defaults(self) + await self.run_migrations() + await self._update_cache() + await self._ensure_default_preset() + + async def handler(e: Event, __): + if isinstance(e.data, ConfigEvent) and CEFeature.PRESETS == e.data.feature: + LOG.debug("Refreshing presets cache due to configuration update.") + await self._update_cache() + + Services.get_instance().add(__class__.__name__, self) + EventBus.get_instance().subscribe( + Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations" + ).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache") + + async def _update_cache(self) -> None: + from app.features.presets.service import Presets + + await Presets.get_instance().refresh_cache(await self.list()) + + @staticmethod + def get_instance() -> PresetsRepository: + return PresetsRepository() + + async def _ensure_default_preset(self) -> None: + config = Config.get_instance() + if not (default_name := preset_name(config.default_preset)): + config.default_preset = "default" + default_name = config.default_preset + + if await self.get_by_name(default_name) is None: + LOG.error("Default preset '%s' not found, using 'default' preset.", default_name) + config.default_preset = "default" + + async def list(self) -> list[PresetModel]: + async with self.session() as session: + result: Result[tuple[PresetModel]] = await session.execute( + select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc()) + ) + return list(result.scalars().all()) + + async def list_paginated(self, page: int, per_page: int) -> tuple[list[PresetModel], int, int, int]: + async with self.session() as session: + total: int = await self.count() + total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1 + + if page > total_pages and total > 0: + page = total_pages + + query: Select[tuple[PresetModel]] = ( + select(PresetModel) + .order_by(PresetModel.priority.desc(), PresetModel.name.asc()) + .limit(per_page) + .offset((page - 1) * per_page) + ) + result: Result[tuple[PresetModel]] = await session.execute(query) + return list(result.scalars().all()), total, page, total_pages + + async def count(self) -> int: + async with self.session() as session: + result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(PresetModel)) + return int(result.scalar_one()) + + async def get(self, identifier: int | str) -> PresetModel | None: + async with self.session() as session: + if not identifier: + return None + + if isinstance(identifier, int): + clause: ColumnElement[bool] = PresetModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(PresetModel.id == int(identifier), PresetModel.name == identifier) + else: + name = preset_name(identifier) if isinstance(identifier, str) else identifier + if not name: + return None + clause = PresetModel.name == name + + result: Result[tuple[PresetModel]] = await session.execute(select(PresetModel).where(clause).limit(1)) + return result.scalar_one_or_none() + + async def get_by_name(self, name: str, exclude_id: int | None = None) -> PresetModel | None: + async with self.session() as session: + if not (name := preset_name(name)): + return None + + query: Select[tuple[PresetModel]] = select(PresetModel).where(PresetModel.name == name) + if None is not exclude_id: + query = query.where(PresetModel.id != exclude_id) + + result: Result[tuple[PresetModel]] = await session.execute(query.limit(1)) + return result.scalar_one_or_none() + + async def create(self, payload: PresetModel | dict) -> PresetModel: + async with self.session() as session: + model: PresetModel = PresetModel(**payload) if isinstance(payload, dict) else payload + if model.id is not None: + model.id = None + + model.name = preset_name(model.name) + + if await self.get_by_name(name=model.name) is not None: + msg: str = f"Preset with name '{model.name}' already exists." + raise ValueError(msg) + + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def update(self, identifier: int | str, payload: dict[str, Any]) -> PresetModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = PresetModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(PresetModel.id == int(identifier), PresetModel.name == identifier) + else: + clause = PresetModel.name == identifier + + result: Result[tuple[PresetModel]] = await session.execute(select(PresetModel).where(clause).limit(1)) + model: PresetModel | None = result.scalar_one_or_none() + + if None is model: + msg: str = f"Preset '{identifier}' not found." + raise KeyError(msg) + + assert None is not model + + payload.pop("id", None) + payload.pop("created_at", None) + payload.pop("updated_at", None) + + for key, value in payload.items(): + if hasattr(model, key): + setattr(model, key, value) + + model.name = preset_name(model.name) + if await self.get_by_name(name=model.name, exclude_id=model.id) is not None: + msg = f"Preset with name '{model.name}' already exists." + raise ValueError(msg) + + await session.commit() + await session.refresh(model) + return model + + async def delete(self, identifier: int | str) -> PresetModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = PresetModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(PresetModel.id == int(identifier), PresetModel.name == identifier) + else: + clause = PresetModel.name == preset_name(identifier) + + result: Result[tuple[PresetModel]] = await session.execute(select(PresetModel).where(clause).limit(1)) + if not (model := result.scalar_one_or_none()): + msg: str = f"Preset '{identifier}' not found." + raise KeyError(msg) + + assert None is not model + + await session.delete(model) + await session.commit() + return model diff --git a/app/features/presets/router.py b/app/features/presets/router.py new file mode 100644 index 00000000..afb9c786 --- /dev/null +++ b/app/features/presets/router.py @@ -0,0 +1,190 @@ +from typing import Any + +from aiohttp import web +from aiohttp.web import Request, Response +from pydantic import ValidationError + +from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination +from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination +from app.features.presets.repository import PresetsRepository +from app.features.presets.schemas import Preset, PresetList, PresetPatch +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events +from app.library.router import route + + +def _model(model: Any) -> Preset: + return Preset.model_validate(model) + + +def _serialize(model: Any) -> dict: + return _model(model).model_dump() + + +@route("GET", "api/presets/", "presets") +async def presets_list(request: Request, encoder: Encoder, repo: PresetsRepository) -> Response: + page, per_page = normalize_pagination(request) + items, total, current_page, total_pages = await repo.list_paginated(page, per_page) + return web.json_response( + data=PresetList( + items=[_model(model) for model in items], + pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)), + ), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("GET", r"api/presets/{id:\d+}", "presets_get") +async def presets_get(request: Request, encoder: Encoder, repo: PresetsRepository) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await repo.get(identifier)): + return web.json_response({"error": "Preset not found"}, status=web.HTTPNotFound.status_code) + + return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("POST", "api/presets/", "presets_add") +async def presets_add(request: Request, encoder: Encoder, notify: EventBus, repo: PresetsRepository) -> Response: + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + item: Preset = Preset.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate preset.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + payload = item.model_dump(exclude_unset=True) + payload.pop("id", None) + payload["default"] = False + + try: + saved = _serialize(await repo.create(payload)) + except ValueError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code) + + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.PRESETS, action=CEAction.CREATE, data=saved), + ) + return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("PATCH", r"api/presets/{id:\d+}", "presets_patch") +async def presets_patch(request: Request, encoder: Encoder, notify: EventBus, repo: PresetsRepository) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await repo.get(identifier)): + return web.json_response({"error": "Preset not found"}, status=web.HTTPNotFound.status_code) + + if model.default: + return web.json_response( + {"error": "Default presets cannot be modified."}, status=web.HTTPBadRequest.status_code + ) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + validated = PresetPatch.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate preset.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if validated.name and await repo.get_by_name(validated.name, exclude_id=model.id): + return web.json_response( + data={"error": f"Preset with name '{validated.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + payload = validated.model_dump(exclude_unset=True) + payload.pop("default", None) + updated = _serialize(await repo.update(model.id, payload)) + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.PRESETS, action=CEAction.UPDATE, data=updated), + ) + return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("PUT", r"api/presets/{id:\d+}", "presets_update") +async def presets_update(request: Request, encoder: Encoder, notify: EventBus, repo: PresetsRepository) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await repo.get(identifier)): + return web.json_response({"error": "Preset not found"}, status=web.HTTPNotFound.status_code) + + if model.default: + return web.json_response( + {"error": "Default presets cannot be modified."}, status=web.HTTPBadRequest.status_code + ) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + validated = Preset.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate preset.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if validated.name and await repo.get_by_name(validated.name, exclude_id=model.id): + return web.json_response( + data={"error": f"Preset with name '{validated.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + payload = validated.model_dump(exclude_unset=True) + payload.pop("default", None) + payload.pop("id", None) + updated = _serialize(await repo.update(model.id, payload)) + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.PRESETS, action=CEAction.UPDATE, data=updated), + ) + return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("DELETE", r"api/presets/{id:\d+}", "presets_delete") +async def presets_delete(request: Request, encoder: Encoder, notify: EventBus, repo: PresetsRepository) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + if not (model := await repo.get(identifier)): + return web.json_response({"error": "Preset not found"}, status=web.HTTPNotFound.status_code) + + if model.default: + return web.json_response({"error": "Default presets cannot be deleted."}, status=web.HTTPBadRequest.status_code) + + deleted = _serialize(await repo.delete(model.id)) + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.PRESETS, action=CEAction.DELETE, data=deleted), + ) + return web.json_response(data=deleted, status=web.HTTPOk.status_code, dumps=encoder.encode) diff --git a/app/features/presets/schemas.py b/app/features/presets/schemas.py new file mode 100644 index 00000000..fd80f639 --- /dev/null +++ b/app/features/presets/schemas.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator + +from app.features.core.schemas import Pagination +from app.features.core.utils import parse_int +from app.features.presets.utils import preset_name +from app.library.config import Config +from app.library.Utils import arg_converter, create_cookies_file + + +class Preset(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True) + + id: int | None = None + name: str = Field(min_length=1) + description: str = "" + folder: str = "" + template: str = "" + cookies: str = "" + cli: str = "" + default: bool = False + priority: int = 0 + + _cookies_file: Path | None = PrivateAttr(default=None) + + @field_validator("name", mode="before") + @classmethod + def _normalize_name(cls, value: Any) -> str: + if not isinstance(value, str): + msg: str = "Name must be a string." + raise ValueError(msg) + + if not (value := preset_name(value)): + msg = "Name cannot be empty." + raise ValueError(msg) + return value + + @field_validator("priority", mode="before") + @classmethod + def _normalize_priority(cls, value: Any) -> int: + return parse_int(value, field="Priority", minimum=0) + + @field_validator("cli", mode="before") + @classmethod + def _validate_cli(cls, value: Any) -> str: + if not value: + return "" + if not isinstance(value, str): + msg: str = "CLI must be a string." + raise ValueError(msg) + + value = value.strip() + if not value: + return "" + + try: + arg_converter(args=value, level=True) + except Exception as e: + msg = f"Invalid command options for yt-dlp: {e!s}" + raise ValueError(msg) from e + return value + + def get_cookies_file(self, config: Config | None = None) -> Path | None: + if None is not self._cookies_file: + return self._cookies_file + + if not self.cookies or self.id is None: + return None + + config = config or Config.get_instance() + self._cookies_file = create_cookies_file(self.cookies, Path(config.config_path) / "cookies" / f"{self.id}.txt") + return self._cookies_file + + +class PresetPatch(Preset): + model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True) + + name: str | None = None + description: str | None = None + folder: str | None = None + template: str | None = None + cookies: str | None = None + cli: str | None = None + priority: int | None = None + + +class PresetList(BaseModel): + model_config = ConfigDict(from_attributes=True) + + items: list[Preset] = Field(default_factory=list) + pagination: Pagination diff --git a/app/features/presets/service.py b/app/features/presets/service.py new file mode 100644 index 00000000..9d483ba5 --- /dev/null +++ b/app/features/presets/service.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from app.features.presets.models import PresetModel +from app.features.presets.repository import PresetsRepository +from app.features.presets.schemas import Preset +from app.features.presets.utils import preset_name +from app.library.Services import Services +from app.library.Singleton import Singleton + + +class Presets(metaclass=Singleton): + def __init__(self, repo: PresetsRepository | None = None) -> None: + self._repo: PresetsRepository = repo or PresetsRepository.get_instance() + self._cache: list[tuple[int, str, Preset]] = [] + Services.get_instance().add(__class__.__name__, self) + + @staticmethod + def get_instance() -> Presets: + return Presets() + + async def refresh_cache(self, items: list[PresetModel]) -> None: + presets = [Preset.model_validate(item) for item in items] + self._cache = [(preset.id if preset.id is not None else -1, preset.name, preset) for preset in presets] + + def get_all(self) -> list[Preset]: + return [preset for _, _, preset in self._cache] + + def get(self, identifier: int | str) -> Preset | None: + if not identifier: + return None + + if isinstance(identifier, str) and not (identifier := preset_name(identifier)): + return None + + for id, name, preset in self._cache: + if isinstance(identifier, int) and id == identifier: + return preset + if isinstance(identifier, str) and name == identifier: + return preset + + return None + + def has(self, identifier: int | str) -> bool: + return self.get(identifier) is not None diff --git a/app/features/presets/tests/test_presets_repository.py b/app/features/presets/tests/test_presets_repository.py new file mode 100644 index 00000000..dbf13843 --- /dev/null +++ b/app/features/presets/tests/test_presets_repository.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import pytest +import pytest_asyncio + +from app.features.presets.repository import PresetsRepository +from app.library.sqlite_store import SqliteStore + + +@pytest_asyncio.fixture +async def repo(): + PresetsRepository._reset_singleton() + SqliteStore._reset_singleton() + + store = SqliteStore(db_path=":memory:") + await store.get_connection() + + repository = PresetsRepository.get_instance() + yield repository + + if store._conn: + await store._conn.close() + if store._engine: + await store._engine.dispose() + + PresetsRepository._reset_singleton() + SqliteStore._reset_singleton() + + +class TestPresetsRepository: + @pytest.mark.asyncio + async def test_create_and_get(self, repo): + preset = await repo.create({"name": "Custom", "cli": "--format best"}) + + assert preset.id is not None, "Should generate ID for new preset" + assert preset.name == "custom", "Should normalize preset name" + assert preset.cli == "--format best", "Should store preset cli" + + fetched = await repo.get(preset.id) + assert fetched is not None, "Should fetch preset by id" + assert fetched.name == "custom", "Should return matching preset" + + @pytest.mark.asyncio + async def test_create_normalizes_spaces(self, repo): + preset = await repo.create({"name": "My Preset"}) + + assert preset.name == "my_preset", "Should normalize spaces to underscores" + + @pytest.mark.asyncio + async def test_list_orders_by_priority_then_name(self, repo): + await repo.create({"name": "B", "priority": 1}) + await repo.create({"name": "A", "priority": 1}) + await repo.create({"name": "C", "priority": 2}) + + items = await repo.list() + + assert items[0].name == "c", "Highest priority should be first" + assert items[1].name == "a", "Same priority should sort by name" + assert items[2].name == "b", "Same priority should sort by name" + + @pytest.mark.asyncio + async def test_list_paginated(self, repo): + for i in range(5): + await repo.create({"name": f"Item {i}", "priority": i}) + + items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2) + + assert len(items) == 2, "Should return 2 items per page" + assert total == 5, "Should report total count" + assert page == 1, "Should be on page 1" + assert total_pages == 3, "Should have 3 pages total" diff --git a/app/features/presets/utils.py b/app/features/presets/utils.py new file mode 100644 index 00000000..0164da9a --- /dev/null +++ b/app/features/presets/utils.py @@ -0,0 +1,55 @@ +import logging +import re +from datetime import UTC, datetime + +NAME_WHITESPACE_PATTERN = re.compile(r"\s+") + +LOG: logging.Logger = logging.getLogger(__name__) + + +async def seed_defaults(repo) -> None: + from .defaults import DEFAULT_PRESETS + + for preset in DEFAULT_PRESETS: + try: + name = preset.get("name") + if not name or not isinstance(name, str): + LOG.warning("Skipping default preset with invalid name: %s", preset) + continue + + if not (name := preset_name(name)): + LOG.warning("Skipping default preset with empty normalized name: %s", name) + continue + + payload = {**preset} + payload.pop("id", None) + payload["default"] = True + payload["name"] = name + + updated_at_value: datetime | None = None + updated_at_raw = payload.get("updated_at") + if isinstance(updated_at_raw, str): + updated_at_value = datetime.fromisoformat(updated_at_raw) + elif isinstance(updated_at_raw, datetime): + updated_at_value = updated_at_raw + if isinstance(updated_at_value, datetime) and updated_at_value.tzinfo is None: + updated_at_value = updated_at_value.replace(tzinfo=UTC) + payload["updated_at"] = updated_at_value + + existing = await repo.get_by_name(name) + if None is existing: + await repo.create(payload) + continue + + if existing.updated_at and updated_at_value and existing.updated_at >= updated_at_value: + if False is existing.default: + await repo.update(existing.id, {"default": True}) + continue + + await repo.update(existing.id, payload) + except Exception as exc: + LOG.exception("Failed to seed default preset '%s': %s", preset.get("name"), exc) + + +def preset_name(value: str) -> str: + return NAME_WHITESPACE_PATTERN.sub("_", value.strip().lower()) diff --git a/app/features/tasks/__init__.py b/app/features/tasks/__init__.py new file mode 100644 index 00000000..8dfaca03 --- /dev/null +++ b/app/features/tasks/__init__.py @@ -0,0 +1 @@ +"""Tasks Feature.""" diff --git a/app/features/tasks/definitions/__init__.py b/app/features/tasks/definitions/__init__.py new file mode 100644 index 00000000..9b24415c --- /dev/null +++ b/app/features/tasks/definitions/__init__.py @@ -0,0 +1 @@ +"""Tasks Definitions Feature.""" diff --git a/app/features/tasks/definitions/deps.py b/app/features/tasks/definitions/deps.py new file mode 100644 index 00000000..34133f11 --- /dev/null +++ b/app/features/tasks/definitions/deps.py @@ -0,0 +1,5 @@ +from app.features.tasks.definitions.repository import TaskDefinitionsRepository + + +def get_task_definitions_repo() -> TaskDefinitionsRepository: + return TaskDefinitionsRepository.get_instance() diff --git a/app/features/tasks/definitions/handlers/__init__.py b/app/features/tasks/definitions/handlers/__init__.py new file mode 100644 index 00000000..38c0b67e --- /dev/null +++ b/app/features/tasks/definitions/handlers/__init__.py @@ -0,0 +1 @@ +"""Handlers package for task definitions.""" diff --git a/app/library/task_handlers/_base_handler.py b/app/features/tasks/definitions/handlers/_base_handler.py similarity index 86% rename from app/library/task_handlers/_base_handler.py rename to app/features/tasks/definitions/handlers/_base_handler.py index bbf3c32f..0ee1e274 100644 --- a/app/library/task_handlers/_base_handler.py +++ b/app/features/tasks/definitions/handlers/_base_handler.py @@ -4,22 +4,22 @@ from typing import Any import httpx from yt_dlp.utils.networking import random_user_agent +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult from app.library.config import Config from app.library.httpx_client import async_client -from app.library.Tasks import Task, TaskFailure, TaskResult class BaseHandler: @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: return False @staticmethod - async def extract(task: Task, config: Config | None = None) -> TaskResult | TaskFailure: + async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: raise NotImplementedError @classmethod - async def inspect(cls, task: Task, config: Config | None = None) -> TaskResult | TaskFailure: + async def inspect(cls, task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: return await cls.extract(task=task, config=config) @staticmethod diff --git a/app/library/task_handlers/generic.py b/app/features/tasks/definitions/handlers/generic.py similarity index 50% rename from app/library/task_handlers/generic.py rename to app/features/tasks/definitions/handlers/generic.py index 24f1dc56..90e68a72 100644 --- a/app/library/task_handlers/generic.py +++ b/app/features/tasks/definitions/handlers/generic.py @@ -8,26 +8,29 @@ import hashlib import json import logging import re -from collections.abc import Iterable, Mapping, MutableMapping -from dataclasses import dataclass, field -from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any from urllib.parse import urljoin import jmespath from parsel import Selector -from parsel.selector import SelectorList from yt_dlp.utils.networking import random_user_agent +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult +from app.features.tasks.definitions.schemas import ( + ExtractionRule, + TaskDefinition, +) from app.library.cache import Cache from app.library.config import Config from app.library.httpx_client import async_client -from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult -from app.library.Utils import extract_info, get_archive_id +from app.library.Utils import fetch_info, get_archive_id from ._base_handler import BaseHandler if TYPE_CHECKING: + from pathlib import Path + import httpx from parsel.selector import SelectorList @@ -35,533 +38,6 @@ LOG: logging.Logger = logging.getLogger(__name__) CACHE: Cache = Cache() -@dataclass(slots=True) -class MatchRule: - """Represents a single URL matcher compiled to regex.""" - - source: str - """Original source pattern (regex or glob).""" - - regex: re.Pattern[str] - """Compiled regex pattern.""" - - @classmethod - def from_value(cls, value: str | Mapping[str, Any]) -> MatchRule | None: - """ - Create a MatchRule from a string or mapping. - - Args: - value (str|Mapping[str, Any]): A string (treated as glob) or a mapping with 'regex' or 'glob' keys. - - Returns: - (MatchRule|None): A MatchRule instance if successful, None otherwise. - - """ - if isinstance(value, Mapping): - pattern: str | None = value.get("regex") - glob_pattern: str | None = value.get("glob") - raw: str | None = None - - if isinstance(pattern, str) and pattern: - raw = pattern - try: - compiled: re.Pattern[str] = re.compile(pattern) - except re.error as exc: - LOG.error(f"Invalid regex pattern '{pattern}': {exc}") - return None - - return cls(source=raw, regex=compiled) - - if isinstance(glob_pattern, str) and glob_pattern: - raw = glob_pattern - compiled = re.compile(fnmatch.translate(glob_pattern)) - return cls(source=raw, regex=compiled) - - LOG.error("Matcher mapping must include 'regex' or 'glob' key with a string value.") - return None - - if not isinstance(value, str) or not value: - LOG.error(f"Matcher value must be a non-empty string, got '{value!r}'.") - return None - - # Treat plain string as glob pattern for convenience. - compiled = re.compile(fnmatch.translate(value)) - return cls(source=value, regex=compiled) - - def matches(self, url: str) -> bool: - """ - Check if the given URL matches this rule. - - Args: - url (str): The URL to check. - - Returns: - (bool): True if the URL matches, False otherwise. - - """ - return bool(self.regex.match(url)) - - -@dataclass(slots=True) -class PostFilter: - """Regex post-filter applied on extracted values.""" - - pattern: re.Pattern[str] - """Compiled regex pattern.""" - - value_key: str | None = None - """Optional group name or index to extract from the match.""" - - @classmethod - def from_mapping(cls, mapping: Mapping[str, Any]) -> PostFilter | None: - """ - Create a PostFilter from a mapping. - - Args: - mapping (Mapping[str,Any]): A mapping with 'filter' (regex pattern) and optional 'value' (group name or index) keys. - - Returns: - (PostFilter|None): A PostFilter instance if successful, None otherwise. - - """ - pattern: str | None = mapping.get("filter") - if not isinstance(pattern, str) or not pattern: - LOG.error("post_filter requires a non-empty 'filter' string.") - return None - - try: - compiled: re.Pattern[str] = re.compile(pattern) - except re.error as exc: - LOG.error(f"Invalid post_filter regex '{pattern}': {exc}") - return None - - value_key: str | None = mapping.get("value") - if value_key is not None and not isinstance(value_key, str): - LOG.error("post_filter 'value' must be a string if provided.") - return None - - return cls(pattern=compiled, value_key=value_key) - - def apply(self, candidate: str) -> str | None: - """ - Apply the post-filter to the candidate string. - - Args: - candidate (str): The string to filter. - - Returns: - (str|None): The filtered value if matched, None otherwise. - - """ - match: re.Match[str] | None = self.pattern.search(candidate) - if not match: - return None - - if self.value_key: - try: - return match.group(self.value_key) - except IndexError: - LOG.warning( - f"post_filter value index '{self.value_key}' not present in pattern {self.pattern.pattern!r}." - ) - except KeyError: - LOG.warning( - f"post_filter value key '{self.value_key}' not present in pattern {self.pattern.pattern!r}." - ) - return None - - if match.groupdict(): - # Prefer first named group when available. - key, value = next(iter(match.groupdict().items())) - if value is not None: - LOG.debug(f"post_filter using named group '{key}'.") - return value - - if match.groups(): - return match.group(1) - - return match.group(0) - - -@dataclass(slots=True) -class ExtractionRule: - """Single field extraction description.""" - - type: Literal["css", "xpath", "regex"] - """Type of extraction to perform.""" - - expression: str - """CSS selector, XPath expression or regex pattern.""" - - attribute: str | None = None - """Optional attribute to extract (e.g. 'href', 'src', 'text', etc.).""" - - post_filter: PostFilter | None = None - """Optional post-filter to apply on extracted values.""" - - -@dataclass(slots=True) -class EngineConfig: - """Engine selection to fetch the page.""" - - type: Literal["httpx", "selenium"] = "httpx" - """Engine type to use.""" - - options: dict[str, Any] = field(default_factory=dict) - """Engine-specific options.""" - - -@dataclass(slots=True) -class RequestConfig: - """HTTP request configuration.""" - - method: str = "GET" - """HTTP method to use.""" - headers: dict[str, str] = field(default_factory=dict) - """HTTP headers to include.""" - params: dict[str, Any] = field(default_factory=dict) - """Query parameters to include.""" - - data: Any | None = None - """Request body data to include.""" - json: Any | None = None - """Request body JSON data to include.""" - timeout: float | None = None - """Request timeout in seconds.""" - url: str | None = None - """Optional URL to use instead of the task URL.""" - - def normalized_method(self) -> str: - """ - Get the HTTP method in uppercase. - - Returns: - (str): The HTTP method in uppercase. - - """ - return self.method.upper() if isinstance(self.method, str) else "GET" - - -@dataclass(slots=True) -class ResponseConfig: - """Defines how to interpret the response body returned by the fetch engine.""" - - format: Literal["html", "json"] = "html" - """Body format. Defaults to HTML.""" - - -@dataclass(slots=True) -class ContainerDefinition: - """Defines a repeating element with nested field extraction.""" - - selector_type: Literal["css", "xpath", "jsonpath"] - """Type of selector to use for locating container elements.""" - - selector: str - """Selector expression for locating container elements.""" - - fields: dict[str, ExtractionRule] - """Field extraction rules relative to the container.""" - - -@dataclass(slots=True) -class TaskDefinition: - """Full task definition as loaded from disk.""" - - name: str - """Human-readable name of the task definition.""" - source: Path - """Path to the source JSON file.""" - matchers: list[MatchRule] - """List of URL matchers.""" - engine: EngineConfig - """Engine configuration.""" - request: RequestConfig - """Request configuration.""" - parsers: dict[str, ExtractionRule] - """Field extraction rules.""" - container: ContainerDefinition | None = None - """Optional container definition for repeating elements.""" - response: ResponseConfig = field(default_factory=ResponseConfig) - """Response configuration.""" - - def matches(self, url: str) -> bool: - """ - Check if the given URL matches any of the defined matchers. - - Args: - url (str): The URL to check. - - Returns: - (bool): True if any matcher matches the URL, False otherwise. - - """ - return any(rule.matches(url) for rule in self.matchers) - - -def _build_extraction_rule(field: str, raw: Mapping[str, Any], *, source: Path) -> ExtractionRule | None: - """ - Build an ExtractionRule from a raw mapping. - - Args: - field (str): The name of the field being defined. - raw (Mapping[str, Any]): The raw mapping defining the extraction rule. - source (Path): Path to the source JSON file for logging context. - - Returns: - (ExtractionRule|None): An ExtractionRule instance if successful, None otherwise. - - """ - type_value: str | None = raw.get("type") - expression: str | None = raw.get("expression") - - if not isinstance(type_value, str): - LOG.error(f"[{source.name}] Field '{field}' is missing a valid 'type'.") - return None - - if type_value not in {"css", "xpath", "regex", "jsonpath"}: - LOG.error(f"[{source.name}] Field '{field}' has unsupported type '{type_value}'.") - return None - - if not isinstance(expression, str) or not expression: - LOG.error(f"[{source.name}] Field '{field}' requires non-empty 'expression'.") - return None - - attribute: str | None = raw.get("attribute") - if attribute is not None and not isinstance(attribute, str): - LOG.error(f"[{source.name}] Field '{field}' attribute must be a string if provided.") - return None - - post_filter: PostFilter | None = None - if isinstance(raw.get("post_filter"), Mapping): - post_filter = PostFilter.from_mapping(raw["post_filter"]) - - return ExtractionRule(type=type_value, expression=expression, attribute=attribute, post_filter=post_filter) - - -def _build_matchers(raw_match: Iterable[Any], *, source: Path) -> list[MatchRule]: - """ - Build a list of MatchRule instances from raw match definitions. - - Args: - raw_match (Iterable[Any]): An iterable of raw match definitions (strings or mappings). - source (Path): Path to the source JSON file for logging context. - - Returns: - (list[MatchRule]): A list of MatchRule instances. - - """ - matchers: list[MatchRule] = [] - for value in raw_match: - rule: MatchRule | None = MatchRule.from_value(value) - if rule: - matchers.append(rule) - - if not matchers: - LOG.error(f"[{source.name}] No valid match rules found.") - - return matchers - - -def _normalize_mapping(value: Any) -> MutableMapping[str, Any]: - """ - Ensure the value is a mutable mapping. - - Args: - value (Any): The value to check. - - Returns: - (MutableMapping[str, Any]): The value if it's a mutable mapping. - - """ - if isinstance(value, MutableMapping): - return value - - msg = "Expected a mapping for parse/request/engine sections." - raise ValueError(msg) - - -def load_task_definitions(config: Config | None = None) -> list[TaskDefinition]: - """ - Load JSON task definitions from the configured tasks directory. - - Args: - config (Config|None): Optional Config instance. If None, the singleton instance is used. - - Returns: - (list[TaskDefinition]): A list of loaded TaskDefinition instances. - - """ - cfg: Config = config or Config.get_instance() - tasks_dir: Path = Path(cfg.config_path) / "tasks" - - if not tasks_dir.exists(): - return [] - - definitions: list[TaskDefinition] = [] - - for path in sorted(tasks_dir.glob("*.json")): - try: - content: str = path.read_text(encoding="utf-8") - except Exception as exc: - LOG.error(f"Failed to read task configuration '{path}': {exc}") - continue - - try: - raw = json.loads(content) - except Exception as exc: - LOG.error(f"Failed to parse JSON for '{path}': {exc}") - continue - - if not isinstance(raw, Mapping): - LOG.error(f"Task definition in '{path}' must be a JSON object.") - continue - - name: str | None = raw.get("name") - if not isinstance(name, str) or not name.strip(): - LOG.error(f"Task definition '{path}' missing a valid 'name'.") - continue - - match_value: list[str] | None = raw.get("match") - if not isinstance(match_value, Iterable) or isinstance(match_value, (str, bytes)): - LOG.error(f"[{path.name}] 'match' must be a list of patterns.") - continue - - matchers: list[MatchRule] = _build_matchers(match_value, source=path) - if not matchers: - continue - - engine_raw: Any = raw.get("engine", {}) - try: - engine_map: MutableMapping[str, Any] = _normalize_mapping(engine_raw) - except ValueError: - LOG.error(f"[{path.name}] 'engine' must be a JSON object when provided.") - continue - - engine_type: str | None = engine_map.get("type", "httpx") - if engine_type not in ("httpx", "selenium"): - LOG.error(f"[{path.name}] Unsupported engine type '{engine_type}'.") - continue - - engine_options: Any = engine_map.get("options") if isinstance(engine_map.get("options"), Mapping) else {} - engine = EngineConfig(type=engine_type, options=dict(engine_options)) - - request_raw: Any = raw.get("request", {}) - try: - request_map: MutableMapping[str, Any] = _normalize_mapping(request_raw) - except ValueError: - LOG.error(f"[{path.name}] 'request' must be a JSON object when provided.") - continue - - request = RequestConfig( - method=str(request_map.get("method", "GET")), - headers=dict(request_map.get("headers", {})) if isinstance(request_map.get("headers"), Mapping) else {}, - params=dict(request_map.get("params", {})) if isinstance(request_map.get("params"), Mapping) else {}, - data=request_map.get("data"), - json=request_map.get("json"), - timeout=float(request_map.get("timeout")) if request_map.get("timeout") is not None else None, - url=str(request_map.get("url")) if isinstance(request_map.get("url"), str) else None, - ) - - response_raw: Any = raw.get("response", {}) - response_config = ResponseConfig() - if response_raw: - if not isinstance(response_raw, Mapping): - LOG.error(f"[{path.name}] 'response' must be an object when provided.") - continue - - response_type: str = str(response_raw.get("type", "html")).lower() - if response_type not in ("html", "json"): - LOG.error(f"[{path.name}] Unsupported response type '{response_type}'.") - continue - - response_config = ResponseConfig(format=response_type) - - parse_raw: Mapping | None = raw.get("parse") - if not isinstance(parse_raw, Mapping): - LOG.error(f"[{path.name}] 'parse' must be a JSON object mapping fields to instructions.") - continue - - container_definition: ContainerDefinition | None = None - parsers: dict[str, ExtractionRule] = {} - - items_block: Mapping | None = parse_raw.get("items") - if isinstance(items_block, Mapping): - raw_fields: Mapping | None = items_block.get("fields") - if not isinstance(raw_fields, Mapping): - LOG.error(f"[{path.name}] 'items.fields' must be a mapping of field definitions.") - continue - - container_fields: dict[str, ExtractionRule] = {} - for _field, rule in raw_fields.items(): - if not isinstance(_field, str): - LOG.error(f"[{path.name}] Container field names must be strings, got {_field!r}.") - continue - - if not isinstance(rule, Mapping): - LOG.error(f"[{path.name}] Container definition for '{_field}' must be an object.") - continue - - extraction_rule: ExtractionRule | None = _build_extraction_rule(_field, rule, source=path) - if extraction_rule: - container_fields[_field] = extraction_rule - - if "link" not in container_fields: - LOG.error(f"[{path.name}] Container definition is missing required 'link' field.") - continue - - selector_value: str | None = items_block.get("selector") or items_block.get("expression") - if not isinstance(selector_value, str) or not selector_value: - LOG.error(f"[{path.name}] 'items.selector' must be a non-empty string.") - continue - - selector_type = str(items_block.get("type", "css")) - if selector_type not in ("css", "xpath", "jsonpath"): - LOG.error(f"[{path.name}] Unsupported container selector type '{selector_type}'.") - continue - - container_definition = ContainerDefinition( - selector_type=selector_type, - selector=selector_value, - fields=container_fields, - ) - - for _field, rule in parse_raw.items(): - if "items" == _field: - continue - - if not isinstance(_field, str): - LOG.error(f"[{path.name}] Parser field names must be strings, got {_field!r}.") - continue - - if not isinstance(rule, Mapping): - LOG.error(f"[{path.name}] Parser definition for '{_field}' must be an object.") - continue - - extraction_rule = _build_extraction_rule(_field, rule, source=path) - if extraction_rule: - parsers[_field] = extraction_rule - - if container_definition is None and "link" not in parsers: - LOG.error(f"[{path.name}] Missing required 'link' parser definition.") - continue - - definition = TaskDefinition( - name=name.strip(), - source=path, - matchers=matchers, - engine=engine, - request=request, - parsers=parsers, - container=container_definition, - response=response_config, - ) - - definitions.append(definition) - - return definitions - - class GenericTaskHandler(BaseHandler): """Handler that scrapes arbitrary web pages based on JSON task definitions.""" @@ -572,18 +48,7 @@ class GenericTaskHandler(BaseHandler): """Modification times of source files to detect changes.""" @classmethod - def _tasks_dir(cls) -> Path: - """ - Get the path to the tasks directory. - - Returns: - (Path): Path to the tasks directory. - - """ - return Path(Config.get_instance().config_path) / "tasks" - - @classmethod - def _refresh_definitions(cls, force: bool = False) -> None: + async def refresh_definitions(cls, force: bool = False) -> None: """ Refresh the cached task definitions if source files have changed. @@ -591,32 +56,29 @@ class GenericTaskHandler(BaseHandler): force (bool): If True, force reload even if no changes detected. """ - tasks_dir: Path = cls._tasks_dir() + if cls._definitions and not force: + return cls._definitions - if not tasks_dir.exists(): - if cls._definitions or cls._sources_mtime: - cls._definitions = [] - cls._sources_mtime = {} - return + try: + from app.features.tasks.definitions.repository import TaskDefinitionsRepository + from app.features.tasks.definitions.utils import model_to_schema - current: dict[Path, float] = {} - for path in tasks_dir.glob("*.json"): - try: - current[path] = path.stat().st_mtime - except OSError: - LOG.warning(f"Unable to stat task definition '{path}'.") + repo = TaskDefinitionsRepository.get_instance() + models = await repo.list() - if force or not cls._definitions or current != cls._sources_mtime: - cls._definitions = load_task_definitions() - cls._sources_mtime = current + definitions: list[TaskDefinition] = [] + for model in models: + td = model_to_schema(model) + definitions.append(td) + + cls._definitions = definitions + return cls._definitions + except Exception as exc: + LOG.error(f"Failed to load task definitions from database: {exc}") + return [] @classmethod - def refresh_definitions(cls, force: bool = False) -> None: - """Public helper to refresh cached task definitions.""" - cls._refresh_definitions(force=force) - - @classmethod - def _find_definition(cls, url: str) -> TaskDefinition | None: + async def _find_definition(cls, url: str) -> TaskDefinition | None: """ Find a task definition that matches the given URL. @@ -627,19 +89,30 @@ class GenericTaskHandler(BaseHandler): (TaskDefinition|None): A matching TaskDefinition if found, None otherwise. """ - cls._refresh_definitions() + await cls.refresh_definitions() for definition in cls._definitions: + if not definition.enabled: + continue + try: - if definition.matches(url): - return definition + for matcher in definition.match_url: + pattern_str = None + + if matcher.startswith("/") and matcher.endswith("/") and len(matcher) > 2: + pattern_str: str = matcher[1:-1] + else: + pattern_str = fnmatch.translate(matcher) + + if pattern_str and re.match(pattern_str, url): + return definition except Exception as exc: LOG.error(f"Error while matching definition '{definition.name}': {exc}") return None @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: """ Determine if this handler can process the given task. @@ -650,7 +123,7 @@ class GenericTaskHandler(BaseHandler): (bool): True if the handler can process the task, False otherwise. """ - definition: TaskDefinition | None = GenericTaskHandler._find_definition(task.url) + definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url) if definition: LOG.debug(f"'{task.name}': Matched generic task definition '{definition.name}'.") return True @@ -658,15 +131,15 @@ class GenericTaskHandler(BaseHandler): return False @staticmethod - async def extract(task: Task, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004 - definition: TaskDefinition | None = GenericTaskHandler._find_definition(task.url) + async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004 + definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url) if not definition: return TaskFailure(message="No generic task definition matched the provided URL.") ytdlp_opts: dict[str, Any] = task.get_ytdlp_opts().get_all() - target_url: str = definition.request.url or task.url + target_url: str = definition.definition.request.url or task.url - LOG.debug(f"{task.name!r}: Fetching '{target_url}' using engine '{definition.engine.type}'.") + LOG.debug(f"{task.name!r}: Fetching '{target_url}' using engine '{definition.definition.engine.type}'.") try: body_text, json_data = await GenericTaskHandler._fetch_content( @@ -676,10 +149,10 @@ class GenericTaskHandler(BaseHandler): LOG.exception(exc) return TaskFailure(message="Failed to fetch target URL.", error=str(exc)) - if "json" == definition.response.format and json_data is None: + if "json" == definition.definition.response.type and json_data is None: return TaskFailure(message="Expected JSON response but decoding failed.") - if "json" != definition.response.format and not body_text: + if "json" != definition.definition.response.type and not body_text: return TaskFailure(message="Received empty response body.") raw_items: list[dict[str, str]] = GenericTaskHandler._parse_items( @@ -690,9 +163,9 @@ class GenericTaskHandler(BaseHandler): def _generic_id(url): import os - import urllib + from urllib import parse - return urllib.parse.unquote(os.path.splitext(url.rstrip("/").split("/")[-1])[0]) + return parse.unquote(os.path.splitext(url.rstrip("/").split("/")[-1])[0]) for entry in raw_items: if not isinstance(entry, dict): @@ -701,8 +174,8 @@ class GenericTaskHandler(BaseHandler): if not (url := entry.get("link") or entry.get("url")): continue - idDict: str | None = get_archive_id(url=url) - archive_id: str | None = idDict.get("archive_id") + id_dict: dict[str, str | None] = get_archive_id(url=url) + archive_id: str | None = id_dict.get("archive_id") if not archive_id: cache_key: str = hashlib.sha256(f"{task.name}-{url}".encode()).hexdigest() if CACHE.has(cache_key): @@ -714,7 +187,7 @@ class GenericTaskHandler(BaseHandler): f"[{definition.name}]: '{task.name}': Unable to generate static archive id for '{url}' in feed. Doing real request to fetch yt-dlp archive id." ) - info = extract_info( + info = await fetch_info( config=task.get_ytdlp_opts().get_all(), url=url, no_archive=True, @@ -755,7 +228,7 @@ class GenericTaskHandler(BaseHandler): items=task_items, metadata={ "definition": definition.name, - "response_format": definition.response.format, + "response_format": definition.definition.response.type, }, ) @@ -770,14 +243,14 @@ class GenericTaskHandler(BaseHandler): Args: url (str): The URL to fetch. - definition (TaskDefinition): The task definition specifying the engine and request details. + definition (TaskDefinitionRuntimeSchema): The task definition specifying the engine and request details. ytdlp_opts (dict[str, Any]): yt-dlp options that may influence fetching Returns: (str|None): The fetched HTML content if successful, None otherwise. """ - if "selenium" == definition.engine.type: + if "selenium" == definition.definition.engine.type: return await GenericTaskHandler._fetch_with_selenium(url=url, definition=definition) return await GenericTaskHandler._fetch_with_httpx(url=url, definition=definition, ytdlp_opts=ytdlp_opts) @@ -793,14 +266,14 @@ class GenericTaskHandler(BaseHandler): Args: url (str): The URL to fetch. - definition (TaskDefinition): The task definition specifying the request details. + definition (TaskDefinitionRuntimeSchema): The task definition specifying the request details. ytdlp_opts (dict[str, Any]): yt-dlp options that may influence fetching Returns: (str|None): The fetched HTML content if successful, None otherwise. """ - headers: dict[str, str] = {**definition.request.headers} + headers: dict[str, str] = {**definition.definition.request.headers} client_options: dict[str, Any] = { "headers": { "User-Agent": random_user_agent(), @@ -825,20 +298,20 @@ class GenericTaskHandler(BaseHandler): if proxy := ytdlp_opts.get("proxy"): client_options["proxy"] = proxy - timeout_value: float | Any = definition.request.timeout or ytdlp_opts.get("socket_timeout", 120) + timeout_value: float | Any = definition.definition.request.timeout or ytdlp_opts.get("socket_timeout", 120) async with async_client(**client_options) as client: response: httpx.Response = await client.request( - method=definition.request.normalized_method(), + method=definition.definition.request.method.upper(), url=url, - params=definition.request.params or None, - data=definition.request.data, - json=definition.request.json, + params=definition.definition.request.params or None, + data=definition.definition.request.data, + json=definition.definition.request.json_data, timeout=timeout_value, ) response.raise_for_status() - if "json" == definition.response.format: + if "json" == definition.definition.response.type: try: json_data: dict[str, Any] = response.json() except Exception as exc: @@ -859,7 +332,7 @@ class GenericTaskHandler(BaseHandler): Args: url (str): The URL to fetch. - definition (TaskDefinition): The task definition specifying the engine options. + definition (TaskDefinitionRuntimeSchema): The task definition specifying the engine options. Returns: (str|None): The fetched HTML content if successful, None otherwise. @@ -873,15 +346,20 @@ class GenericTaskHandler(BaseHandler): from selenium.webdriver.support.ui import WebDriverWait except ImportError as exc: LOG.error(f"Selenium engine requested but selenium is not installed: {exc!s}.") - return None + return (None, None) - options_map: dict[str, Any] = definition.engine.options - command_executor: str | None = options_map.get("url", "http://localhost:4444/wd/hub") + options_map: dict[str, Any] = definition.definition.engine.options + command_executor_value = options_map.get("url") + command_executor: str = ( + str(command_executor_value) + if isinstance(command_executor_value, str) and command_executor_value + else "http://localhost:4444/wd/hub" + ) browser: str = str(options_map.get("browser", "chrome")).lower() if "chrome" != browser: LOG.error(f"Unsupported selenium browser '{browser}'. Only 'chrome' is supported.") - return None + return (None, None) arguments: list[str] | str = options_map.get("arguments", ["--headless", "--disable-gpu"]) if isinstance(arguments, str): @@ -890,8 +368,8 @@ class GenericTaskHandler(BaseHandler): wait_for: Mapping | None = ( options_map.get("wait_for") if isinstance(options_map.get("wait_for"), Mapping) else None ) - wait_timeout = float(options_map.get("wait_timeout", 15)) - page_load_timeout = float(options_map.get("page_load_timeout", 60)) + wait_timeout = float(options_map.get("wait_timeout") or 15) + page_load_timeout = float(options_map.get("page_load_timeout") or 60) def load_page() -> str | None: chrome_options = ChromeOptions() @@ -929,7 +407,7 @@ class GenericTaskHandler(BaseHandler): Parse the HTML content and extract items based on the definition. Args: - definition (TaskDefinition): The task definition specifying the parsers. + definition (TaskDefinitionRuntimeSchema): The task definition specifying the parsers. html (str): The HTML content to parse. base_url (str): The base URL to resolve relative links. json_data (Any|None): The JSON data to parse if applicable. @@ -938,12 +416,12 @@ class GenericTaskHandler(BaseHandler): (list[dict[str, str]]): A list of extracted items as dictionaries. """ - if "json" == definition.response.format: + if "json" == definition.definition.response.type: return GenericTaskHandler._parse_json_items(definition, json_data, base_url) selector = Selector(text=html) - if definition.container: + if definition.definition.parse.get("items"): return GenericTaskHandler._parse_with_container( definition=definition, selector=selector, @@ -953,7 +431,10 @@ class GenericTaskHandler(BaseHandler): extracted: dict[str, list[str]] = {} - for _field, rule in definition.parsers.items(): + for _field, rule_data in definition.definition.parse.field_items(): + if not isinstance(rule_data, dict): + continue + rule = ExtractionRule.model_validate(rule_data) values: list[str] = GenericTaskHandler._execute_rule(field=_field, selector=selector, html=html, rule=rule) extracted[_field] = values @@ -997,13 +478,16 @@ class GenericTaskHandler(BaseHandler): LOG.debug(f"Definition '{definition.name}' expects JSON but no data was parsed.") return [] - if definition.container: + if definition.definition.parse.get("items"): return GenericTaskHandler._parse_json_with_container(definition, json_data, base_url) items: list[dict[str, str]] = [] entry: dict[str, str] = {} - for _field, rule in definition.parsers.items(): + for _field, rule_data in definition.definition.parse.field_items(): + if not isinstance(rule_data, dict): + continue + rule = ExtractionRule.model_validate(rule_data) values: list[str] = GenericTaskHandler._execute_json_rule(_field, json_data, rule) if values: if "link" == _field: @@ -1023,18 +507,19 @@ class GenericTaskHandler(BaseHandler): html: str, base_url: str, ) -> list[dict[str, str]]: - container: ContainerDefinition | None = definition.container + container: dict[str, Any] | None = definition.definition.parse.get("items") if not container: return [] - if "jsonpath" == container.selector_type: - LOG.error( - f"Container selector type 'jsonpath' requires response type 'json'. Definition '{definition.name}'." - ) + container_type = container.get("type", "css") + container_selector = container.get("selector") or container.get("expression") or "" + if not container_selector: + LOG.error(f"Container missing selector/expression. Definition '{definition.name}'.") return [] + container_fields = container.get("fields", {}) selection: SelectorList[Selector] = ( - selector.css(container.selector) if "css" == container.selector_type else selector.xpath(container.selector) + selector.css(container_selector) if "css" == container_type else selector.xpath(container_selector) ) items: list[dict[str, str]] = [] @@ -1043,7 +528,8 @@ class GenericTaskHandler(BaseHandler): node_html: Any | str = node.get() or html entry: dict[str, str] = {} - for _field, rule in container.fields.items(): + for _field, rule_data in container_fields.items(): + rule = ExtractionRule.model_validate(rule_data) values: list[str] = GenericTaskHandler._execute_rule( field=_field, selector=node, @@ -1073,15 +559,19 @@ class GenericTaskHandler(BaseHandler): json_data: Any, base_url: str, ) -> list[dict[str, str]]: - container: ContainerDefinition | None = definition.container + container: dict[str, Any] | None = definition.definition.parse.get("items") if not container: return [] - if "jsonpath" != container.selector_type: + container_type = container.get("type", "css") + container_selector = container.get("selector") or container.get("expression", "") + container_fields = container.get("fields", {}) + + if "jsonpath" != container_type: LOG.error(f"JSON response requires container selector type 'jsonpath'. Definition '{definition.name}'.") return [] - nodes: Any = GenericTaskHandler._json_search(json_data, container.selector) + nodes: Any = GenericTaskHandler._json_search(json_data, container_selector) if nodes is None: return [] @@ -1093,7 +583,8 @@ class GenericTaskHandler(BaseHandler): for node in nodes: entry: dict[str, str] = {} - for _field, rule in container.fields.items(): + for _field, rule_data in container_fields.items(): + rule = ExtractionRule.model_validate(rule_data) values: list[str] = GenericTaskHandler._execute_json_rule(_field, node, rule) if not values: continue @@ -1175,7 +666,7 @@ class GenericTaskHandler(BaseHandler): field (str): The name of the field being extracted. selector (Selector): The parsel Selector for the HTML content. html (str): The raw HTML content. - rule (ExtractionRule): The extraction rule to execute. + rule (ExtractionRuleSchema): The extraction rule to execute. Returns: (list[str]): A list of extracted values. @@ -1267,7 +758,7 @@ class GenericTaskHandler(BaseHandler): if attr and attr not in {"html", "outer_html", "text", "inner_text"}: try: - attributes: dict[str, str] = sel.attrib + attributes: dict[str, str] | None = sel.attrib except AttributeError: attributes = None @@ -1281,7 +772,7 @@ class GenericTaskHandler(BaseHandler): if attr is None and "link" == field.lower(): href = None try: - attributes = sel.attrib + attributes: dict[str, str] | None = sel.attrib except AttributeError: attributes = None @@ -1307,7 +798,7 @@ class GenericTaskHandler(BaseHandler): Args: value (str|None): The extracted value to filter. - rule (ExtractionRule): The extraction rule containing the post-filter. + rule (ExtractionRuleSchema): The extraction rule containing the post-filter. Returns: (str|None): The filtered value if applicable, None otherwise. @@ -1318,6 +809,30 @@ class GenericTaskHandler(BaseHandler): cleaned: str = value.strip() if rule.post_filter: - return rule.post_filter.apply(cleaned) + # Apply post-filter inline (removed helper method) + try: + pattern = re.compile(rule.post_filter.filter) + match = pattern.search(cleaned) + if not match: + return None + + if rule.post_filter.value: + try: + return match.group(rule.post_filter.value) + except (IndexError, KeyError): + return None + + if match.groupdict(): + # Prefer first named group when available + for group_value in match.groupdict().values(): + if group_value is not None: + return group_value + + if match.groups(): + return match.group(1) + + return match.group(0) + except re.error: + return None return cleaned or None diff --git a/app/library/task_handlers/rss.py b/app/features/tasks/definitions/handlers/rss.py similarity index 96% rename from app/library/task_handlers/rss.py rename to app/features/tasks/definitions/handlers/rss.py index 40bb62bc..0f538a04 100644 --- a/app/library/task_handlers/rss.py +++ b/app/features/tasks/definitions/handlers/rss.py @@ -4,9 +4,9 @@ import re from typing import TYPE_CHECKING, Any from xml.etree.ElementTree import Element +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.library.cache import Cache -from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult -from app.library.Utils import extract_info, get_archive_id +from app.library.Utils import fetch_info, get_archive_id from ._base_handler import BaseHandler @@ -24,13 +24,13 @@ class RssGenericHandler(BaseHandler): ) @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: LOG.debug(f"'{task.name}': Checking if task URL is parsable RSS feed: {task.url}") return RssGenericHandler.parse(task.url) is not None @staticmethod async def _get( - task: Task, + task: HandleTask, params: dict, parsed: dict[str, str], ) -> tuple[str, list[dict[str, str]], int]: @@ -131,7 +131,7 @@ class RssGenericHandler(BaseHandler): return feed_url, items, real_count @staticmethod - async def extract(task: Task) -> TaskResult | TaskFailure: + async def extract(task: HandleTask) -> TaskResult | TaskFailure: """ Extract items from an RSS/Atom feed. @@ -179,7 +179,7 @@ class RssGenericHandler(BaseHandler): "Doing real request to fetch yt-dlp archive ID." ) - info = extract_info( + info = await fetch_info( config=params, url=url, no_archive=True, diff --git a/app/library/task_handlers/tver.py b/app/features/tasks/definitions/handlers/tver.py similarity index 93% rename from app/library/task_handlers/tver.py rename to app/features/tasks/definitions/handlers/tver.py index 167a384f..81c2a4e8 100644 --- a/app/library/task_handlers/tver.py +++ b/app/features/tasks/definitions/handlers/tver.py @@ -1,7 +1,7 @@ import logging import re -from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.library.Utils import get_archive_id from ._base_handler import BaseHandler @@ -10,9 +10,9 @@ LOG: logging.Logger = logging.getLogger(__name__) class TverHandler(BaseHandler): - SERIES_API = "https://platform-api.tver.jp/service/api/v1/callSeriesEpisodes/{id}" - SESSION_API = "https://platform-api.tver.jp/v2/api/platform_users/browser/create" - HEADERS = { + SERIES_API: str = "https://platform-api.tver.jp/service/api/v1/callSeriesEpisodes/{id}" + SESSION_API: str = "https://platform-api.tver.jp/v2/api/platform_users/browser/create" + HEADERS: dict[str, str] = { "x-tver-platform-type": "web", "Origin": "https://tver.jp", "Referer": "https://tver.jp/", @@ -21,7 +21,7 @@ class TverHandler(BaseHandler): RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?tver\.jp\/series\/(?Psr[a-z0-9_]+)$") @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: LOG.debug(f"Checking if task '{task.name}' is using parsable Tver series URL: {task.url}") return TverHandler.parse(task.url) is not None @@ -58,7 +58,7 @@ class TverHandler(BaseHandler): @staticmethod async def _collect_feed( - task: Task, + task: HandleTask, params: dict, series_id: str, ) -> tuple[str, list[dict[str, str]], bool]: @@ -142,7 +142,7 @@ class TverHandler(BaseHandler): return feed_url, items, has_items @staticmethod - async def extract(task: Task) -> TaskResult | TaskFailure: + async def extract(task: HandleTask) -> TaskResult | TaskFailure: series_id: str | None = TverHandler.parse(task.url) if not series_id: return TaskFailure(message="Unrecognized Tver series URL.") diff --git a/app/library/task_handlers/twitch.py b/app/features/tasks/definitions/handlers/twitch.py similarity index 93% rename from app/library/task_handlers/twitch.py rename to app/features/tasks/definitions/handlers/twitch.py index 8ba3c91b..25714bc2 100644 --- a/app/library/task_handlers/twitch.py +++ b/app/features/tasks/definitions/handlers/twitch.py @@ -3,7 +3,7 @@ import re from typing import TYPE_CHECKING from xml.etree.ElementTree import Element -from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.library.Utils import get_archive_id from ._base_handler import BaseHandler @@ -15,18 +15,18 @@ LOG: logging.Logger = logging.getLogger(__name__) class TwitchHandler(BaseHandler): - FEED = "https://twitchrss.appspot.com/vodonly/{handle}" + FEED: str = "https://twitchrss.appspot.com/vodonly/{handle}" RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?twitch\.tv\/(?P[a-z0-9_]{3,25})(?:\/.*)?$") @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: LOG.debug(f"Checking if task '{task.name}' is using parsable Twitch URL: {task.url}") return TwitchHandler.parse(task.url) is not None @staticmethod async def _collect_feed( - task: Task, + task: HandleTask, params: dict, handle_name: str, ) -> tuple[str, list[dict[str, str]], bool]: @@ -74,7 +74,7 @@ class TwitchHandler(BaseHandler): return feed_url, items, has_items @staticmethod - async def extract(task: Task) -> TaskResult | TaskFailure: + async def extract(task: HandleTask) -> TaskResult | TaskFailure: handle_name: str | None = TwitchHandler.parse(task.url) if not handle_name: return TaskFailure(message="Unrecognized Twitch channel URL.") diff --git a/app/library/task_handlers/youtube.py b/app/features/tasks/definitions/handlers/youtube.py similarity index 93% rename from app/library/task_handlers/youtube.py rename to app/features/tasks/definitions/handlers/youtube.py index 4db7be5c..887ba78a 100644 --- a/app/library/task_handlers/youtube.py +++ b/app/features/tasks/definitions/handlers/youtube.py @@ -3,7 +3,7 @@ import re from typing import TYPE_CHECKING, Any from xml.etree.ElementTree import Element -from app.library.Tasks import Task, TaskFailure, TaskItem, TaskResult +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult from app.library.Utils import get_archive_id from ._base_handler import BaseHandler @@ -15,7 +15,7 @@ LOG: logging.Logger = logging.getLogger(__name__) class YoutubeHandler(BaseHandler): - FEED = "https://www.youtube.com/feeds/videos.xml?{type}={id}" + FEED: str = "https://www.youtube.com/feeds/videos.xml?{type}={id}" CHANNEL_REGEX: re.Pattern[str] = re.compile( r"^https?://(?:www\.)?youtube\.com/(?:channel/(?PUC[0-9A-Za-z_-]{22})|)/?$" @@ -26,12 +26,12 @@ class YoutubeHandler(BaseHandler): ) @staticmethod - def can_handle(task: Task) -> bool: + async def can_handle(task: HandleTask) -> bool: LOG.debug(f"'{task.name}': Checking if task URL is parsable YouTube URL: {task.url}") return YoutubeHandler.parse(task.url) is not None @staticmethod - async def _get(task: Task, params: dict, parsed: dict[str, str]) -> tuple[str, list[dict[str, str]], int]: + async def _get(task: HandleTask, params: dict, parsed: dict[str, str]) -> tuple[str, list[dict[str, str]], int]: """ Fetch the feed and return raw entries. @@ -89,7 +89,7 @@ class YoutubeHandler(BaseHandler): return feed_url, items, real_count @staticmethod - async def extract(task: Task) -> TaskResult | TaskFailure: + async def extract(task: HandleTask) -> TaskResult | TaskFailure: parsed: dict[str, str] | None = YoutubeHandler.parse(task.url) if not parsed: return TaskFailure(message="Unrecognized YouTube channel or playlist URL.") diff --git a/app/features/tasks/definitions/migration.py b/app/features/tasks/definitions/migration.py new file mode 100644 index 00000000..74d519c9 --- /dev/null +++ b/app/features/tasks/definitions/migration.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from app.features.core.migration import Migration as FeatureMigration +from app.library.config import Config + +if TYPE_CHECKING: + from app.features.tasks.definitions.repository import TaskDefinitionsRepository + +LOG: logging.Logger = logging.getLogger(__name__) + + +class Migration(FeatureMigration): + name: str = "task_definitions" + + def __init__(self, repo: TaskDefinitionsRepository, config: Config | None = None) -> None: + self._config: Config = config or Config.get_instance() + super().__init__(config=self._config) + self._repo: TaskDefinitionsRepository = repo + self._source_dir: Path = Path(self._config.config_path) / "tasks" + + async def should_run(self) -> bool: + if not self._source_dir.exists(): + return False + + return any(self._source_dir.glob("*.json")) + + async def migrate(self) -> None: + if await self._repo.count() > 0: + LOG.warning("Task definitions already exist in the database; skipping migration.") + await self._archive_sources() + return + + inserted = 0 + seen_names: dict[str, int] = {} + + for path in sorted(self._source_dir.glob("*.json")): + normalized = await self._normalize(path, seen_names) + if not normalized: + await self._move_file(path) + continue + + try: + await self._repo.create(normalized) + inserted += 1 + except Exception as exc: + LOG.exception("Failed to insert task definition '%s': %s", normalized.get("name"), exc) + finally: + await self._move_file(path) + + LOG.info("Migrated %s task definition(s) from %s.", inserted, self._source_dir) + + async def _archive_sources(self) -> None: + for path in self._source_dir.glob("*.json"): + await self._move_file(path) + + async def _normalize(self, path: Path, seen_names: dict[str, int]) -> dict[str, Any] | None: + try: + content = path.read_text(encoding="utf-8") + except Exception as exc: + LOG.error("Failed to read task definition '%s': %s", path, exc) + return None + + try: + payload = json.loads(content) + except Exception as exc: + LOG.error("Failed to parse JSON for '%s': %s", path, exc) + return None + + if not isinstance(payload, dict): + LOG.error("Task definition in '%s' must be a JSON object.", path) + return None + + if "match" in payload and "match_url" not in payload: + payload["match_url"] = payload.pop("match") + + # Normalize match_url from old object format to new string format + if "match_url" in payload and isinstance(payload["match_url"], list): + normalized_match: list[str] = [] + for item in payload["match_url"]: + if isinstance(item, str): + normalized_match.append(item) + elif isinstance(item, dict): + if "regex" in item and isinstance(item["regex"], str): + # Convert {regex: "pattern"} to /pattern/ + normalized_match.append(f"/{item['regex']}/") + elif "glob" in item and isinstance(item["glob"], str): + # Convert {glob: "pattern"} to pattern + normalized_match.append(item["glob"]) + payload["match_url"] = normalized_match + + # Rename request.json to request.json_data + if "request" in payload and isinstance(payload["request"], dict) and "json" in payload["request"]: + payload["request"]["json_data"] = payload["request"].pop("json") + + if "definition" not in payload: + definition_fields = {} + for field in ["parse", "engine", "request", "response"]: + if field in payload: + definition_fields[field] = payload.pop(field) + + if definition_fields: + payload["definition"] = definition_fields + # Also handle nested definition.request.json + elif isinstance(payload["definition"], dict): + if ( + "request" in payload["definition"] + and isinstance(payload["definition"]["request"], dict) + and "json" in payload["definition"]["request"] + ): + payload["definition"]["request"]["json_data"] = payload["definition"]["request"].pop("json") + + name_value = payload.get("name") + if not isinstance(name_value, str) or not name_value.strip(): + LOG.error("Task definition in '%s' missing a valid name.", path) + return None + + name = self._unique_name(name_value.strip(), seen_names) + payload["name"] = name + + # Repository will handle validation and field extraction + return payload diff --git a/app/features/tasks/definitions/models.py b/app/features/tasks/definitions/models.py new file mode 100644 index 00000000..def000fd --- /dev/null +++ b/app/features/tasks/definitions/models.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from datetime import datetime # noqa: TC003 + +from sqlalchemy import JSON, Boolean, Index, Integer, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.features.core.models import Base, UTCDateTime, utcnow + + +class TaskDefinitionModel(Base): + __tablename__: str = "task_definitions" + __table_args__: tuple[Index, ...] = ( + Index("ix_task_definitions_name", "name"), + Index("ix_task_definitions_priority", "priority"), + Index("ix_task_definitions_match_url", "match_url"), + Index("ix_task_definitions_enabled", "enabled"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + match_url: Mapped[list] = mapped_column(JSON, nullable=False) + definition: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False) + updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False) diff --git a/app/features/tasks/definitions/repository.py b/app/features/tasks/definitions/repository.py new file mode 100644 index 00000000..80dbcf4b --- /dev/null +++ b/app/features/tasks/definitions/repository.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from sqlalchemy import func, or_, select + +from app.features.core.deps import get_session +from app.features.core.schemas import CEFeature, ConfigEvent +from app.features.tasks.definitions.migration import Migration +from app.features.tasks.definitions.models import TaskDefinitionModel +from app.library.Events import Event, EventBus, Events +from app.library.Services import Services +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from sqlalchemy.engine.result import Result + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + from sqlalchemy.sql.selectable import Select + +LOG: logging.Logger = logging.getLogger(__name__) + + +class TaskDefinitionsRepository(metaclass=Singleton): + def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + self._migrated = False + self.session: AsyncGenerator[AsyncSession] = session or get_session + + async def run_migrations(self) -> None: + if self._migrated: + return + + self._migrated = True + await Migration(repo=self).run() + + @staticmethod + def get_instance() -> TaskDefinitionsRepository: + return TaskDefinitionsRepository() + + def attach(self, _: Any) -> None: + async def handle_event(_, __): + await self.run_migrations() + + async def handler(e: Event, __): + from app.features.tasks.definitions.handlers.generic import GenericTaskHandler + + if isinstance(e.data, ConfigEvent) and CEFeature.TASKS_DEFINITIONS == e.data.feature: + LOG.debug("Refreshing task definitions due to configuration update.") + await GenericTaskHandler.refresh_definitions(force=True) + + Services.get_instance().add(__class__.__name__, self) + EventBus.get_instance().subscribe( + Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations" + ).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions") + + async def list(self) -> list[TaskDefinitionModel]: + async with self.session() as session: + result: Result[tuple[TaskDefinitionModel]] = await session.execute( + select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc()) + ) + return list(result.scalars().all()) + + async def list_paginated(self, page: int, per_page: int) -> tuple[list[TaskDefinitionModel], int, int, int]: + async with self.session() as session: + total: int = await self.count() + total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1 + + if page > total_pages and total > 0: + page = total_pages + + query: Select[tuple[TaskDefinitionModel]] = ( + select(TaskDefinitionModel) + .order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc()) + .limit(per_page) + .offset((page - 1) * per_page) + ) + result: Result[tuple[TaskDefinitionModel]] = await session.execute(query) + return list(result.scalars().all()), total, page, total_pages + + async def count(self) -> int: + async with self.session() as session: + result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(TaskDefinitionModel)) + return int(result.scalar_one()) + + async def get(self, identifier: int | str) -> TaskDefinitionModel | None: + async with self.session() as session: + if not identifier: + return None + + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskDefinitionModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskDefinitionModel.id == int(identifier), TaskDefinitionModel.name == identifier) + else: + clause = TaskDefinitionModel.name == identifier + + result: Result[tuple[TaskDefinitionModel]] = await session.execute( + select(TaskDefinitionModel).where(clause).limit(1) + ) + return result.scalar_one_or_none() + + async def get_by_name(self, name: str, exclude_id: int | None = None) -> TaskDefinitionModel | None: + async with self.session() as session: + query: Select[tuple[TaskDefinitionModel]] = select(TaskDefinitionModel).where( + TaskDefinitionModel.name == name + ) + if exclude_id is not None: + query = query.where(TaskDefinitionModel.id != exclude_id) + + result: Result[tuple[TaskDefinitionModel]] = await session.execute(query.limit(1)) + return result.scalar_one_or_none() + + async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel: + async with self.session() as session: + model: TaskDefinitionModel = TaskDefinitionModel(**payload) if isinstance(payload, dict) else payload + if model.id is not None: + model.id = None + + if await self.get_by_name(name=model.name) is not None: + msg: str = f"Task definition with name '{model.name}' already exists." + raise ValueError(msg) + + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def update(self, identifier: int | str, payload: dict[str, Any]) -> TaskDefinitionModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskDefinitionModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskDefinitionModel.id == int(identifier), TaskDefinitionModel.name == identifier) + else: + clause = TaskDefinitionModel.name == identifier + + result: Result[tuple[TaskDefinitionModel]] = await session.execute( + select(TaskDefinitionModel).where(clause).limit(1) + ) + model: TaskDefinitionModel | None = result.scalar_one_or_none() + + if not model: + msg: str = f"Task definition '{identifier}' not found." + raise KeyError(msg) + + payload.pop("id", None) + payload.pop("created_at", None) + payload.pop("updated_at", None) + + if "name" in payload and await self.get_by_name(name=payload["name"], exclude_id=model.id) is not None: + msg = f"Task definition with name '{payload['name']}' already exists." + raise ValueError(msg) + + for key, value in payload.items(): + if hasattr(model, key): + setattr(model, key, value) + + await session.commit() + await session.refresh(model) + return model + + async def delete(self, identifier: int | str) -> TaskDefinitionModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskDefinitionModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskDefinitionModel.id == int(identifier), TaskDefinitionModel.name == identifier) + else: + clause = TaskDefinitionModel.name == identifier + + result: Result[tuple[TaskDefinitionModel]] = await session.execute( + select(TaskDefinitionModel).where(clause).limit(1) + ) + + if not (model := result.scalar_one_or_none()): + msg: str = f"Task definition '{identifier}' not found." + raise KeyError(msg) + + await session.delete(model) + await session.commit() + return model diff --git a/app/features/tasks/definitions/results.py b/app/features/tasks/definitions/results.py new file mode 100644 index 00000000..e9d293a9 --- /dev/null +++ b/app/features/tasks/definitions/results.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from app.features.tasks.schemas import Task as TaskSchema +from app.library.YTDLPOpts import YTDLPOpts + +from .utils import split_inspect_metadata + + +class HandleTask(TaskSchema): + def get_ytdlp_opts(self) -> YTDLPOpts: + """ + Get the yt-dlp options for the task. + + Returns: + YTDLPOpts: The yt-dlp options. + + """ + from app.library.YTDLPOpts import YTDLPOpts + + params: YTDLPOpts = YTDLPOpts.get_instance() + + if self.preset: + params = params.preset(name=self.preset) + + if self.cli: + params = params.add_cli(self.cli, from_user=True) + + if self.template: + params = params.add({"outtmpl": {"default": self.template}}, from_user=False) + + return params + + async def mark(self) -> tuple[bool, str]: + """ + Mark the task's items as downloaded in the archive file. + + Returns: + tuple[bool, str]: A tuple indicating success and a message. + + """ + from app.library.Utils import archive_add + + ret: tuple[bool, str] | dict[str, Any] = await self._mark_logic() + if isinstance(ret, tuple): + return ret + + archive_file: Path = ret.get("file") + items: set[str] = ret.get("items", set()) + + if len(items) < 1 or not archive_add(archive_file, list(items)): + return (True, "No new items to mark as downloaded.") + + return (True, f"Task '{self.name}' items marked as downloaded.") + + async def unmark(self) -> tuple[bool, str]: + """ + Unmark the task's items from the archive file. + + Returns: + tuple[bool, str]: A tuple indicating success and a message. + + """ + from app.library.Utils import archive_delete + + ret: tuple[bool, str] | dict[str, Any] = await self._mark_logic() + if isinstance(ret, tuple): + return ret + + archive_file: Path = ret.get("file") + items: set[str] = ret.get("items", set()) + + if len(items) < 1 or not archive_delete(archive_file, list(items)): + return (True, "No items to remove from archive file.") + + return (True, f"Removed '{self.name}' items from archive file.") + + async def fetch_metadata(self, full: bool = False) -> tuple[dict[str, Any] | None, bool, str]: + """ + Fetch metadata for the task's URL. + + Args: + full (bool): Whether to fetch full metadata including all entries for playlists. + + Returns: + tuple[dict[str, Any]|None, bool, str]: A tuple containing the metadata (or None on failure), a boolean + indicating if the operation was successful, and a message. + + """ + from app.library.ytdlp import fetch_info + + if not self.url: + return ({}, False, "No URL found in task parameters.") + + params = self.get_ytdlp_opts() + if not full: + params.add_cli("-I0", from_user=False) + + params_dict = params.get_all() + + ie_info: dict | None = await fetch_info( + params_dict, + self.url, + no_archive=True, + follow_redirect=False, + sanitize_info=True, + ) + + if not ie_info or not isinstance(ie_info, dict): + return ({}, False, "Failed to extract information from URL.") + + return (ie_info, True, "") + + async def _mark_logic(self) -> tuple[bool, str] | dict[str, Any]: + """ + Internal logic for marking/un-marking items. + + Returns: + tuple[bool, str] | dict[str, Any]: Either an error tuple or a dict with 'file' and 'items' keys. + + """ + from app.library.ytdlp import fetch_info + + if not self.url: + return (False, "No URL found in task parameters.") + + params: dict = self.get_ytdlp_opts().get_all() + if not (archive_file := params.get("download_archive")): + return (False, "No archive file found.") + + archive_file: Path = Path(archive_file) + + ie_info: dict | None = await fetch_info(params, self.url, no_archive=True, follow_redirect=True) + if not ie_info or not isinstance(ie_info, dict): + return (False, "Failed to extract information from URL.") + + if "playlist" != ie_info.get("_type"): + return (False, "Expected a playlist type from extract_info.") + + items: set[str] = set() + + def _process(item: dict): + for entry in item.get("entries", []): + if not isinstance(entry, dict): + continue + + if "playlist" == entry.get("_type"): + _process(entry) + continue + + if entry.get("_type") not in ("video", "url"): + continue + + if not entry.get("id") or not entry.get("ie_key"): + continue + + archive_id: str = f"{entry.get('ie_key', '').lower()} {entry.get('id')}" + + items.add(archive_id) + + _process(ie_info) + + return {"file": archive_file, "items": items} + + +@dataclass(slots=True) +class TaskItem: + """Represents a single item in a task result.""" + + url: str + "The URL of the item." + title: str | None = None + "The title of the item." + archive_id: str | None = None + "The archive ID of the item." + metadata: dict[str, Any] = field(default_factory=dict) + "Additional metadata related to the item." + + +@dataclass(slots=True) +class TaskResult: + """Represents a successful task handler execution result.""" + + items: list[TaskItem] = field(default_factory=list) + "The list of items." + metadata: dict[str, Any] = field(default_factory=dict) + "Additional metadata related to the result." + + def serialize(self) -> dict[str, Any]: + primary, extra = split_inspect_metadata(self.metadata) + payload: dict[str, Any] = {**primary, "items": [asdict(item) for item in self.items]} + + if extra: + payload["metadata"] = extra + + return payload + + +@dataclass(slots=True) +class TaskFailure: + """Represents a failed task handler execution result.""" + + message: str + "A human-readable message describing the failure." + error: str | None = None + "An optional error code or string." + metadata: dict[str, Any] = field(default_factory=dict) + "Additional metadata related to the failure." + + def serialize(self) -> dict[str, Any]: + primary, extra = split_inspect_metadata(self.metadata) + payload: dict[str, Any] = dict(primary) + + if self.error: + payload["error"] = self.error + + if self.message and (not self.error or self.message != self.error): + payload["message"] = self.message + + if extra: + payload["metadata"] = extra + + return payload diff --git a/app/features/tasks/definitions/router.py b/app/features/tasks/definitions/router.py new file mode 100644 index 00000000..56ee5054 --- /dev/null +++ b/app/features/tasks/definitions/router.py @@ -0,0 +1,235 @@ +import logging +from typing import Any + +from aiohttp import web +from aiohttp.web import Request, Response +from pydantic import ValidationError + +from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination +from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination +from app.features.tasks.definitions.repository import TaskDefinitionsRepository as Repo +from app.features.tasks.definitions.schemas import ( + TaskDefinition, + TaskDefinitionList, + TaskDefinitionPatch, +) +from app.features.tasks.definitions.utils import model_to_schema, schema_to_payload +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events +from app.library.router import route + +LOG: logging.Logger = logging.getLogger(__name__) + + +@route("GET", "api/tasks/definitions/", "task_definitions") +async def task_definitions_list(request: Request, encoder: Encoder, repo: Repo) -> Response: + page, per_page = normalize_pagination(request) + models, total, current_page, total_pages = await repo.list_paginated(page, per_page) + + include: str | None = request.query.get("include") + summary: bool = "definition" != include + + return web.json_response( + data=TaskDefinitionList( + items=[model_to_schema(model, summary=summary) for model in models], + pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)), + ), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("GET", r"api/tasks/definitions/{id:\d+}", "task_definitions_get") +async def task_definitions_get(request: Request, encoder: Encoder, repo: Repo) -> Response: + identifier: str = request.match_info.get("id", "").strip() + if not identifier: + return web.json_response( + data={"error": "Missing task definition identifier."}, + status=web.HTTPBadRequest.status_code, + ) + + if not (model := await repo.get(identifier)): + return web.json_response( + data={"error": f"Task definition '{identifier}' not found."}, + status=web.HTTPNotFound.status_code, + ) + + definition = model_to_schema(model) + return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("POST", "api/tasks/definitions/", "task_definitions_create") +async def task_definitions_create(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response: + try: + payload: Any = await request.json() + except Exception: + return web.json_response( + data={"error": "Invalid JSON in request body."}, + status=web.HTTPBadRequest.status_code, + ) + + if not isinstance(payload, dict): + return web.json_response( + data={"error": "Invalid request body; expected JSON object."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + definition_input = TaskDefinition.model_validate(payload) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task definition.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + try: + repo_payload = schema_to_payload(definition_input) + model = await repo.create(repo_payload) + definition = model_to_schema(model) + + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.CREATE, data=definition.model_dump()), + ) + return web.json_response(data=definition, status=web.HTTPCreated.status_code, dumps=encoder.encode) + except ValueError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code) + except Exception as exc: + LOG.exception(exc) + return web.json_response( + data={"error": "Failed to create task definition."}, + status=web.HTTPInternalServerError.status_code, + ) + + +@route("PUT", r"api/tasks/definitions/{id:\d+}", "task_definitions_update") +async def task_definitions_update(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response: + if not (identifier := request.match_info.get("id", "").strip()): + return web.json_response( + data={"error": "Missing task definition identifier."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + payload: dict | None = await request.json() + except Exception: + return web.json_response( + data={"error": "Invalid JSON in request body."}, + status=web.HTTPBadRequest.status_code, + ) + + if not isinstance(payload, dict): + return web.json_response( + data={"error": "Invalid request body; expected JSON object."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + definition_input: TaskDefinition = TaskDefinition.model_validate(payload) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task definition.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + try: + definition: TaskDefinition = model_to_schema(await repo.update(identifier, schema_to_payload(definition_input))) + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.UPDATE, data=definition.model_dump()), + ) + return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode) + except KeyError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code) + except ValueError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code) + except Exception as exc: + LOG.exception(exc) + return web.json_response( + data={"error": "Failed to update task definition."}, + status=web.HTTPInternalServerError.status_code, + ) + + +@route("PATCH", r"api/tasks/definitions/{id:\d+}", "task_definitions_patch") +async def task_definitions_patch(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response: + if not (identifier := request.match_info.get("id", "").strip()): + return web.json_response( + data={"error": "Missing task definition identifier."}, + status=web.HTTPBadRequest.status_code, + ) + + if not await repo.get(identifier): + return web.json_response( + data={"error": f"Task definition '{identifier}' not found."}, + status=web.HTTPNotFound.status_code, + ) + + try: + payload: dict | None = await request.json() + except Exception: + return web.json_response( + data={"error": "Invalid JSON in request body."}, + status=web.HTTPBadRequest.status_code, + ) + + if not isinstance(payload, dict): + return web.json_response( + data={"error": "Invalid request body; expected JSON object."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + patch_input: TaskDefinitionPatch = TaskDefinitionPatch.model_validate(payload) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task definition patch.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + try: + definition: TaskDefinition = model_to_schema( + await repo.update(identifier, patch_input.model_dump(exclude_unset=True)) + ) + + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.UPDATE, data=definition.model_dump()), + ) + return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode) + except KeyError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code) + except ValueError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code) + except Exception as exc: + LOG.exception(exc) + return web.json_response( + data={"error": "Failed to patch task definition."}, + status=web.HTTPInternalServerError.status_code, + ) + + +@route("DELETE", r"api/tasks/definitions/{id:\d+}", "task_definitions_delete") +async def task_definitions_delete(request: Request, encoder: Encoder, notify: EventBus, repo: Repo) -> Response: + if not (identifier := request.match_info.get("id", "").strip()): + return web.json_response( + data={"error": "Missing task definition identifier."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + definition: TaskDefinition = model_to_schema(await repo.delete(identifier)) + + notify.emit( + Events.CONFIG_UPDATE, + data=ConfigEvent(feature=CEFeature.TASKS_DEFINITIONS, action=CEAction.DELETE, data=definition.model_dump()), + ) + return web.json_response(data=definition, status=web.HTTPOk.status_code, dumps=encoder.encode) + except KeyError as exc: + return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code) + except Exception as exc: + LOG.exception(exc) + return web.json_response( + data={"error": "Failed to delete task definition."}, + status=web.HTTPInternalServerError.status_code, + ) diff --git a/app/features/tasks/definitions/schemas.py b/app/features/tasks/definitions/schemas.py new file mode 100644 index 00000000..19412e20 --- /dev/null +++ b/app/features/tasks/definitions/schemas.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import re +from datetime import datetime # noqa: TC003 +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from app.features.core.schemas import Pagination +from app.features.core.utils import parse_int + + +class PostFilter(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + filter: str = Field(min_length=1) + value: str | None = None + + @field_validator("filter") + @classmethod + def _validate_filter(cls, value: str) -> str: + try: + re.compile(value) + except re.error as exc: + msg: str = f"Invalid post_filter regex pattern: {exc}" + raise ValueError(msg) from exc + return value + + +class ExtractionRule(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + type: Literal["css", "xpath", "regex", "jsonpath"] + expression: str = Field(min_length=1) + attribute: str | None = None + post_filter: PostFilter | None = None + + def __getitem__(self, key: str) -> Any: + """Support bracket notation access.""" + if not hasattr(self, key): + raise KeyError(key) + return getattr(self, key) + + +class ParseItems(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + type: Literal["css", "xpath", "jsonpath"] = "css" + selector: str | None = Field(None, min_length=1) + expression: str | None = Field(None, min_length=1) + fields: dict[str, ExtractionRule] + + def get(self, key: str, default: Any = None) -> Any: + """Get a field value by key, supporting dict-like access.""" + return getattr(self, key, default) + + def __getitem__(self, key: str) -> Any: + """Support bracket notation access.""" + if not hasattr(self, key): + raise KeyError(key) + return getattr(self, key) + + @model_validator(mode="after") + def _validate_items(self) -> ParseItems: + if not self.selector and not self.expression: + msg = "Either 'selector' or 'expression' must be provided." + raise ValueError(msg) + if not self.selector: + self.selector = self.expression + if "link" not in self.fields: + msg = "Container 'fields' must include a 'link' field." + raise ValueError(msg) + return self + + +class Parse(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, extra="allow") + items: ParseItems | None = None + + def get(self, key: str, default: Any = None) -> Any: + """Get a field value by key, supporting dict-like access.""" + return getattr(self, key, default) + + def field_items(self) -> list[tuple[str, Any]]: + """Return field items like a dict, excluding private fields and 'items'.""" + data: dict[str, Any] = self.model_dump() + return [(k, v) for k, v in data.items() if k not in ("items",)] + + def __getitem__(self, key: str) -> Any: + """Support bracket notation access.""" + if not hasattr(self, key): + raise KeyError(key) + return getattr(self, key) + + def __contains__(self, key: str) -> bool: + """Support 'in' operator.""" + return hasattr(self, key) and not key.startswith("_") + + @model_validator(mode="before") + @classmethod + def _validate_parse(cls, value: Any) -> Any: + """Validate that we have either items or direct parsers with link.""" + if not isinstance(value, dict): + msg: str = "Parse must be a dict" + raise ValueError(msg) + + has_items: bool = "items" in value and value["items"] is not None + direct_parsers: dict[str, Any] = { + k: v for k, v in value.items() if k not in ("items",) and not k.startswith("_") + } + has_direct_parsers: bool = len(direct_parsers) > 0 + has_link_parser: bool = "link" in direct_parsers + + if not has_items and not has_direct_parsers: + msg: str = "Field 'parse' must contain either 'items' or direct parsers." + raise ValueError(msg) + + if not has_items and not has_link_parser: + msg: str = "Missing required 'link' parser definition." + raise ValueError(msg) + + for field_name, field_value in direct_parsers.items(): + if not isinstance(field_value, dict): + msg: str = f"Parse field '{field_name}' must be an object." + raise ValueError(msg) + + return value + + +class EngineConfig(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + type: Literal["httpx", "selenium"] = "httpx" + options: dict[str, Any] = Field(default_factory=dict) + + +class RequestConfig(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True, protected_namespaces=()) + method: str = "GET" + headers: dict[str, str] = Field(default_factory=dict) + params: dict[str, Any] = Field(default_factory=dict) + data: dict[str, Any] | None = None + json_data: dict[str, Any] | None = None + timeout: float | None = None + url: str | None = None + + +class ResponseConfig(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + type: Literal["html", "json"] = "html" + + +class Definition(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + parse: Parse + engine: EngineConfig = Field(default_factory=EngineConfig) + request: RequestConfig = Field(default_factory=RequestConfig) + response: ResponseConfig = Field(default_factory=ResponseConfig) + + +class TaskDefinitionSummary(BaseModel): + model_config = ConfigDict(from_attributes=True, str_strip_whitespace=True) + id: int | None = None + name: str = Field(min_length=1) + priority: int = Field(default=0, ge=0) + match_url: list[str] = Field(min_length=1) + enabled: bool = Field(default=True) + created_at: datetime | None = None + updated_at: datetime | None = None + + +class TaskDefinition(TaskDefinitionSummary): + definition: Definition + + @field_validator("priority", mode="before") + @classmethod + def _normalize_priority(cls, value: Any) -> int: + if value is None: + return 0 + return parse_int(value, field="Priority", minimum=0) + + @field_validator("match_url", mode="before") + @classmethod + def _validate_match_url(cls, value: Any) -> list[str]: + """Validate that match_url is a list of strings and validate regex patterns.""" + if not isinstance(value, list): + msg = "match_url must be a list" + raise ValueError(msg) + + validated: list[str] = [] + for item in value: + if not isinstance(item, str): + msg: str = f"match_url items must be strings, got {type(item).__name__}" + raise ValueError(msg) + + item: str = item.strip() + if not item: + msg = "match_url items cannot be empty" + raise ValueError(msg) + + if item.startswith("/") and item.endswith("/") and len(item) > 2: + pattern = item[1:-1] + try: + re.compile(pattern) + except re.error as exc: + msg = f"Invalid regex pattern '{pattern}': {exc}" + raise ValueError(msg) from exc + + validated.append(item) + + return validated + + +class TaskDefinitionPatch(TaskDefinition): + model_config = ConfigDict(str_strip_whitespace=True) + name: str | None = None + priority: int | None = None + match_url: list[str] | None = None + definition: Definition | None = None + enabled: bool | None = None + + +class TaskDefinitionList(BaseModel): + model_config = ConfigDict(from_attributes=True) + items: list[TaskDefinitionSummary | TaskDefinition] = Field(default_factory=list) + pagination: Pagination diff --git a/app/features/tasks/definitions/service.py b/app/features/tasks/definitions/service.py new file mode 100644 index 00000000..f7702feb --- /dev/null +++ b/app/features/tasks/definitions/service.py @@ -0,0 +1,456 @@ +from __future__ import annotations + +import asyncio +import importlib +import inspect +import logging +import pkgutil +import random +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any + +from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult +from app.features.tasks.models import TaskModel +from app.library.downloads.queue_manager import DownloadQueue +from app.library.Events import EventBus, Events +from app.library.ItemDTO import Item, ItemDTO +from app.library.Services import Services +from app.library.Utils import archive_read + +if TYPE_CHECKING: + from app.features.tasks.repository import TasksRepository + from app.library.config import Config + from app.library.Scheduler import Scheduler + +LOG: logging.Logger = logging.getLogger("tasks.definitions.service") + + +class TaskHandle: + def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None: + self._handlers: list[type] = [] + "The available handlers." + self._repo: TasksRepository = tasks + "The tasks manager." + self._scheduler: Scheduler = scheduler + "The scheduler." + self._config: Config = config + "The configuration." + self._task_name: str = f"{__class__.__name__}._dispatcher" + "The task name for the scheduler." + self._queued: dict[str, set[str]] = {} + "Queued archive IDs per handler." + self._failure_count: dict[str, dict[str, int]] = {} + "Failure counts per handler and archive ID." + + EventBus.get_instance().subscribe( + Events.ITEM_ERROR, + self._handle_item_error, + f"{__class__.__name__}.item_error", + ) + + def load(self) -> None: + self._handlers: list[type] = self._discover() + + timer: str = self._config.tasks_handler_timer + try: + from cronsim import CronSim + + CronSim(timer, datetime.now(UTC)) + except Exception as e: + timer = "15 */1 * * *" + LOG.error(f"Invalid timer format. '{e!s}'. Defaulting to '{timer}'.") + + self._scheduler.add( + timer=timer, + func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"), + id=f"{__class__.__name__}._dispatcher", + ) + + async def _dispatcher(self): + s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []} + + handler_groups: dict[str, list[tuple[HandleTask, type]]] = {} + + tasks: list[TaskModel] = await self._repo.list() + + for task_model in tasks: + task: HandleTask = HandleTask.model_validate(task_model) + + if not task.enabled or not task.handler_enabled: + s["d"].append(task.name) + continue + + if not task.get_ytdlp_opts().get_all().get("download_archive"): + LOG.debug(f"Task '{task.name}' does not have an archive file configured.") + s["f"].append(task.name) + continue + + try: + handler: type | None = await self._find_handler(task) + if handler is None: + s["u"].append(task.name) + continue + + handler_name: str = handler.__name__ + if handler_name not in handler_groups: + handler_groups[handler_name] = [] + handler_groups[handler_name].append((task, handler)) + s["h"].append(task.name) + except Exception as e: + LOG.error(f"Failed to handle task '{task.name}'. '{e!s}'.") + s["f"].append(task.name) + + for tasks_with_handlers in handler_groups.values(): + for idx, (task, handler) in enumerate(tasks_with_handlers): + try: + t: asyncio.Task[TaskResult | TaskFailure | None] = asyncio.create_task( + coro=self._dispatch( + task, + handler, + delay=0.0 if 0 == idx else random.uniform(1.0, self._config.task_handler_random_delay), + ), + name=f"taskHandler-{task.id}", + ) + t.add_done_callback(lambda fut, t=task: self._handle_exception(fut, t)) + except Exception as e: + LOG.error(f"Failed to dispatch task '{task.name}'. '{e!s}'.") + + if len(tasks) > 0: + LOG.info( + f"Tasks handler summary: Handled: {len(s['h'])}, Unhandled: {len(s['u'])}, Disabled: {len(s['d'])}, Failed: {len(s['f'])}." + ) + + async def _dispatch(self, task: HandleTask, handler: type, delay: float) -> TaskResult | TaskFailure | None: + """ + Dispatch a task after a random delay to avoid rate limiting. + + Args: + task: The task to dispatch. + handler: The handler to use. + delay: The delay in seconds before dispatching. + + Returns: + The dispatch result. + + """ + if delay > 0: + LOG.debug(f"Delaying dispatch of task '{task.name}' by {delay:.1f} seconds.") + await asyncio.sleep(delay) + return await self.dispatch(task, handler=handler) + + def _handle_exception(self, fut: asyncio.Task, task: HandleTask) -> None: + if fut.cancelled(): + return + + if exc := fut.exception(): + LOG.error(f"Exception while handling task '{task.name}': {exc}") + + async def _find_handler(self, task: HandleTask) -> type | None: + for cls in self._handlers: + try: + if await Services.get_instance().handle_async(handler=cls.can_handle, task=task): + return cls + except Exception as e: + LOG.exception(e) + continue + + return None + + async def dispatch( + self, + task: HandleTask, + handler: type | None = None, + **kwargs, # noqa: ARG002 + ) -> TaskResult | TaskFailure | None: + """ + Dispatch a task to the appropriate handler. + + Args: + task: The task to dispatch. + handler: Optional specific handler to use instead of finding one. + **kwargs: Additional context to pass to the handler. + + Returns: + The extraction outcome, or None if no handler matched. + + """ + if not handler: + handler = await self._find_handler(task) + if handler is None: + return None + + services: Services = Services.get_instance() + + try: + extraction: TaskResult | TaskFailure = await services.handle_async( + handler=handler.extract, task=task, config=self._config + ) + except NotImplementedError: + LOG.error(f"Handler '{handler.__name__}' does not implement extract().") + return TaskFailure(message="Handler does not support extraction.") + except Exception as exc: + LOG.exception(exc) + raise + + if isinstance(extraction, TaskFailure): + LOG.error(f"Handler '{handler.__name__}' failed to extract items: {extraction.message}") + return extraction + + if not isinstance(extraction, TaskResult): + LOG.error( + f"Handler '{handler.__name__}' returned unexpected result type '{type(extraction).__name__}'.", + ) + return TaskFailure( + message="Handler returned invalid result type.", metadata={"type": type(extraction).__name__} + ) + + raw_items: list[TaskItem] = extraction.items or [] + metadata: dict[str, Any] = extraction.metadata or {} + + handler_name: str = handler.__name__ + queued: set[str] = self._queued.setdefault(handler_name, set()) + failures: dict[str, int] = self._failure_count.setdefault(handler_name, {}) + + params: dict = task.get_ytdlp_opts().get_all() + archive_file: str | None = params.get("download_archive") + + download_queue: DownloadQueue = services.get("queue") or DownloadQueue.get_instance() + notify: EventBus = services.get("notify") or EventBus.get_instance() + + archive_ids: list[str] = [ + item.archive_id for item in raw_items if isinstance(item, TaskItem) and item.archive_id + ] + downloaded: list[str] = archive_read(archive_file, archive_ids) if archive_file else [] + + filtered: list[TaskItem] = [] + + for item in raw_items: + if not isinstance(item, TaskItem): + LOG.warning("Handler '{handler.__name__}' produced non-TaskItem entry: {item!r}") + continue + + url: str = item.url + if not url: + continue + + archive_id: str | None = item.archive_id + if not archive_id: + LOG.warning(f"'{task.name}': Item with URL '{url}' is missing an archive ID. Skipping.") + continue + + if archive_id in queued: + continue + + queued.add(archive_id) + + if archive_file and archive_id in downloaded: + continue + + if await download_queue.queue.exists(url=url): + continue + + try: + done = await download_queue.done.get(url=url) + if "error" != done.info.status: + continue + except KeyError: + pass + + if archive_id not in failures: + failures[archive_id] = 0 + + filtered.append(item) + + if not filtered: + if raw_items: + LOG.debug( + f"Handler '{handler.__name__}' produced '{len(raw_items)}' for '{task.name}' items, none queued after filtering." + ) + return TaskResult(items=[], metadata=metadata) + + LOG.info( + f"Handler '{handler.__name__}' Found '{len(filtered)}' new items for '{task.name}' (raw={len(raw_items)})." + ) + + base_item = Item.format( + { + "url": task.url, + "preset": task.preset or self._config.default_preset, + "folder": task.folder or "", + "template": task.template or "", + "cli": task.cli or "", + "auto_start": task.auto_start, + "extras": {"source_name": task.name, "source_id": task.id, "source_handler": handler.__name__}, + } + ) + + for item in filtered: + metadata_entry: dict[str, Any] = item.metadata if isinstance(item.metadata, dict) else {} + extras: dict[str, Any] = base_item.extras.copy() + if metadata_entry: + extras["metadata"] = metadata_entry + + notify.emit( + Events.ADD_URL, + data=base_item.new_with(url=item.url, extras=extras).serialize(), + ) + + return TaskResult(items=filtered, metadata=metadata) + + async def inspect( + self, + url: str, + preset: str | None = None, + handler_name: str | None = None, + static_only: bool = False, + ) -> TaskResult | TaskFailure: + """ + Inspect a URL to find a matching handler and optionally extract items. + + Args: + url: The URL to inspect. + preset: Optional preset name to use. + handler_name: Optional specific handler name to use. + static_only: If True, only check if a handler matches without extraction. + + Returns: + TaskResult or TaskFailure with inspection results. + + """ + if not self._handlers: + self._handlers = self._discover() + + task = HandleTask( + id=None, + name="Inspector", + url=url, + preset=preset or self._config.default_preset, + auto_start=False, + ) + + services = Services.get_instance() + + handler_cls: type | None + if handler_name: + handler_cls = next((cls for cls in self._handlers if cls.__name__.lower() == handler_name.lower()), None) + if handler_cls is None: + message: str = f"Handler '{handler_name}' not found." + return TaskFailure( + message=message, + error=message, + metadata={"matched": False, "handler": handler_name}, + ) + + try: + matched = await services.handle_async(handler=handler_cls.can_handle, task=task) + except Exception as exc: # pragma: no cover - defensive + LOG.exception(exc) + message = str(exc) + return TaskFailure( + message=message, + error=message, + metadata={"matched": False, "handler": handler_cls.__name__}, + ) + + if not matched: + return TaskFailure( + message="Handler cannot process the supplied URL.", + metadata={"matched": False, "handler": handler_cls.__name__}, + ) + else: + handler_cls = await self._find_handler(task) + if handler_cls is None: + message = "No handler matched the supplied URL." + return TaskFailure( + message=message, + error=message, + metadata={"matched": False, "handler": None}, + ) + + base_metadata: dict[str, Any] = {"matched": True, "handler": handler_cls.__name__} + + if static_only: + return TaskResult(items=[], metadata=base_metadata) + + try: + extraction: TaskResult | TaskFailure = await services.handle_async( + handler=handler_cls.extract, task=task, config=self._config + ) + except NotImplementedError: + return TaskFailure( + message="Handler does not support manual inspection.", + metadata={**base_metadata, "supported": False}, + ) + except Exception as exc: + LOG.exception(exc) + message = str(exc) + return TaskFailure( + message=message, + error=message, + metadata={**base_metadata, "supported": True}, + ) + + if isinstance(extraction, TaskFailure): + combined_failure_metadata: dict[str, Any] = {**base_metadata, "supported": True} + if extraction.metadata: + combined_failure_metadata.update(extraction.metadata) + + return TaskFailure( + message=extraction.message, + error=extraction.error if extraction.error else extraction.message, + metadata=combined_failure_metadata, + ) + + if not isinstance(extraction, TaskResult): + LOG.error( + f"Handler '{handler_cls.__name__}' returned unexpected result type '{type(extraction).__name__}' during inspection.", + ) + extraction = TaskResult() + + combined_metadata: dict[str, Any] = {**base_metadata, "supported": True} + if extraction.metadata: + combined_metadata.update(extraction.metadata) + + return TaskResult(items=list(extraction.items), metadata=combined_metadata) + + def _discover(self) -> list[type]: + """Discover all available task handlers.""" + import app.features.tasks.definitions.handlers as handlers_pkg + + handlers: list[type] = [] + + for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__): + if module_name.startswith("_"): + continue + + module = importlib.import_module(f"{handlers_pkg.__name__}.{module_name}") + for _, cls in inspect.getmembers(module, inspect.isclass): + if cls.__module__ != module.__name__: + continue + + if callable(getattr(cls, "can_handle", None)) and callable(getattr(cls, "extract", None)): + handlers.append(cls) + + return handlers + + async def _handle_item_error(self, event, _name, **_kwargs): + """Handle item error events to clean up queued items and track failures.""" + item: ItemDTO | None = getattr(event, "data", None) + if not isinstance(item, ItemDTO): + return + + extras: dict[Any, Any] = getattr(item, "extras", {}) or {} + handler_name: Any | None = extras.get("source_handler") + if not handler_name: + return + + archive_id: str | None = item.archive_id + if not archive_id: + return + + queued: set[str] | None = self._queued.get(handler_name) + if queued: + queued.discard(archive_id) + + failures: dict[str, int] = self._failure_count.setdefault(handler_name, {}) + failures[archive_id] = failures.get(archive_id, 0) + 1 diff --git a/app/features/tasks/definitions/tests/test_generic_task_handler.py b/app/features/tasks/definitions/tests/test_generic_task_handler.py new file mode 100644 index 00000000..b55f78e2 --- /dev/null +++ b/app/features/tasks/definitions/tests/test_generic_task_handler.py @@ -0,0 +1,396 @@ +from datetime import datetime +from unittest.mock import patch + +import pytest + +from app.features.tasks.definitions.handlers.generic import GenericTaskHandler +from app.features.tasks.definitions.results import TaskFailure, TaskResult +from app.features.tasks.definitions.schemas import ( + Definition, + EngineConfig, + RequestConfig, + ResponseConfig, + TaskDefinition, +) +from app.features.tasks.definitions.results import HandleTask + + +@pytest.fixture(autouse=True) +def reset_generic_handler(monkeypatch): + monkeypatch.setattr(GenericTaskHandler, "_definitions", []) + monkeypatch.setattr(GenericTaskHandler, "_sources_mtime", {}) + + +def test_build_task_definition_parses_valid_payload(): + definition = TaskDefinition( + id=1, + name="example", + priority=0, + match_url=["https://example.com/articles/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "link": {"type": "css", "expression": ".article a.link::attr(href)"}, + "title": {"type": "css", "expression": ".article .title", "attribute": "text"}, + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(), + ), + ) + + assert definition is not None, "TaskDefinition should be created" + assert "example" == definition.name, "Name should match" + assert "https://example.com/articles/*" in definition.match_url, "Match URL should be in list" + assert "link" in definition.definition.parse, "Parse should contain link field" + assert ".article a.link::attr(href)" == definition.definition.parse["link"]["expression"], ( + "Link expression should match" + ) + + +def test_build_task_definition_handles_container(): + definition = TaskDefinition( + id=2, + name="container", + priority=0, + match_url=["https://example.com/cards"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "selector": ".cards .card", + "fields": { + "link": {"type": "css", "expression": ".card-header a", "attribute": "href"}, + "title": {"type": "css", "expression": ".card-header a", "attribute": "text"}, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(), + ), + ) + + assert definition is not None, "TaskDefinition should be created" + assert "items" in definition.definition.parse, "Parse should contain items container" + assert ".cards .card" == definition.definition.parse["items"]["selector"], "Items selector should match" + assert "link" in definition.definition.parse["items"]["fields"], "Items fields should contain link" + + +def test_build_task_definition_handles_json(): + definition = TaskDefinition( + id=3, + name="json-def", + priority=0, + match_url=["https://example.com/api"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "type": "jsonpath", + "selector": "items", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(type="json"), + ), + ) + + assert definition is not None, "TaskDefinition should be created" + assert "json" == definition.definition.response.type, "Response type should be json" + assert "items" in definition.definition.parse, "Parse should contain items container" + assert "jsonpath" == definition.definition.parse["items"]["type"], "Items type should be jsonpath" + assert "jsonpath" == definition.definition.parse["items"]["fields"]["link"]["type"], ( + "Link field type should be jsonpath" + ) + + +def test_parse_items_extracts_values(): + definition = TaskDefinition( + id=4, + name="example", + priority=0, + match_url=["https://example.com/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None}, + "title": {"type": "css", "expression": ".article .title", "attribute": "text"}, + "id": {"type": "css", "expression": ".article", "attribute": "data-id"}, + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(), + ), + ) + + html = """ +
+ First + First Title +
+
+ Second + Second Title +
+ """ + + items = GenericTaskHandler._parse_items(definition, html, "https://example.com/base/") + + assert 2 == len(items), "Should extract 2 items" + assert "https://example.com/article-101" == items[0]["link"], "First item link should be absolute URL" + assert "First Title" == items[0]["title"], "First item title should match" + assert "101" == items[0]["id"], "First item id should match" + assert "https://example.com/article-102" == items[1]["link"], "Second item link should match" + + +def test_parse_items_handles_nested_card_layout(): + definition = TaskDefinition( + id=5, + name="nested", + priority=0, + match_url=["https://example.com/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "type": "css", + "selector": ".columns .card", + "fields": { + "link": { + "type": "css", + "expression": ".card-header a[href]", + "attribute": "href", + }, + "title": { + "type": "css", + "expression": ".card-header a[href]", + "attribute": "text", + }, + "poet": { + "type": "css", + "expression": "footer .card-footer-item:first-child a", + "attribute": "text", + }, + "category": { + "type": "css", + "expression": "footer .card-footer-item:nth-child(2) a", + "attribute": "text", + }, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(), + ), + ) + + html = """ +
+
+
+
+

+ First Poem +

+
+ +
+
+
+
+
+

+ Second Poem +

+
+ +
+
+
+ """ + + items = GenericTaskHandler._parse_items(definition, html, "https://example.com") + + assert 2 == len(items), "Should extract 2 items" + assert "https://example.com/poems/view/111" == items[0]["link"], "First item link should match" + assert "First Poem" == items[0]["title"], "First item title should match" + assert "Poet Alpha" == items[0]["poet"], "First item poet should match" + assert "Category One" == items[0]["category"], "First item category should match" + + assert "https://example.com/poems/view/222" == items[1]["link"], "Second item link should match" + assert "Second Poem" == items[1]["title"], "Second item title should match" + assert "Poet Beta" == items[1]["poet"], "Second item poet should match" + assert "category" not in items[1], "Second item should not have category" + + +def test_parse_items_handles_json_container(): + definition = TaskDefinition( + id=6, + name="json", + priority=0, + match_url=["https://example.com/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "type": "jsonpath", + "selector": "entries", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + "id": {"type": "jsonpath", "expression": "id"}, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(type="json"), + ), + ) + + payload = { + "entries": [ + {"url": "/video/1", "title": "First", "id": 1}, + {"url": "https://example.com/video/2", "title": "Second", "id": 2}, + {"title": "Missing Link", "id": 3}, + ] + } + + items = GenericTaskHandler._parse_items( + definition=definition, + html="", + base_url="https://example.com", + json_data=payload, + ) + + assert 2 == len(items), "Should extract 2 items (third missing link)" + assert "https://example.com/video/1" == items[0]["link"], "First item link should be absolute" + assert "First" == items[0]["title"], "First item title should match" + assert "1" == items[0]["id"], "First item id should match" + + assert "https://example.com/video/2" == items[1]["link"], "Second item link should match" + assert "Second" == items[1]["title"], "Second item title should match" + assert "2" == items[1]["id"], "Second item id should match" + + +@pytest.mark.asyncio +async def test_generic_task_handler_inspect(monkeypatch): + definition = TaskDefinition( + id=7, + name="json-inspect", + priority=0, + match_url=["https://example.com/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "type": "jsonpath", + "selector": "items", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(type="json"), + ), + ) + + async def fake_find_definition(cls, url): # noqa: ARG001 + return definition + + monkeypatch.setattr( + GenericTaskHandler, + "_find_definition", + classmethod(fake_find_definition), + ) + + async def fake_fetch_content(url, definition, ytdlp_opts): # noqa: ARG001 + return "", {"items": [{"url": "/video/1", "title": "First"}]} + + monkeypatch.setattr(GenericTaskHandler, "_fetch_content", staticmethod(fake_fetch_content)) + + # Mock fetch_info to return valid info with required fields for archive ID generation + async def fake_fetch_info(config, url, **kwargs): # noqa: ARG001 + return {"id": "test_video_1", "extractor_key": "Example"} + + with patch("app.features.tasks.definitions.handlers.generic.fetch_info", side_effect=fake_fetch_info): + task = HandleTask(id=1, name="Inspect", url="https://example.com/api") + result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task) + + assert isinstance(result, TaskResult), "Result should be TaskResult" + assert 1 == len(result.items), "Should have 1 item" + item = result.items[0] + assert "https://example.com/video/1" == item.url, "Item URL should match" + assert "First" == item.title, "Item title should match" + + +def test_parse_items_handles_json_top_level_list(): + definition = TaskDefinition( + id=8, + name="json-list", + priority=0, + match_url=["https://example.com/*"], + created_at=datetime.now(), + updated_at=datetime.now(), + definition=Definition( + parse={ + "items": { + "type": "jsonpath", + "selector": "[]", + "fields": { + "link": {"type": "jsonpath", "expression": "url"}, + "title": {"type": "jsonpath", "expression": "title"}, + }, + } + }, + engine=EngineConfig(), + request=RequestConfig(), + response=ResponseConfig(type="json"), + ), + ) + + payload = [ + {"url": "/video/1", "title": "First"}, + {"url": "/video/2", "title": "Second"}, + ] + + items = GenericTaskHandler._parse_items( + definition=definition, + html="", + base_url="https://example.com", + json_data=payload, + ) + + assert 2 == len(items), "Should extract 2 items" + assert "https://example.com/video/1" == items[0]["link"], "First item link should match" + assert "First" == items[0]["title"], "First item title should match" + assert "https://example.com/video/2" == items[1]["link"], "Second item link should match" + assert "Second" == items[1]["title"], "Second item title should match" diff --git a/app/tests/test_rss_handler.py b/app/features/tasks/definitions/tests/test_rss_handler.py similarity index 82% rename from app/tests/test_rss_handler.py rename to app/features/tasks/definitions/tests/test_rss_handler.py index b8a5cb0a..04333094 100644 --- a/app/tests/test_rss_handler.py +++ b/app/features/tasks/definitions/tests/test_rss_handler.py @@ -1,7 +1,8 @@ import pytest -from app.library.task_handlers.rss import RssGenericHandler -from app.library.Tasks import Task, TaskResult +from app.features.tasks.definitions.handlers.rss import RssGenericHandler +from app.features.tasks.definitions.results import TaskResult +from app.features.tasks.definitions.results import HandleTask class DummyResponse: @@ -68,10 +69,10 @@ class TestRssHandlerExtraction: return DummyResponse(atom_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_rss", + task = HandleTask( + id=1, name="Test Atom Feed", url="https://example.com/feed.atom", preset="default", @@ -112,10 +113,10 @@ class TestRssHandlerExtraction: return DummyResponse(rss_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_rss", + task = HandleTask( + id=1, name="Test RSS Feed", url="https://example.com/feed.rss", preset="default", @@ -146,25 +147,25 @@ class TestRssHandlerExtraction: return DummyResponse(atom_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_rss", - name="Test Feed", + task = HandleTask( + id=1, + name="Test rss Feed", url="https://example.com/feed.atom", preset="default", ) - assert RssGenericHandler.can_handle(task) is True + assert await RssGenericHandler.can_handle(task) is True - non_feed_task = Task( - id="test_youtube", + non_feed_task = HandleTask( + id=1, name="YouTube Video", url="https://www.youtube.com/watch?v=abc123", preset="default", ) - assert RssGenericHandler.can_handle(non_feed_task) is False + assert await RssGenericHandler.can_handle(non_feed_task) is False class TestRssHandlerEdgeCases: @@ -185,10 +186,10 @@ class TestRssHandlerEdgeCases: return DummyResponse(empty_feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_empty", + task = HandleTask( + id=1, name="Empty Feed", url="https://example.com/feed.rss", preset="default", @@ -203,17 +204,17 @@ class TestRssHandlerEdgeCases: @pytest.mark.asyncio async def test_invalid_feed_url(self, monkeypatch): """Test handling of invalid feed URL.""" - from app.library.Tasks import TaskFailure + from app.features.tasks.definitions.results import TaskFailure async def fake_request(**kwargs): # noqa: ARG001 msg = "Network error" raise Exception(msg) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_invalid", + task = HandleTask( + id=1, name="Invalid Feed", url="https://example.com/feed.rss", preset="default", @@ -244,10 +245,10 @@ class TestRssHandlerEdgeCases: return DummyResponse(feed) monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="test_missing", + task = HandleTask( + id=1, name="Feed with Missing URLs", url="https://example.com/feed.rss", preset="default", diff --git a/app/features/tasks/definitions/tests/test_task_definitions.py b/app/features/tasks/definitions/tests/test_task_definitions.py new file mode 100644 index 00000000..388bd3b4 --- /dev/null +++ b/app/features/tasks/definitions/tests/test_task_definitions.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import json +from collections.abc import AsyncGenerator +from unittest.mock import AsyncMock, MagicMock + +import pytest +import pytest_asyncio +from aiohttp import web +from aiohttp.web import Request + +from app.features.tasks.definitions.repository import TaskDefinitionsRepository +from app.features.tasks.definitions.router import ( + task_definitions_create, + task_definitions_delete, + task_definitions_get, + task_definitions_list, + task_definitions_patch, + task_definitions_update, +) +from app.library.encoder import Encoder +from app.library.sqlite_store import SqliteStore +from app.main import EventBus + + +def _sample_definition(name: str = "example", *, priority: int = 0) -> dict: + """Returns a properly structured task definition payload for the repository.""" + return { + "name": name, + "match_url": ["https://example.com/*"], + "priority": priority, + "definition": { + "parse": { + "link": { + "type": "css", + "expression": "a", + "attribute": "href", + } + }, + "engine": {"type": "httpx"}, + "request": {}, + "response": {"type": "html"}, + }, + } + + +@pytest_asyncio.fixture +async def repo() -> AsyncGenerator[TaskDefinitionsRepository, None]: + TaskDefinitionsRepository._reset_singleton() + SqliteStore._reset_singleton() + + store = SqliteStore(db_path=":memory:") + await store.get_connection() + + repository = TaskDefinitionsRepository.get_instance() + + yield repository + + if store._conn: + await store._conn.close() + if store._engine: + await store._engine.dispose() + + TaskDefinitionsRepository._reset_singleton() + SqliteStore._reset_singleton() + + +class TestTaskDefinitionsRepository: + @pytest.mark.asyncio + async def test_create_and_list(self, repo: TaskDefinitionsRepository) -> None: + await repo.create(_sample_definition("Alpha", priority=2)) + await repo.create(_sample_definition("Beta", priority=1)) + + items = await repo.list() + assert len(items) == 2, "Should return two task definitions" + assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name" + + @pytest.mark.asyncio + async def test_create_duplicate_name_raises(self, repo: TaskDefinitionsRepository) -> None: + payload = _sample_definition("Dup") + await repo.create(payload) + + with pytest.raises(ValueError, match="already exists"): + await repo.create(payload) + + with pytest.raises(ValueError, match="already exists"): + await repo.create(payload) + + @pytest.mark.asyncio + async def test_update_missing_raises(self, repo: TaskDefinitionsRepository) -> None: + with pytest.raises(KeyError, match="not found"): + await repo.update(999, {"name": "Missing"}) + + @pytest.mark.asyncio + async def test_list_paginated(self, repo: TaskDefinitionsRepository) -> None: + for idx in range(5): + await repo.create(_sample_definition(f"Item {idx}", priority=idx)) + + items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2) + assert len(items) == 2, "Should return two items per page" + assert total == 5, "Should report total count of 5" + assert page == 1, "Should return requested page" + assert total_pages == 3, "Should compute total pages" + + +@pytest.mark.asyncio +class TestTaskDefinitionRoutes: + async def test_list_definitions(self, repo: TaskDefinitionsRepository) -> None: + await repo.create(_sample_definition("Sample")) + request = MagicMock(spec=Request) + request.query = {} + + response = await task_definitions_list(request, Encoder(), repo) + payload = json.loads(response.text) + + assert response.status == web.HTTPOk.status_code, "Should return 200 for list" + assert payload["items"][0]["name"] == "Sample", "Should include created definition" + + async def test_get_definition_not_found(self, repo: TaskDefinitionsRepository) -> None: + request = MagicMock(spec=Request) + request.match_info = {"id": "999"} + + response = await task_definitions_get(request, Encoder(), repo) + payload = json.loads(response.text) + + assert response.status == web.HTTPNotFound.status_code, "Should return 404 for missing definition" + assert "error" in payload, "Should include error payload" + + async def test_create_definition_success(self, repo: TaskDefinitionsRepository) -> None: + request = MagicMock(spec=Request) + request.json = AsyncMock(return_value=_sample_definition("New", priority=3)) + + response = await task_definitions_create(request, Encoder(), MagicMock(spec=EventBus), repo) + body = json.loads(response.text) + + assert response.status == web.HTTPCreated.status_code, "Should create task definition" + assert body["name"] == "New", "Should return created name" + assert body["priority"] == 3, "Should return created priority" + + async def test_update_definition_success(self, repo: TaskDefinitionsRepository) -> None: + created = await repo.create(_sample_definition("Original", priority=0)) + + request = MagicMock(spec=Request) + request.match_info = {"id": str(created.id)} + request.json = AsyncMock(return_value=_sample_definition("Updated", priority=4)) + + response = await task_definitions_update(request, Encoder(), MagicMock(spec=EventBus), repo) + body = json.loads(response.text) + + assert response.status == web.HTTPOk.status_code, "Should update task definition" + assert body["name"] == "Updated", "Should return updated name" + assert body["priority"] == 4, "Should return updated priority" + + async def test_delete_definition_success(self, repo: TaskDefinitionsRepository) -> None: + created = await repo.create(_sample_definition("Delete")) + + request = MagicMock(spec=Request) + request.match_info = {"id": str(created.id)} + + response = await task_definitions_delete(request, Encoder(), MagicMock(spec=EventBus), repo) + assert response.status == web.HTTPOk.status_code, "Should delete task definition" + + async def test_patch_definition_enabled(self, repo: TaskDefinitionsRepository) -> None: + created = await repo.create(_sample_definition("PatchTest", priority=5)) + assert created.enabled is True, "Should be enabled by default" + + request = MagicMock(spec=Request) + request.match_info = {"id": str(created.id)} + request.json = AsyncMock(return_value={"enabled": False}) + + response = await task_definitions_patch(request, Encoder(), MagicMock(spec=EventBus), repo) + body = json.loads(response.text) + + assert response.status == web.HTTPOk.status_code, "Should patch task definition" + assert body["name"] == "PatchTest", "Should keep original name" + assert body["priority"] == 5, "Should keep original priority" + assert body["enabled"] is False, "Should update enabled status" + + async def test_patch_definition_priority(self, repo: TaskDefinitionsRepository) -> None: + created = await repo.create(_sample_definition("PatchPriority", priority=1)) + + request = MagicMock(spec=Request) + request.match_info = {"id": str(created.id)} + request.json = AsyncMock(return_value={"priority": 10}) + + response = await task_definitions_patch(request, Encoder(), MagicMock(spec=EventBus), repo) + body = json.loads(response.text) + + assert response.status == web.HTTPOk.status_code, "Should patch task definition" + assert body["priority"] == 10, "Should update priority" + assert body["enabled"] is True, "Should keep original enabled status" + + async def test_patch_definition_not_found(self, repo: TaskDefinitionsRepository) -> None: + request = MagicMock(spec=Request) + request.match_info = {"id": "999"} + request.json = AsyncMock(return_value={"enabled": False}) + + response = await task_definitions_patch(request, Encoder(), MagicMock(spec=EventBus), repo) + payload = json.loads(response.text) + + assert response.status == web.HTTPNotFound.status_code, "Should return 404 for missing definition" + assert "error" in payload, "Should include error payload" + + async def test_create_with_regex_pattern(self, repo: TaskDefinitionsRepository) -> None: + payload = _sample_definition("RegexTest", priority=0) + payload["match_url"] = ["/https://example\\.com/post/[0-9]+/"] + + request = MagicMock(spec=Request) + request.json = AsyncMock(return_value=payload) + + response = await task_definitions_create(request, Encoder(), MagicMock(spec=EventBus), repo) + body = json.loads(response.text) + + assert response.status == web.HTTPCreated.status_code, "Should create task definition with regex pattern" + assert body["match_url"][0] == "/https://example\\.com/post/[0-9]+/", "Should preserve regex pattern format" + + async def test_create_with_invalid_regex_pattern(self, repo: TaskDefinitionsRepository) -> None: + payload = _sample_definition("BadRegex", priority=0) + payload["match_url"] = ["/[invalid(/"] + + request = MagicMock(spec=Request) + request.json = AsyncMock(return_value=payload) + + response = await task_definitions_create(request, Encoder(), MagicMock(spec=EventBus), repo) + payload_response = json.loads(response.text) + + assert response.status == web.HTTPBadRequest.status_code, "Should reject invalid regex pattern" + assert "error" in payload_response, "Should include error payload" diff --git a/app/tests/test_tver_handler.py b/app/features/tasks/definitions/tests/test_tver_handler.py similarity index 88% rename from app/tests/test_tver_handler.py rename to app/features/tasks/definitions/tests/test_tver_handler.py index b778e6f7..b2179c1a 100644 --- a/app/tests/test_tver_handler.py +++ b/app/features/tasks/definitions/tests/test_tver_handler.py @@ -1,7 +1,8 @@ import pytest -from app.library.task_handlers.tver import TverHandler -from app.library.Tasks import Task, TaskResult +from app.features.tasks.definitions.handlers.tver import TverHandler +from app.features.tasks.definitions.results import TaskResult +from app.features.tasks.definitions.results import HandleTask class DummyResponse: @@ -118,9 +119,9 @@ async def test_tver_handler_extract(monkeypatch): raise RuntimeError(msg) monkeypatch.setattr(TverHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": "/tmp/archive"})) + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": "/tmp/archive"})) - task = Task(id="test_tver", name="Test Tver Series", url="https://tver.jp/series/sr8sb9pnhc", preset="default") + task = HandleTask(id=1, name="Test Tver Series", url="https://tver.jp/series/sr8sb9pnhc", preset="default") result = await TverHandler.extract(task) @@ -151,9 +152,10 @@ def test_tver_handler_parse(url: str, should_match: bool): assert result is None -def test_tver_handler_can_handle(): +@pytest.mark.asyncio +async def test_tver_handler_can_handle(): """Test tver handler can_handle method.""" - task_valid = Task(id="test1", name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default") - task_invalid = Task(id="test2", name="Test", url="https://youtube.com/watch?v=123", preset="default") - assert TverHandler.can_handle(task_valid) is True - assert TverHandler.can_handle(task_invalid) is False + task_valid = HandleTask(id=1, name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default") + task_invalid = HandleTask(id=2, name="Test", url="https://youtube.com/watch?v=123", preset="default") + assert await TverHandler.can_handle(task_valid) is True + assert await TverHandler.can_handle(task_invalid) is False diff --git a/app/tests/test_twitch_handler.py b/app/features/tasks/definitions/tests/test_twitch_handler.py similarity index 77% rename from app/tests/test_twitch_handler.py rename to app/features/tasks/definitions/tests/test_twitch_handler.py index 821eb605..642c751c 100644 --- a/app/tests/test_twitch_handler.py +++ b/app/features/tasks/definitions/tests/test_twitch_handler.py @@ -1,7 +1,8 @@ import pytest -from app.library.task_handlers.twitch import TwitchHandler -from app.library.Tasks import Task, TaskResult +from app.features.tasks.definitions.handlers.twitch import TwitchHandler +from app.features.tasks.definitions.results import TaskResult +from app.features.tasks.definitions.results import HandleTask class DummyResponse: @@ -39,10 +40,10 @@ async def test_twitch_handler_inspect(monkeypatch): return DummyResponse(feed) monkeypatch.setattr(TwitchHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="inspect", + task = HandleTask( + id=1, name="Inspect", url="https://www.twitch.tv/testchannel", preset="default", diff --git a/app/tests/test_youtube_handler.py b/app/features/tasks/definitions/tests/test_youtube_handler.py similarity index 80% rename from app/tests/test_youtube_handler.py rename to app/features/tasks/definitions/tests/test_youtube_handler.py index 26ee9c91..3482eefa 100644 --- a/app/tests/test_youtube_handler.py +++ b/app/features/tasks/definitions/tests/test_youtube_handler.py @@ -1,7 +1,8 @@ import pytest -from app.library.task_handlers.youtube import YoutubeHandler -from app.library.Tasks import Task, TaskResult +from app.features.tasks.definitions.handlers.youtube import YoutubeHandler +from app.features.tasks.definitions.results import TaskResult +from app.features.tasks.definitions.results import HandleTask class DummyResponse: @@ -41,10 +42,10 @@ async def test_youtube_handler_inspect(monkeypatch): return DummyResponse(feed) monkeypatch.setattr(YoutubeHandler, "request", staticmethod(fake_request)) - monkeypatch.setattr(Task, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 + monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005 - task = Task( - id="inspect", + task = HandleTask( + id=1, name="Inspect", url="https://www.youtube.com/channel/UCabcdefghijklmnopqrstuv", preset="default", diff --git a/app/features/tasks/definitions/utils.py b/app/features/tasks/definitions/utils.py new file mode 100644 index 00000000..c1fba3d8 --- /dev/null +++ b/app/features/tasks/definitions/utils.py @@ -0,0 +1,69 @@ +from typing import Any + +from app.features.tasks.definitions.models import TaskDefinitionModel +from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary + + +def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary: + """ + Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema. + + Args: + model (TaskDefinitionModel): The model instance to convert. + summary (bool): Whether to return a summary schema. + + Returns: + TaskDefinition | TaskDefinitionSummary: The corresponding schema instance. + + """ + dct = { + "id": model.id, + "name": model.name, + "priority": model.priority, + "match_url": model.match_url, + "enabled": model.enabled, + "created_at": model.created_at, + "updated_at": model.updated_at, + } + return TaskDefinitionSummary(**dct) if summary else TaskDefinition(**dct, definition=Definition(**model.definition)) + + +def schema_to_payload(item: TaskDefinition) -> dict[str, Any]: + """ + Convert a TaskDefinition schema to a dictionary payload for database operations. + + Args: + item (TaskDefinition): The schema instance to convert. + + Returns: + dict[str, Any]: The corresponding dictionary payload. + + """ + return { + "name": item.name, + "priority": item.priority, + "match_url": item.match_url, + "enabled": item.enabled, + "definition": item.definition.model_dump(exclude_unset=True, exclude_none=True), + } + + +def split_inspect_metadata(metadata: dict[str, Any] | None) -> tuple[dict[str, Any], dict[str, Any]]: + """ + Split commonly consumed metadata keys from the rest. + + Args: + metadata (dict[str, Any]|None): The metadata to split. + + Returns: + tuple[dict[str, Any], dict[str, Any]]: The primary and extra metadata. + + """ + metadata = dict(metadata or {}) + primary: dict[str, Any] = {} + + for key in ("matched", "handler", "supported"): + if key in metadata: + primary[key] = metadata.pop(key) + + return primary, metadata diff --git a/app/features/tasks/deps.py b/app/features/tasks/deps.py new file mode 100644 index 00000000..eef934ba --- /dev/null +++ b/app/features/tasks/deps.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from app.features.tasks.repository import TasksRepository +from app.features.tasks.service import Tasks + + +def get_tasks_repo() -> TasksRepository: + """Get tasks repository instance.""" + return TasksRepository.get_instance() + + +def get_tasks_service() -> Tasks: + """Get tasks service instance.""" + return Tasks.get_instance() diff --git a/app/features/tasks/migration.py b/app/features/tasks/migration.py new file mode 100644 index 00000000..77e4928a --- /dev/null +++ b/app/features/tasks/migration.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from app.features.core.migration import Migration as FeatureMigration +from app.features.tasks.schemas import Task +from app.library.config import Config + +if TYPE_CHECKING: + from app.features.tasks.repository import TasksRepository + +LOG: logging.Logger = logging.getLogger(__name__) + + +class Migration(FeatureMigration): + name: str = "tasks" + + def __init__(self, repo: TasksRepository, config: Config | None = None): + self._config: Config = config or Config.get_instance() + super().__init__(config=self._config) + self._repo: TasksRepository = repo + self._source_file: Path = Path(self._config.config_path) / "tasks.json" + + async def should_run(self) -> bool: + return self._source_file.exists() + + async def migrate(self) -> None: + if await self._repo.count() > 0: + LOG.warning("Tasks already exist in the database; skipping migration.") + await self._move_file(self._source_file) + return + + try: + items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text()) + except Exception as exc: + LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc) + await self._move_file(self._source_file) + return + + if items is None: + LOG.warning("No tasks found in %s; skipping migration.", self._source_file) + await self._move_file(self._source_file) + return + + inserted = 0 + seen_names: dict[str, int] = {} + for index, item in enumerate(items): + if not (normalized := self._normalize(item, index, seen_names)): + continue + try: + await self._repo.create(normalized) + inserted += 1 + except Exception as exc: + LOG.exception("Failed to insert task '%s': %s", normalized["name"], exc) + + LOG.info("Migrated %s task(s) from %s.", inserted, self._source_file) + await self._move_file(self._source_file) + + def _normalize(self, item: Any, index: int, seen_names: dict[str, int]) -> dict[str, Any] | None: + if not isinstance(item, dict): + LOG.warning("Skipping task at index %s due to invalid type.", index) + return None + + assert isinstance(item, dict) + + name: str | None = item.get("name") + if not name or not isinstance(name, str): + LOG.warning("Skipping task at index %s due to missing name.", index) + return None + + normalized_name = name.strip() + if not normalized_name: + LOG.warning("Skipping task at index %s due to empty name.", index) + return None + + name = self._unique_name(normalized_name, seen_names) + + url: str | None = item.get("url") + if not url or not isinstance(url, str): + LOG.warning("Skipping task '%s' at index %s due to missing URL.", name, index) + return None + + url = url.strip() + if not url: + LOG.warning("Skipping task '%s' at index %s due to empty URL.", name, index) + return None + + folder: str = item.get("folder") if isinstance(item.get("folder"), str) else "" + preset: str = item.get("preset") if isinstance(item.get("preset"), str) else "" + timer: str = item.get("timer") if isinstance(item.get("timer"), str) else "" + template: str = item.get("template") if isinstance(item.get("template"), str) else "" + cli: str = item.get("cli") if isinstance(item.get("cli"), str) else "" + auto_start: bool = item.get("auto_start") if isinstance(item.get("auto_start"), bool) else True + handler_enabled: bool = item.get("handler_enabled") if isinstance(item.get("handler_enabled"), bool) else True + enabled: bool = item.get("enabled") if isinstance(item.get("enabled"), bool) else True + + try: + validated = Task( + name=name, + url=url, + folder=folder, + preset=preset, + timer=timer, + template=template, + cli=cli, + auto_start=auto_start, + handler_enabled=handler_enabled, + enabled=enabled, + ) + return validated.model_dump() + except Exception as e: + LOG.warning("Skipping task '%s' at index %s due to validation error: %s", name, index, e) + return None diff --git a/app/features/tasks/models.py b/app/features/tasks/models.py new file mode 100644 index 00000000..9993025d --- /dev/null +++ b/app/features/tasks/models.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from datetime import datetime # noqa: TC003 + +from sqlalchemy import Boolean, Index, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.features.core.models import Base, UTCDateTime, utcnow + + +class TaskModel(Base): + __tablename__: str = "tasks" + __table_args__: tuple[Index, ...] = ( + Index("ix_tasks_name", "name"), + Index("ix_tasks_enabled", "enabled"), + Index("ix_tasks_timer", "timer"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + url: Mapped[str] = mapped_column(String(2048), nullable=False) + folder: Mapped[str] = mapped_column(String(512), nullable=False, default="") + preset: Mapped[str] = mapped_column(String(255), nullable=False, default="") + timer: Mapped[str] = mapped_column(String(255), nullable=False, default="") + template: Mapped[str] = mapped_column(String(1024), nullable=False, default="") + cli: Mapped[str] = mapped_column(Text, nullable=False, default="") + auto_start: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + handler_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False) + updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False) diff --git a/app/features/tasks/repository.py b/app/features/tasks/repository.py new file mode 100644 index 00000000..4af8ede5 --- /dev/null +++ b/app/features/tasks/repository.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from sqlalchemy import func, or_, select + +from app.features.core.deps import get_session +from app.features.tasks.models import TaskModel +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from sqlalchemy.engine.result import Result + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + from sqlalchemy.sql.selectable import Select + +LOG: logging.Logger = logging.getLogger(__name__) + + +class TasksRepository(metaclass=Singleton): + def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None: + self._migrated = False + self.session = session or get_session + + async def run_migrations(self) -> None: + if self._migrated: + return + + self._migrated = True + from app.features.tasks.migration import Migration + + await Migration(repo=self).run() + + @staticmethod + def get_instance() -> TasksRepository: + return TasksRepository() + + async def list(self) -> list[TaskModel]: + async with self.session() as session: + result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc())) + return list(result.scalars().all()) + + async def list_paginated(self, page: int, per_page: int) -> tuple[list[TaskModel], int, int, int]: + async with self.session() as session: + total: int = await self.count() + total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1 + + if page > total_pages and total > 0: + page = total_pages + + query: Select[tuple[TaskModel]] = ( + select(TaskModel).order_by(TaskModel.name.asc()).limit(per_page).offset((page - 1) * per_page) + ) + result: Result[tuple[TaskModel]] = await session.execute(query) + return list(result.scalars().all()), total, page, total_pages + + async def count(self) -> int: + async with self.session() as session: + result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(TaskModel)) + return int(result.scalar_one()) + + async def get(self, identifier: int | str) -> TaskModel | None: + async with self.session() as session: + if not identifier: + return None + + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskModel.id == int(identifier), TaskModel.name == identifier) + else: + clause = TaskModel.name == identifier + + result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).where(clause).limit(1)) + return result.scalar_one_or_none() + + async def get_by_name(self, name: str, exclude_id: int | None = None) -> TaskModel | None: + async with self.session() as session: + query: Select[tuple[TaskModel]] = select(TaskModel).where(TaskModel.name == name) + if exclude_id is not None: + query = query.where(TaskModel.id != exclude_id) + + result: Result[tuple[TaskModel]] = await session.execute(query.limit(1)) + return result.scalar_one_or_none() + + async def get_all_enabled(self) -> list[TaskModel]: + """Get all enabled tasks.""" + async with self.session() as session: + result: Result[tuple[TaskModel]] = await session.execute( + select(TaskModel).where(TaskModel.enabled == True).order_by(TaskModel.name.asc()) # noqa: E712 + ) + return list(result.scalars().all()) + + async def get_all_with_timer(self) -> list[TaskModel]: + """Get all tasks that have a timer configured.""" + async with self.session() as session: + result: Result[tuple[TaskModel]] = await session.execute( + select(TaskModel).where(TaskModel.timer != "").order_by(TaskModel.name.asc()) + ) + return list(result.scalars().all()) + + async def create(self, payload: TaskModel | dict) -> TaskModel: + async with self.session() as session: + model: TaskModel = TaskModel(**payload) if isinstance(payload, dict) else payload + if model.id is not None: + model.id = None + + if await self.get_by_name(name=model.name) is not None: + msg: str = f"Task with name '{model.name}' already exists." + raise ValueError(msg) + + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def update(self, identifier: int | str, payload: dict[str, Any]) -> TaskModel: + """Update an existing task.""" + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskModel.id == int(identifier), TaskModel.name == identifier) + else: + clause = TaskModel.name == identifier + + result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).where(clause).limit(1)) + model: TaskModel | None = result.scalar_one_or_none() + + if model is None: + msg: str = f"Task '{identifier}' not found." + raise KeyError(msg) + + if "name" in payload: + existing: TaskModel | None = await self.get_by_name(name=payload["name"], exclude_id=model.id) + if existing is not None: + msg = f"Task with name '{payload['name']}' already exists." + raise ValueError(msg) + + for key, value in payload.items(): + if hasattr(model, key): + setattr(model, key, value) + + await session.commit() + await session.refresh(model) + return model + + async def delete(self, identifier: int | str) -> TaskModel: + async with self.session() as session: + if isinstance(identifier, int): + clause: ColumnElement[bool] = TaskModel.id == identifier + elif isinstance(identifier, str) and identifier.isdigit(): + clause = or_(TaskModel.id == int(identifier), TaskModel.name == identifier) + else: + clause = TaskModel.name == identifier + + result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).where(clause).limit(1)) + model: TaskModel | None = result.scalar_one_or_none() + + if model is None: + msg: str = f"Task '{identifier}' not found." + raise KeyError(msg) + + await session.delete(model) + await session.commit() + return model diff --git a/app/features/tasks/router.py b/app/features/tasks/router.py new file mode 100644 index 00000000..6c086ee5 --- /dev/null +++ b/app/features/tasks/router.py @@ -0,0 +1,480 @@ +import logging +from typing import TYPE_CHECKING, Any + +from aiohttp import web +from aiohttp.web import Request, Response +from pydantic import ValidationError + +from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination +from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination +from app.features.tasks.definitions.results import HandleTask as ExtendedTask +from app.features.tasks.definitions.results import TaskFailure, TaskResult +from app.features.tasks.definitions.service import TaskHandle +from app.features.tasks.repository import TasksRepository +from app.features.tasks.schemas import Task, TaskList, TaskPatch +from app.library.ag_utils import ag +from app.library.config import Config +from app.library.encoder import Encoder +from app.library.Events import EventBus, Events +from app.library.router import route +from app.library.Utils import get_channel_images, get_file, parse_outtmpl, validate_url + +if TYPE_CHECKING: + from pathlib import Path + + +LOG: logging.Logger = logging.getLogger(__name__) + + +def _model(model: Any) -> Task: + return Task.model_validate(model) + + +def _serialize(model: Any) -> dict: + return _model(model).model_dump() + + +@route("GET", "api/tasks/", "tasks_list") +async def tasks_list(request: Request, repo: TasksRepository, encoder: Encoder) -> Response: + page, per_page = normalize_pagination(request) + items, total, current_page, total_pages = await repo.list_paginated(page, per_page) + return web.json_response( + data=TaskList( + items=[_model(model) for model in items], + pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)), + ), + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", "api/tasks/", "tasks_add") +async def tasks_add(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response: + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + item: Task = Task.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if await repo.get_by_name(item.name): + return web.json_response( + data={"error": f"Task with name '{item.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + try: + created = await repo.create(item.model_dump()) + saved = _serialize(created) + except ValueError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code) + + notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.CREATE, data=saved)) + + return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("GET", r"api/tasks/{id:\d+}", "tasks_get") +async def tasks_get(request: Request, repo: TasksRepository, encoder: Encoder) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + model = await repo.get(identifier) + if not model: + return web.json_response({"error": "Task not found"}, status=web.HTTPNotFound.status_code) + + return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("DELETE", r"api/tasks/{id:\d+}", "tasks_delete") +async def tasks_delete(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + try: + deleted = _serialize(await repo.delete(identifier)) + notify.emit( + Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.DELETE, data=deleted) + ) + return web.json_response( + data=deleted, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + except KeyError as exc: + return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code) + + +@route("PATCH", r"api/tasks/{id:\d+}", "tasks_patch") +async def tasks_patch(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + model = await repo.get(identifier) + if not model: + return web.json_response({"error": "Task not found"}, status=web.HTTPNotFound.status_code) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + validated = TaskPatch.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if validated.name and await repo.get_by_name(validated.name, exclude_id=model.id): + return web.json_response( + data={"error": f"Task with name '{validated.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + updated = _serialize(await repo.update(model.id, validated.model_dump(exclude_unset=True))) + notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.UPDATE, data=updated)) + return web.json_response( + data=updated, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("PUT", r"api/tasks/{id:\d+}", "tasks_update") +async def tasks_update(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response: + if not (identifier := request.match_info.get("id")): + return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code) + + model = await repo.get(identifier) + if not model: + return web.json_response({"error": "Task not found"}, status=web.HTTPNotFound.status_code) + + data = await request.json() + + if not isinstance(data, dict): + return web.json_response( + {"error": "Invalid request body expecting dict."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + validated = Task.model_validate(data) + except ValidationError as exc: + return web.json_response( + data={"error": "Failed to validate task.", "detail": format_validation_errors(exc)}, + status=web.HTTPBadRequest.status_code, + ) + + if validated.name and await repo.get_by_name(validated.name, exclude_id=model.id): + return web.json_response( + data={"error": f"Task with name '{validated.name}' already exists."}, + status=web.HTTPConflict.status_code, + ) + + updated = _serialize(await repo.update(model.id, validated.model_dump(exclude_unset=True))) + notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.UPDATE, data=updated)) + + return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("POST", "api/tasks/inspect", "task_handler_inspect") +async def task_handler_inspect(request: Request, handler: TaskHandle, encoder: Encoder, config: Config) -> Response: + """ + Check if handler can process the given URL. + + Args: + request: The request object. + handler: The handler service instance. + encoder: The encoder instance. + config: The config instance. + + Returns: + The response object. + + """ + data = await request.json() + + url: str | None = data.get("url") if isinstance(data, dict) else None + if not url: + return web.json_response({"error": "url is required."}, status=web.HTTPBadRequest.status_code) + + static_only: bool = data.get("static_only", False) if isinstance(data, dict) else False + if not static_only: + try: + validate_url(url, allow_internal=config.allow_internal_urls) + except ValueError as e: + return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code) + + preset: str = data.get("preset", "") if isinstance(data, dict) else "" + handler_name: str | None = data.get("handler") if isinstance(data, dict) else None + + try: + result: TaskResult | TaskFailure = await handler.inspect( + url=url, preset=preset, handler_name=handler_name, static_only=static_only + ) + except Exception as e: + LOG.exception(e) + return web.json_response( + {"error": "Failed to inspect handler.", "message": str(e)}, + status=web.HTTPInternalServerError.status_code, + ) + + return web.json_response( + data=result, + status=web.HTTPBadRequest.status_code if isinstance(result, TaskFailure) else web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", r"api/tasks/{id:\d+}/mark", "tasks_mark") +async def task_mark(request: Request, repo: TasksRepository, encoder: Encoder) -> Response: + """ + Mark all items from task as downloaded. + + Args: + request: The request object. + repo: The tasks repository instance. + encoder: The encoder instance. + + Returns: + The response object. + + """ + if not (task_id := request.match_info.get("id")): + return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) + + try: + model = await repo.get(int(task_id)) + if not model: + return web.json_response( + data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code + ) + + # Convert to extended Task with handler methods + task = ExtendedTask.model_validate(model) + _status, _message = await task.mark() + + if not _status: + return web.json_response(data={"error": _message}, status=web.HTTPBadRequest.status_code) + + return web.json_response(data={"message": _message}, status=web.HTTPOk.status_code, dumps=encoder.encode) + except ValueError as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) + + +@route("DELETE", r"api/tasks/{id:\d+}/mark", "tasks_unmark") +async def task_unmark(request: Request, repo: TasksRepository, encoder: Encoder) -> Response: + """ + Remove all task items from download archive. + + Args: + request: The request object. + repo: The tasks repository instance. + encoder: The encoder instance. + + Returns: + The response object. + + """ + if not (task_id := request.match_info.get("id")): + return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) + + try: + model = await repo.get(int(task_id)) + if not model: + return web.json_response( + data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code + ) + + # Convert to extended Task with handler methods + task = ExtendedTask.model_validate(model) + _status, _message = await task.unmark() + + if not _status: + return web.json_response(data={"error": _message}, status=web.HTTPBadRequest.status_code) + + return web.json_response(data={"message": _message}, status=web.HTTPOk.status_code, dumps=encoder.encode) + except ValueError as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) + + +@route("POST", r"api/tasks/{id:\d+}/metadata", "tasks_metadata") +async def task_metadata(request: Request, repo: TasksRepository, config: Config, encoder: Encoder) -> Response: + """ + Generate metadata for the task. + + Args: + request: The request object. + repo: The tasks repository instance. + config: The config instance. + encoder: The encoder instance. + + Returns: + The response object. + + """ + task_id = request.match_info.get("id") + + try: + if not (model := await repo.get(int(task_id))): + return web.json_response( + data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code + ) + + # Convert to extended Task with handler methods + task = ExtendedTask.model_validate(model) + + (save_path, _) = get_file(config.download_path, task.folder) + if not str(save_path or "").startswith(str(config.download_path)): + return web.json_response(data={"error": "Invalid task folder."}, status=web.HTTPBadRequest.status_code) + + if not save_path.exists(): + save_path.mkdir(parents=True, exist_ok=True) + + metadata, status, message = await task.fetch_metadata() + if not status: + return web.json_response(data={"error": message}, status=web.HTTPBadRequest.status_code) + + if not task.folder: + try: + ytdlp_opts: dict = task.get_ytdlp_opts().get_all() + outtmpl: str = parse_outtmpl( + output_template=ytdlp_opts.get("outtmpl", {}).get("default", "{title} [{id}]"), + info_dict=metadata, + params=ytdlp_opts, + ) + if outtmpl: + _path: Path = save_path / outtmpl + if not _path.is_dir(): + _path: Path = _path.parent + + (save_path, _) = get_file(config.download_path, _path.relative_to(config.download_path)) + if not str(save_path or "").startswith(str(config.download_path)): + return web.json_response( + data={"error": "Invalid final path folder."}, status=web.HTTPBadRequest.status_code + ) + + if not save_path.exists(): + save_path.mkdir(parents=True, exist_ok=True) + except Exception as e: + LOG.warning(f"Failed to resolve final path from outtmpl. '{e!s}'") + + info = { + "id": ag(metadata, ["id", "channel_id"]), + "id_type": metadata.get("extractor", "").split(":")[0].lower() if metadata.get("extractor") else None, + "title": ag(metadata, ["title", "fulltitle"]) or None, + "description": metadata.get("description", ""), + "uploader": metadata.get("uploader", ""), + "tags": metadata.get("tags", []), + "year": metadata.get("release_year"), + "thumbnails": get_channel_images(metadata.get("thumbnails", {})), + } + + if not info.get("title"): + return web.json_response( + data={"error": "Failed to get title from metadata."}, status=web.HTTPBadRequest.status_code + ) + + LOG.info(f"Generating metadata for task '{task.name}' in '{save_path!s}'") + + from yt_dlp.utils import sanitize_filename + + from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP + + title: str = sanitize_filename(info.get("title")) + info_file: Path = save_path / f"{title} [{info.get('id')}].info.json" + info_file.write_text(encoder.encode(metadata), encoding="utf-8") + info["json_file"] = str(info_file.relative_to(config.download_path)) + + xml_file: Path = save_path / "tvshow.nfo" + info["nfo_file"] = str(xml_file.relative_to(config.download_path)) + + xml_content = "\n" + xml_content += f" {NFOMakerPP._escape_text(info.get('title'))}\n" + if info.get("description"): + xml_content += ( + f" {NFOMakerPP._escape_text(NFOMakerPP._clean_description(info.get('description')))}\n" + ) + if info.get("id"): + xml_content += f" {NFOMakerPP._escape_text(info.get('id'))}\n" + if info.get("id_type") and info.get("id"): + xml_content += f' {NFOMakerPP._escape_text(info.get("id"))}\n' + if info.get("uploader"): + xml_content += f" {NFOMakerPP._escape_text(info.get('uploader'))}\n" + if info.get("tags", []): + for tag in info.get("tags", []): + xml_content += f" {NFOMakerPP._escape_text(tag)}\n" + if info.get("year"): + xml_content += f" {info.get('year')}\n" + xml_content += " Continuing\n" + xml_content += "\n" + xml_file.write_text(xml_content, encoding="utf-8") + + try: + from yt_dlp.utils.networking import random_user_agent + + from app.library.httpx_client import async_client + + ytdlp_args: dict = task.get_ytdlp_opts().get_all() + opts: dict[str, Any] = { + "headers": { + "User-Agent": request.headers.get("User-Agent", ytdlp_args.get("user_agent", random_user_agent())), + }, + } + if proxy := ytdlp_args.get("proxy"): + opts["proxy"] = proxy + + try: + from httpx_curl_cffi import AsyncCurlTransport, CurlOpt + + opts["transport"] = AsyncCurlTransport( + impersonate="chrome", + default_headers=True, + curl_options={CurlOpt.FRESH_CONNECT: True}, + ) + opts.pop("headers", None) + except Exception: + pass + + async with async_client(**opts) as client: + for key in info.get("thumbnails", {}): + try: + url = info["thumbnails"][key] + LOG.info(f"Fetching thumbnail '{key}' from '{url}'") + if not url: + continue + + try: + validate_url(url, allow_internal=config.allow_internal_urls) + except ValueError: + LOG.warning(f"Invalid thumbnail url '{url}'") + continue + + resp = await client.request(method="GET", url=url, follow_redirects=True) + + img_file = save_path / f"{key}.jpg" + img_file.write_bytes(resp.content) + info["thumbnails"][key] = str(img_file.relative_to(config.download_path)) + except Exception as e: + LOG.warning(f"Failed to fetch thumbnail '{key}' from '{url}'. '{e!s}'") + continue + except Exception as e: + LOG.warning(f"Failed to fetch thumbnails. '{e!s}'") + + return web.json_response(data=info, status=web.HTTPOk.status_code, dumps=encoder.encode) + except ValueError as e: + return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) diff --git a/app/features/tasks/schemas.py b/app/features/tasks/schemas.py new file mode 100644 index 00000000..a1b1fb0d --- /dev/null +++ b/app/features/tasks/schemas.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +from datetime import datetime # noqa: TC003 +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from app.features.core.schemas import Pagination + + +class Task(BaseModel): + model_config = ConfigDict(from_attributes=True, str_strip_whitespace=True) + + id: int | None = None + name: str = Field(min_length=1) + url: str = Field(min_length=1) + folder: str = "" + preset: str = "" + timer: str = "" + template: str = "" + cli: str = "" + auto_start: bool = True + handler_enabled: bool = True + enabled: bool = True + created_at: datetime | None = None + updated_at: datetime | None = None + + @field_validator("name", mode="before") + @classmethod + def _normalize_name(cls, value: Any) -> str: + if not isinstance(value, str): + msg: str = "Name must be a string." + raise ValueError(msg) + value = value.strip() + if not value: + msg = "Name cannot be empty." + raise ValueError(msg) + return value + + @field_validator("url", mode="before") + @classmethod + def _normalize_url(cls, value: Any) -> str: + if not isinstance(value, str): + msg: str = "URL must be a string." + raise ValueError(msg) + value = value.strip() + if not value: + msg = "URL cannot be empty." + raise ValueError(msg) + + from app.library.Utils import validate_url + + try: + validate_url(value, allow_internal=True) + except ValueError as e: + msg = f"Invalid URL format: {e!s}" + raise ValueError(msg) from e + + return value + + @field_validator("timer", mode="before") + @classmethod + def _validate_timer(cls, value: Any) -> str: + if not value: + return "" + + if not isinstance(value, str): + msg: str = "Timer must be a string." + raise ValueError(msg) + + value = value.strip() + if not value: + return "" + + from datetime import UTC, datetime + + try: + from cronsim import CronSim + + CronSim(value, datetime.now(UTC)) + except Exception as e: + msg = f"Invalid timer format: {e!s}" + raise ValueError(msg) from e + + return value + + @field_validator("cli", mode="before") + @classmethod + def _validate_cli(cls, value: Any) -> str: + if not value: + return "" + + if not isinstance(value, str): + msg: str = "CLI must be a string." + raise ValueError(msg) + + value = value.strip() + if not value: + return "" + + from app.library.Utils import arg_converter + + try: + arg_converter(args=value) + except Exception as e: + msg = f"Invalid command options for yt-dlp: {e!s}" + raise ValueError(msg) from e + + return value + + +class TaskPatch(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + + name: str | None = None + url: str | None = None + folder: str | None = None + preset: str | None = None + timer: str | None = None + template: str | None = None + cli: str | None = None + auto_start: bool | None = None + handler_enabled: bool | None = None + enabled: bool | None = None + + +class TaskList(BaseModel): + model_config = ConfigDict(from_attributes=True) + + items: list[Task] = Field(default_factory=list) + pagination: Pagination diff --git a/app/features/tasks/service.py b/app/features/tasks/service.py new file mode 100644 index 00000000..2e4900d7 --- /dev/null +++ b/app/features/tasks/service.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from app.features.core.schemas import CEAction, CEFeature, ConfigEvent +from app.features.tasks.models import TaskModel +from app.features.tasks.utils import cron_time +from app.library.Events import Event, EventBus, Events +from app.library.Scheduler import Scheduler +from app.library.Services import Services +from app.library.Singleton import Singleton + +if TYPE_CHECKING: + from aiohttp import web + +LOG: logging.Logger = logging.getLogger(__name__) + + +class Tasks(metaclass=Singleton): + def __init__(self): + from app.features.tasks.deps import get_tasks_repo + + self._repo = get_tasks_repo() + self._loaded: bool = False + self._handlers_service = None + self._scheduler = Scheduler.get_instance() + + @staticmethod + def get_instance() -> Tasks: + """Get the singleton instance of Tasks.""" + return Tasks() + + def attach(self, _: web.Application) -> None: + async def handle_started(_, __): + await self._repo.run_migrations() + await self._load_tasks() + await self._init_handlers_service(self._scheduler) + + Services.get_instance().add("tasks_service", self).add("tasks_repository", self._repo) + + async def handle_config_update(e: Event, _): + if isinstance(e.data, ConfigEvent) and CEFeature.TASKS == e.data.feature: + await self._handle_task_change(e.data) + + EventBus.get_instance().subscribe( + Events.CONFIG_UPDATE, handle_config_update, "Tasks.config_update_scheduler" + ).subscribe(Events.STARTED, handle_started, "TasksRepository.run_migrations") + + async def on_shutdown(self, _: web.Application) -> None: + pass + + async def _load_tasks(self) -> None: + tasks = await self._repo.list() + + for task in tasks: + if not task.timer or not task.enabled: + continue + + try: + self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=f"task-cronjob-{task.id}") + LOG.info(f"Task '{task.id}: {task.name}' queued to be executed '{cron_time(task.timer)}'.") + except Exception as e: + LOG.exception(e) + LOG.error(f"Failed to queue task '{task.name}'. '{e!s}'.") + + async def _init_handlers_service(self, scheduler) -> None: + """Initialize the handlers service after migrations.""" + if self._handlers_service is not None: + return + + from app.features.tasks.definitions.service import TaskHandle + from app.library.config import Config + + config = Config.get_instance() + self._handlers_service = TaskHandle(scheduler, self._repo, config) + self._handlers_service.load() + LOG.debug("Task handlers service initialized.") + Services.get_instance().add("task_handle_service", self._handlers_service) + + async def _handle_task_change(self, event_data) -> None: + task_data: dict = event_data.data + task_id: str = f"task-cronjob-{task_data['id']}" + + if CEAction.DELETE == event_data.action: + if self._scheduler.has(task_id): + self._scheduler.remove(task_id) + + elif event_data.action in (CEAction.CREATE, CEAction.UPDATE): + if not (task := await self._repo.get(int(task_data["id"]))): + return + + if self._scheduler.has(task_id): + self._scheduler.remove(task_id) + + if task.timer and task.enabled: + self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=task_id) + LOG.info(f"Task '{task.id}: {task.name}' queued to be executed '{cron_time(task.timer)}'.") + + async def _runner(self, task: TaskModel) -> None: + """ + Execute a scheduled task. + + Args: + task: The TaskModel to execute. + + """ + import time + from datetime import UTC, datetime + + from app.library.config import Config + from app.library.downloads import DownloadQueue + from app.library.ItemDTO import Item + + timeNow: str = datetime.now(UTC).isoformat() + try: + if not (task := await self._repo.get(task.id)): + LOG.info(f"Task '{task.name}' no longer exists.") + return + + if not task.enabled: + LOG.debug(f"Task '{task.name}' is disabled. Skipping execution.") + return + + if not task.url: + LOG.error(f"Failed to dispatch '{task.name}'. No URL found.") + return + + started: float = time.time() + + config = Config.get_instance() + preset: str = task.preset or config.default_preset + folder: str = task.folder or "" + template: str = task.template or "" + cli: str = task.cli or "" + + notify: EventBus = EventBus.get_instance() + + status = await DownloadQueue.get_instance().add( + item=Item.format( + { + "url": task.url, + "preset": preset, + "folder": folder, + "template": template, + "cli": cli, + "auto_start": task.auto_start, + "extras": { + "source_name": task.name, + "source_id": str(task.id), + "source_handler": "Tasks", + }, + } + ) + ) + + timeNow = datetime.now(UTC).isoformat() + ended: float = time.time() + LOG.info(f"Task '{task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.") + + notify.emit( + Events.TASK_DISPATCHED, + data={**status, "preset": task.preset} if status else {"preset": task.preset}, + title=f"Task '{task.name}' dispatched", + message=f"Task '{task.name}' dispatched at '{timeNow}'.", + ) + notify.emit( + Events.LOG_SUCCESS, + data={"preset": task.preset, "lowPriority": True}, + title="Task completed", + message=f"Task '{task.name}' completed in '{ended - started:.2f}'.", + ) + except Exception as e: + LOG.error(f"Failed to execute '{task.name}' at '{timeNow}'. '{e!s}'.") + EventBus.get_instance().emit( + Events.LOG_ERROR, + data={"preset": task.preset}, + title="Task failed", + message=f"Failed to execute '{task.name}'. '{e!s}'", + ) + + @property + def handlers(self): + """Get the handlers service instance.""" + return self._handlers_service diff --git a/app/features/tasks/tests/__init__.py b/app/features/tasks/tests/__init__.py new file mode 100644 index 00000000..4c4f8811 --- /dev/null +++ b/app/features/tasks/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for tasks feature.""" diff --git a/app/features/tasks/tests/test_tasks_repository.py b/app/features/tasks/tests/test_tasks_repository.py new file mode 100644 index 00000000..6ceacbe6 --- /dev/null +++ b/app/features/tasks/tests/test_tasks_repository.py @@ -0,0 +1,289 @@ +"""Tests for TasksRepository.""" + +from __future__ import annotations + +import pytest +import pytest_asyncio + +from app.features.tasks.models import TaskModel +from app.features.tasks.repository import TasksRepository +from app.library.sqlite_store import SqliteStore + + +@pytest_asyncio.fixture +async def repo(tmp_path): + """Provide a fresh repository instance with initialized database for each test.""" + TasksRepository._reset_singleton() + SqliteStore._reset_singleton() + + store = SqliteStore(db_path=str(":memory:")) + await store.get_connection() + + repository = TasksRepository.get_instance() + + yield repository + + if store._conn: + await store._conn.close() + if store._engine: + await store._engine.dispose() + + TasksRepository._reset_singleton() + SqliteStore._reset_singleton() + + +class TestTasksRepository: + """Test suite for TasksRepository database operations.""" + + @pytest.mark.asyncio + async def test_repository_singleton(self, repo): + """Verify repository follows singleton pattern.""" + instance1 = TasksRepository.get_instance() + instance2 = TasksRepository.get_instance() + assert instance1 is instance2, "Should return same singleton instance" + + @pytest.mark.asyncio + async def test_list_empty(self, repo): + """List returns empty when no tasks exist.""" + tasks = await repo.list() + assert tasks == [], "Should return empty list when no tasks" + + @pytest.mark.asyncio + async def test_count_empty(self, repo): + """Count returns 0 when no tasks exist.""" + count = await repo.count() + assert count == 0, "Should return 0 when no tasks exist" + + @pytest.mark.asyncio + async def test_create_task(self, repo): + """Create task with valid data.""" + data = { + "name": "Daily Download", + "url": "https://example.com/video", + "folder": "/downloads", + "preset": "audio", + "timer": "0 0 * * *", + "template": "%(title)s.%(ext)s", + "cli": "--format best", + "auto_start": True, + "handler_enabled": True, + "enabled": True, + } + + model = await repo.create(data) + + assert model.id is not None, "Should generate ID for new task" + assert model.name == "Daily Download", "Should store name correctly" + assert model.url == "https://example.com/video", "Should store URL correctly" + assert model.folder == "/downloads", "Should store folder correctly" + assert model.preset == "audio", "Should store preset correctly" + assert model.timer == "0 0 * * *", "Should store timer correctly" + assert model.template == "%(title)s.%(ext)s", "Should store template correctly" + assert model.cli == "--format best", "Should store CLI correctly" + assert model.auto_start is True, "Should store auto_start correctly" + assert model.handler_enabled is True, "Should store handler_enabled correctly" + assert model.enabled is True, "Should store enabled correctly" + assert model.created_at is not None, "Should have created_at timestamp" + assert model.updated_at is not None, "Should have updated_at timestamp" + + @pytest.mark.asyncio + async def test_create_with_minimal_data(self, repo): + """Create task with minimal required data.""" + data = { + "name": "Simple Task", + "url": "https://example.com", + } + + model = await repo.create(data) + + assert model.name == "Simple Task", "Should store name" + assert model.url == "https://example.com", "Should store URL" + assert model.folder == "", "Should default folder to empty string" + assert model.preset == "", "Should default preset to empty string" + assert model.timer == "", "Should default timer to empty string" + assert model.template == "", "Should default template to empty string" + assert model.cli == "", "Should default CLI to empty string" + assert model.auto_start is True, "Should default auto_start to True" + assert model.handler_enabled is True, "Should default handler_enabled to True" + assert model.enabled is True, "Should default enabled to True" + + @pytest.mark.asyncio + async def test_get_by_id(self, repo): + """Get task by integer ID.""" + created = await repo.create( + { + "name": "Get Test", + "url": "https://example.com", + } + ) + + retrieved = await repo.get(created.id) + + assert retrieved is not None, "Should retrieve created task" + assert retrieved.id == created.id, "Should retrieve correct task by ID" + assert retrieved.name == "Get Test", "Should match created task name" + + @pytest.mark.asyncio + async def test_get_by_string_id(self, repo): + """Get task by string ID.""" + created = await repo.create( + { + "name": "String ID Test", + "url": "https://example.com", + } + ) + + retrieved = await repo.get(str(created.id)) + + assert retrieved is not None, "Should retrieve by string ID" + assert retrieved.id == created.id, "Should match created task ID" + + @pytest.mark.asyncio + async def test_get_nonexistent(self, repo): + """Get nonexistent task returns None.""" + result = await repo.get(99999) + assert result is None, "Should return None for nonexistent ID" + + result = await repo.get("99999") + assert result is None, "Should return None for nonexistent string ID" + + @pytest.mark.asyncio + async def test_update_task(self, repo): + """Update existing task.""" + created = await repo.create( + { + "name": "Update Test", + "url": "https://example.com", + "preset": "video", + "enabled": True, + } + ) + + updated = await repo.update( + created.id, + { + "name": "Updated Name", + "preset": "audio", + "enabled": False, + }, + ) + + assert updated.name == "Updated Name", "Should update name" + assert updated.preset == "audio", "Should update preset" + assert updated.enabled is False, "Should update enabled" + assert updated.url == "https://example.com", "Should preserve unchanged URL" + + @pytest.mark.asyncio + async def test_update_nonexistent_raises(self, repo): + """Update nonexistent task raises KeyError.""" + with pytest.raises(KeyError, match="not found"): + await repo.update(99999, {"name": "should_fail"}) + + @pytest.mark.asyncio + async def test_delete_task(self, repo): + """Delete existing task.""" + created = await repo.create( + { + "name": "Delete Test", + "url": "https://example.com", + } + ) + + deleted = await repo.delete(created.id) + + assert deleted.id == created.id, "Should return deleted task" + + result = await repo.get(created.id) + assert result is None, "Deleted task should not be retrievable" + + @pytest.mark.asyncio + async def test_delete_nonexistent_raises(self, repo): + """Delete nonexistent task raises KeyError.""" + with pytest.raises(KeyError, match="not found"): + await repo.delete(99999) + + @pytest.mark.asyncio + async def test_get_by_name(self, repo): + """Get task by name.""" + await repo.create( + { + "name": "Named Task", + "url": "https://example.com", + } + ) + + retrieved = await repo.get_by_name("Named Task") + + assert retrieved is not None, "Should retrieve by name" + assert retrieved.name == "Named Task", "Should match task name" + + @pytest.mark.asyncio + async def test_get_by_name_excludes_id(self, repo): + """Get by name can exclude specific ID.""" + first = await repo.create( + { + "name": "duplicate", + "url": "https://example.com", + } + ) + + result = await repo.get_by_name("duplicate", exclude_id=first.id) + assert result is None, "Should not find when excluding only match" + + result = await repo.get_by_name("duplicate", exclude_id=None) + assert result is not None, "Should find without exclusion" + + @pytest.mark.asyncio + async def test_get_all_enabled(self, repo): + """Get all enabled tasks.""" + await repo.create({"name": "Enabled 1", "url": "https://example.com", "enabled": True}) + await repo.create({"name": "Disabled", "url": "https://example.com", "enabled": False}) + await repo.create({"name": "Enabled 2", "url": "https://example.com", "enabled": True}) + + enabled = await repo.get_all_enabled() + + assert len(enabled) == 2, "Should return only enabled tasks" + assert all(task.enabled for task in enabled), "All tasks should be enabled" + + @pytest.mark.asyncio + async def test_get_all_with_timer(self, repo): + """Get all tasks with timer configured.""" + await repo.create({"name": "With Timer", "url": "https://example.com", "timer": "0 0 * * *"}) + await repo.create({"name": "No Timer", "url": "https://example.com", "timer": ""}) + await repo.create({"name": "Another Timer", "url": "https://example.com", "timer": "0 12 * * *"}) + + with_timer = await repo.get_all_with_timer() + + assert len(with_timer) == 2, "Should return only tasks with timer" + assert all(task.timer for task in with_timer), "All tasks should have timer" + + @pytest.mark.asyncio + async def test_list_paginated(self, repo): + """List paginated returns correct subset.""" + for i in range(5): + await repo.create( + { + "name": f"Task {i}", + "url": "https://example.com", + } + ) + + items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2) + + assert len(items) == 2, "Should return 2 items per page" + assert total == 5, "Should report total count of 5" + assert page == 1, "Should be on page 1" + assert total_pages == 3, "Should have 3 pages total" + + @pytest.mark.asyncio + async def test_list_ordering(self, repo): + """List orders by name ascending.""" + await repo.create({"name": "Charlie", "url": "https://example.com"}) + await repo.create({"name": "Alice", "url": "https://example.com"}) + await repo.create({"name": "Bob", "url": "https://example.com"}) + + items = await repo.list() + + assert items[0].name == "Alice", "Should be alphabetically first" + assert items[1].name == "Bob", "Should be alphabetically second" + assert items[2].name == "Charlie", "Should be alphabetically third" diff --git a/app/features/tasks/utils.py b/app/features/tasks/utils.py new file mode 100644 index 00000000..f411a576 --- /dev/null +++ b/app/features/tasks/utils.py @@ -0,0 +1,16 @@ +import logging + +LOG: logging.Logger = logging.getLogger(__name__) + + +def cron_time(timer: str) -> str: + try: + from datetime import UTC, datetime + + from cronsim import CronSim + + cs = CronSim(timer, datetime.now(UTC)) + return cs.explain() + except Exception as exc: + LOG.exception(exc) + return timer diff --git a/app/library/Events.py b/app/library/Events.py index 05afb8cd..2cef42a2 100644 --- a/app/library/Events.py +++ b/app/library/Events.py @@ -1,15 +1,16 @@ import asyncio import datetime import logging -import uuid -from collections.abc import Awaitable +from collections.abc import Callable from dataclasses import dataclass, field from typing import Any +from app.features.core.utils import gen_random + from .BackgroundWorker import BackgroundWorker from .Singleton import Singleton -LOG: logging.Logger = logging.getLogger("library.events") +LOG: logging.Logger = logging.getLogger("events") class Events: @@ -25,6 +26,7 @@ class Events: CONNECTED: str = "connected" CONFIGURATION: str = "configuration" + CONFIG_UPDATE: str = "config_update" ACTIVE_QUEUE: str = "active_queue" LOG_INFO: str = "log_info" @@ -49,29 +51,13 @@ class Events: PAUSED: str = "paused" RESUMED: str = "resumed" - CLI_POST: str = "cli_post" - CLI_CLOSE: str = "cli_close" - CLI_OUTPUT: str = "cli_output" - TASKS_ADD: str = "task_add" TASK_DISPATCHED: str = "task_dispatched" TASK_FINISHED: str = "task_finished" TASK_ERROR: str = "task_error" - PRESETS_ADD: str = "presets_add" - PRESETS_UPDATE: str = "presets_update" - - DLFIELDS_ADD: str = "dlfields_add" - DLFIELDS_UPDATE: str = "dlfields_update" - SCHEDULE_ADD: str = "schedule_add" - CONDITIONS_ADD: str = "conditions_add" - CONDITIONS_UPDATE: str = "conditions_update" - - SUBSCRIBED: str = "subscribed" - UNSUBSCRIBED: str = "unsubscribed" - def get_all() -> list: """ Get all the events. @@ -94,6 +80,7 @@ class Events: """ return [ Events.CONFIGURATION, + Events.CONFIG_UPDATE, Events.CONNECTED, Events.ACTIVE_QUEUE, Events.LOG_INFO, @@ -108,10 +95,6 @@ class Events: Events.ITEM_STATUS, Events.PAUSED, Events.RESUMED, - Events.CLI_CLOSE, - Events.CLI_OUTPUT, - Events.PRESETS_UPDATE, - Events.DLFIELDS_UPDATE, ] def only_debug() -> list: @@ -122,7 +105,7 @@ class Events: list: The list of debug events. """ - return [Events.ITEM_UPDATED, Events.CLI_OUTPUT] + return [Events.ITEM_UPDATED] @dataclass(kw_only=True) @@ -131,7 +114,7 @@ class Event: Event is a data transfer object that represents an event that was emitted. """ - id: str = field(default_factory=lambda: str(uuid.uuid4()), init=False) + id: str = field(default_factory=lambda: str(gen_random(16)), init=False) """The id of the event.""" created_at: str = field(default_factory=lambda: str(datetime.datetime.now(tz=datetime.UTC).isoformat())) @@ -198,10 +181,10 @@ class Event: class EventListener: - def __init__(self, name: str, callback: callable): + def __init__(self, name: str, callback: Callable[..., Any]): self.name: str = name "The name of the listener." - self.call_back: callable = callback + self.call_back: Callable[..., Any] = callback "The callback function to call when the event is emitted." self.is_coroutine: bool = asyncio.iscoroutinefunction(callback) "Whether the callback is a coroutine function or not." @@ -219,13 +202,13 @@ class EventBus(metaclass=Singleton): """ def __init__(self): - self._listeners: dict[str, list[str, EventListener]] = {} + self._listeners: dict[str, list[tuple[str, EventListener]]] = {} "The listeners for the events." self.debug: bool = False "Whether to log debug messages or not." - self._offload: BackgroundWorker = None + self._offload: BackgroundWorker | None = None "The background worker to offload tasks to." @staticmethod @@ -239,7 +222,7 @@ class EventBus(metaclass=Singleton): """ return EventBus() - def subscribe(self, event: str | list | tuple, callback: Awaitable, name: str | None = None) -> "EventBus": + def subscribe(self, event: str | list | tuple, callback: Callable[..., Any], name: str | None = None) -> "EventBus": """ Subscribe to an event. @@ -267,7 +250,7 @@ class EventBus(metaclass=Singleton): event = [event] if not name: - name = str(uuid.uuid4()) + name = gen_random(12) for e in event: if e not in all_events: @@ -275,9 +258,10 @@ class EventBus(metaclass=Singleton): continue if e not in self._listeners: - self._listeners[e] = {} + self._listeners[e] = [] - self._listeners[e][name] = EventListener(name, callback) + self._listeners[e] = [(n, listener) for n, listener in self._listeners[e] if n != name] + self._listeners[e].append((name, EventListener(name, callback))) LOG.debug(f"'{name}' subscribed to '{event}'.") @@ -300,9 +284,11 @@ class EventBus(metaclass=Singleton): events = [] for e in event: - if e in self._listeners and name in self._listeners[e]: - events.append(e) - del self._listeners[e][name] + if e in self._listeners: + original_len: int = len(self._listeners[e]) + self._listeners[e] = [(n, listener) for n, listener in self._listeners[e] if n != name] + if len(self._listeners[e]) < original_len: + events.append(e) if len(events) > 0: LOG.debug(f"'{name}' unsubscribed from '{events}'.") @@ -337,22 +323,25 @@ class EventBus(metaclass=Singleton): try: loop = asyncio.get_running_loop() - for handler in self._listeners[event].values(): - try: - if handler.is_coroutine: - coro = handler.call_back(ev, handler.name, **kwargs) - if asyncio.iscoroutine(coro): - loop.create_task(coro) + async def execute_handlers(): + for _, handler in self._listeners[event]: + try: + if handler.is_coroutine: + coro = handler.call_back(ev, handler.name, **kwargs) + if asyncio.iscoroutine(coro): + await coro + else: + LOG.warning(f"Expected coroutine from async handler '{handler.name}', got {type(coro)}") else: - LOG.warning(f"Expected coroutine from async handler '{handler.name}', got {type(coro)}") - else: - loop.create_task(self._call(handler, ev, kwargs), name=f"sync-handler-{handler.name}-{ev.id}") - except Exception as e: - LOG.exception(e) - LOG.error(f"Failed to emit event '{ev.event}' to '{handler.name}'. Error message '{e!s}'.") + await self._call(handler, ev, kwargs) + except Exception as e: + LOG.exception(e) + LOG.error(f"Failed to emit event '{ev.event}' to '{handler.name}'. Error message '{e!s}'.") + + loop.create_task(execute_handlers()) except RuntimeError: LOG.debug(f"No event loop detected - using BackgroundWorker for {len(self._listeners[event])} handlers") - for handler in self._listeners[event].values(): + for _, handler in self._listeners[event]: try: if not self._offload: self._offload = BackgroundWorker.get_instance() diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 7c00ee2d..f8dc5f00 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -28,7 +28,7 @@ class HttpAPI: self.config: Config = Config.get_instance() self._notify: EventBus = EventBus.get_instance() self.rootPath: Path = root_path - self.cache = Cache() + self.cache: Cache = Cache.get_instance() self.app: web.Application | None = None services: Services = Services.get_instance() diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index b73ac236..33163277 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -1,12 +1,13 @@ import functools +import json import logging from pathlib import Path from typing import Any -import socketio from aiohttp import web -from app.library.router import RouteType, get_routes +from app.features.core.utils import gen_random +from app.library.router import Route, RouteType, get_routes from app.library.Services import Services from app.library.Utils import load_modules @@ -18,13 +19,55 @@ from .ItemDTO import Item LOG: logging.Logger = logging.getLogger("socket_api") +class WebSocketHub: + def __init__(self, encoder: Encoder): + self._encoder: Encoder = encoder + self._clients: dict[str, web.WebSocketResponse] = {} + + def add(self, sid: str, ws: web.WebSocketResponse) -> None: + self._clients[sid] = ws + + def remove(self, sid: str) -> None: + self._clients.pop(sid, None) + + async def emit(self, event: str, data: Any, to: str | None = None) -> None: + payload: str = self._encoder.encode({"event": event, "data": data}) + + if to: + await self._send(to, payload) + return + + for sid in list(self._clients.keys()): + await self._send(sid, payload) + + async def disconnect(self, sid: str) -> None: + ws: web.WebSocketResponse | None = self._clients.pop(sid, None) + if ws and not ws.closed: + await ws.close() + + async def disconnect_all(self) -> None: + for sid in list(self._clients.keys()): + await self.disconnect(sid) + + async def _send(self, sid: str, payload: str) -> None: + ws: web.WebSocketResponse | None = self._clients.get(sid) + if not ws or ws.closed: + self._clients.pop(sid, None) + return + + try: + await ws.send_str(payload) + except ConnectionResetError: + self._clients.pop(sid, None) + + class HttpSocket: """ This class is used to handle WebSocket events. """ config: Config - sio: socketio.AsyncServer + sio: WebSocketHub di_context: dict[str, Any] = {} def __init__( @@ -32,26 +75,18 @@ class HttpSocket: root_path: Path, encoder: Encoder | None = None, config: Config | None = None, - sio: socketio.AsyncServer | None = None, + sio: WebSocketHub | None = None, ): self.config = config or Config.get_instance() self._notify = EventBus.get_instance() - self.sio = sio or socketio.AsyncServer( - async_handlers=True, - async_mode="aiohttp", - cors_allowed_origins="*", - transports=["websocket", "polling"], - logger=self.config.debug, - engineio_logger=self.config.debug, - ping_interval=10, - ping_timeout=5, - ) encoder = encoder or Encoder() + self.sio = sio or WebSocketHub(encoder=encoder) self.rootPath: Path = root_path async def event_handler(e: Event, _, **kwargs): - await self.sio.emit(event=e.event, data=encoder.encode(e), **kwargs) + payload = json.loads(encoder.encode(e)) + await self.sio.emit(event=e.event, data=payload, **kwargs) services: Services = Services.get_instance() services.add_all( @@ -85,38 +120,86 @@ class HttpSocket: async def on_shutdown(self, _: web.Application): LOG.debug("Shutting down socket server.") - - for sid in self.sio.manager.get_participants("/", None): - LOG.debug(f"Disconnecting client '{sid}'.") - await self.sio.disconnect(sid[0], namespace="/") - + await self.sio.disconnect_all() LOG.debug("Socket server shutdown complete.") def attach(self, app: web.Application): app.on_shutdown.append(self.on_shutdown) - self.sio.attach(app, socketio_path=f"{self.config.base_path.rstrip('/')}/socket.io") + base_path: str = self.config.base_path.rstrip("/") + ws_path: str = f"{base_path}/ws" if base_path else "/ws" async def event_handler(data: Event, _): - if data and data.data: - await Services.get_instance().get("queue").add(item=Item.format(data.data)) + if not (data and data.data): + return + + await Services.get_instance().get("queue").add(item=Item.format(data.data)) self._notify.subscribe(Events.ADD_URL, event_handler, f"{__class__.__name__}.add") load_modules(self.rootPath, self.rootPath / "routes" / "socket") - for route in get_routes(RouteType.SOCKET).values(): + socket_routes: dict[str, Route] = {route.path: route for route in get_routes(RouteType.SOCKET).values()} + + for route in socket_routes.values(): if self.config.debug: LOG.debug( f"Add ({route.name}) {route.method.value if isinstance(route.method, RouteType) else route.method}: {route.path}." ) - self.sio.on(route.path)(HttpSocket._injector(route.handler, route.path)) - @staticmethod - def _injector(func, event: str): - async def wrapper(sid, data=None, **kwargs): - if not data: - data = {} - return await Services.get_instance().handle_async(func, sid=sid, data=data, event=event, **kwargs) + async def handle_message(sid: str, message: str) -> None: + try: + payload = json.loads(message) + except json.JSONDecodeError: + LOG.debug("Invalid websocket payload received.") + return - return wrapper + event = payload.get("event") if isinstance(payload, dict) else None + if not event or not isinstance(event, str): + LOG.debug("Missing websocket event name.") + return + + if not (route := socket_routes.get(event)): + LOG.debug(f"Unknown websocket event '{event}'.") + return + + data = payload.get("data") if isinstance(payload, dict) else None + await Services.get_instance().handle_async(route.handler, sid=sid, data=data, event=event) + + async def handle_connect(sid: str) -> None: + if not (route := socket_routes.get("connect")): + return + await Services.get_instance().handle_async(route.handler, sid=sid, event="connect") + + async def handle_disconnect(sid: str, reason: str | None) -> None: + if not (route := socket_routes.get("disconnect")): + return + await Services.get_instance().handle_async(route.handler, sid=sid, data=reason, event="disconnect") + + async def ws_handler(request: web.Request) -> web.WebSocketResponse: + ws = web.WebSocketResponse(heartbeat=10) + await ws.prepare(request) + + sid: str = gen_random(14) + self.sio.add(sid, ws) + LOG.debug(f"WebSocket client '{sid}' connected.") + + await handle_connect(sid) + + try: + async for msg in ws: + if msg.type == web.WSMsgType.TEXT: + await handle_message(sid, msg.data) + elif msg.type == web.WSMsgType.ERROR: + LOG.error(f"WebSocket connection closed with exception {ws.exception()}") + finally: + await handle_disconnect(sid, getattr(ws, "close_reason", None)) + self.sio.remove(sid) + LOG.debug(f"WebSocket client '{sid}' disconnected.") + + return ws + + if self.config.debug: + LOG.debug(f"Add (ws) GET: {ws_path}.") + + app.router.add_get(ws_path, ws_handler, name="ws") diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py index d3faaf2f..d1439bbf 100644 --- a/app/library/ItemDTO.py +++ b/app/library/ItemDTO.py @@ -20,7 +20,7 @@ from app.library.Utils import ( from app.library.YTDLPOpts import YTDLPOpts if TYPE_CHECKING: - from app.library.Presets import Preset + from app.features.presets.schemas import Preset LOG: logging.Logger = logging.getLogger("ItemDTO") @@ -184,7 +184,7 @@ class Item: preset: str | None = item.get("preset") if preset and isinstance(preset, str) and preset != Item._default_preset(): - from .Presets import Presets + from app.features.presets.service import Presets if not Presets.get_instance().has(preset): msg: str = f"Preset '{preset}' does not exist." @@ -224,15 +224,15 @@ class Item: return Item(**data) - def get_preset(self) -> "Preset": + def get_preset(self) -> "Preset | None": """ Get the preset for the item. Returns: - Preset: The preset for the item. If not found, None. + Preset | None: The preset for the item. If not found, None. """ - from .Presets import Presets + from app.features.presets.service import Presets return Presets.get_instance().get(self.preset if self.preset else self._default_preset()) @@ -558,7 +558,7 @@ class ItemDTO: Preset | None: The preset for the item. If not found, None. """ - from .Presets import Presets + from app.features.presets.service import Presets return Presets.get_instance().get(self.preset if self.preset else "default") diff --git a/app/library/Notifications.py b/app/library/Notifications.py deleted file mode 100644 index e10e5145..00000000 --- a/app/library/Notifications.py +++ /dev/null @@ -1,559 +0,0 @@ -import asyncio -import json -import logging -import traceback -from collections.abc import Awaitable -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -import httpx -from aiohttp import web - -from .ag_utils import ag -from .BackgroundWorker import BackgroundWorker -from .config import Config -from .encoder import Encoder -from .Events import Event, EventBus, Events -from .httpx_client import async_client -from .ItemDTO import Item, ItemDTO -from .Presets import Preset, Presets -from .Singleton import Singleton -from .Utils import validate_uuid - -LOG = logging.getLogger("notifications") - - -@dataclass(kw_only=True) -class TargetRequestHeader: - """Request header details for a notification target.""" - - key: str - value: str - - def serialize(self) -> dict: - return {"key": self.key, "value": self.value} - - def json(self) -> str: - return Encoder().encode(self.serialize()) - - def get(self, key: str, default: Any = None) -> Any: - return self.serialize().get(key, default) - - -@dataclass(kw_only=True) -class TargetRequest: - """Request details for a notification target.""" - - type: str - method: str - url: str - headers: list[TargetRequestHeader] = field(default_factory=list) - data_key: str = "data" - - def serialize(self) -> dict: - return { - "type": self.type, - "method": self.method, - "url": self.url, - "data_key": self.data_key, - "headers": [h.serialize() for h in self.headers], - } - - def json(self) -> str: - return Encoder().encode(self.serialize()) - - def get(self, key: str, default: Any = None) -> Any: - return ag(self.serialize(), key, default) - - -@dataclass(kw_only=True) -class Target: - """Notification target details.""" - - id: str - name: str - on: list[str] = field(default_factory=list) - presets: list[str] = field(default_factory=list) - request: TargetRequest - enabled: bool = True - - def serialize(self) -> dict: - return { - "id": self.id, - "name": self.name, - "on": self.on, - "presets": self.presets, - "request": self.request.serialize(), - "enabled": self.enabled, - } - - def json(self) -> str: - return Encoder().encode(self.serialize()) - - def get(self, key: str, default: Any = None) -> Any: - return self.serialize().get(key, default) - - -class NotificationEvents: - TEST: str = Events.TEST - - ITEM_ADDED: str = Events.ITEM_ADDED - ITEM_COMPLETED: str = Events.ITEM_COMPLETED - ITEM_CANCELLED: str = Events.ITEM_CANCELLED - ITEM_DELETED: str = Events.ITEM_DELETED - ITEM_PAUSED: str = Events.ITEM_PAUSED - ITEM_RESUMED: str = Events.ITEM_RESUMED - ITEM_MOVED: str = Events.ITEM_MOVED - - PAUSED: str = Events.PAUSED - RESUMED: str = Events.RESUMED - - LOG_INFO: str = Events.LOG_INFO - LOG_SUCCESS: str = Events.LOG_SUCCESS - LOG_WARNING: str = Events.LOG_WARNING - LOG_ERROR: str = Events.LOG_ERROR - - TASK_DISPATCHED: str = Events.TASK_DISPATCHED - - @staticmethod - def get_events() -> dict[str, str]: - return {k: v for k, v in vars(NotificationEvents).items() if not k.startswith("__") and not callable(v)} - - def events() -> list: - return [ - getattr(NotificationEvents, ev) - for ev in dir(NotificationEvents) - if not ev.startswith("_") and not callable(getattr(NotificationEvents, ev)) - ] - - @staticmethod - def is_valid(event: str) -> bool: - return event in NotificationEvents.get_events().values() - - -class Notification(metaclass=Singleton): - def __init__( - self, - file: str | None = None, - client: httpx.AsyncClient | None = None, - encoder: Encoder | None = None, - config: Config | None = None, - background_worker: BackgroundWorker | None = None, - ): - self._targets: list[Target] = [] - "Notification targets to send events to." - - config: Config = config or Config.get_instance() - - self._debug: bool = config.debug - "Debug mode." - self._file: Path = Path(file) if file else Path(config.config_path).joinpath("notifications.json") - "File to store notification targets." - self._client: httpx.AsyncClient = client or async_client() - "HTTP client to send requests." - self._encoder: Encoder = encoder or Encoder() - "Encoder to encode data." - self._version: str = config.app_version - "Application version." - self._offload: BackgroundWorker = background_worker or BackgroundWorker.get_instance() - "Background worker to offload tasks to." - - if self._file.exists() and "600" != self._file.stat().st_mode: - try: - self._file.chmod(0o600) - except Exception: - pass - - @staticmethod - def get_instance( - file: str | None = None, - client: httpx.AsyncClient | None = None, - encoder: Encoder | None = None, - config: Config | None = None, - background_worker: BackgroundWorker | None = None, - ) -> "Notification": - """ - Get the instance of the class. - """ - return Notification( - file=file, client=client, encoder=encoder, config=config, background_worker=background_worker - ) - - def attach(self, _: web.Application): - """ - Attach the class to the application. - - Args: - _ (web.Application): The aiohttp application. - - """ - self.load() - EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{__class__.__name__}.emit") - - def get_targets(self) -> list[Target]: - """Get the list of notification targets.""" - return self._targets - - def clear(self) -> "Notification": - """Clear the list of notification targets.""" - self._targets.clear() - return self - - def save(self, targets: list[Target]) -> "Notification": - """ - Save notification targets to the file. - - Args: - targets (list[Target]|None): The list of targets to save. - - Returns: - Notification: The Notification instance. - - """ - try: - self._file.write_text(json.dumps([t.serialize() for t in targets], indent=4)) - LOG.info(f"Updated '{self._file}'.") - except Exception as e: - LOG.exception(e) - LOG.error(f"Error saving '{self._file}'. '{e!s}'") - - return self - - def load(self) -> "Notification": - """Load or reload notification targets from the file.""" - if len(self._targets) > 0: - self.clear() - - if not self._file.exists() or self._file.stat().st_size < 1: - return self - - targets: list = [] - - try: - LOG.info(f"Loading '{self._file}'.") - targets = json.loads(self._file.read_text()) - except Exception as e: - LOG.error(f"Error loading '{self._file}'. '{e!s}'") - - if not targets or len(targets) < 1: - LOG.debug(f"No targets were found in '{self._file}'.") - return self - - need_save = False - - for target in targets: - try: - if "enabled" not in target: - target["enabled"] = True - need_save = True - - try: - Notification.validate(target) - target: Target = self.make_target(target) - except ValueError as e: - name = target.get("name") or target.get("id") or target.get("request", {}).get("url") or "unknown" - LOG.error(f"Invalid notification target '{name}'. '{e!s}'") - continue - - self._targets.append(target) - - LOG.info( - f"Send '{target.request.type}' request on '{', '.join(target.on) if len(target.on) > 0 else 'all events'}' to '{target.name}'." - ) - except Exception as e: - LOG.error(f"Error loading notification target '{target}'. '{e!s}'") - - if need_save: - LOG.warning("Saving notifications due to schema changes.") - self.save(self._targets) - - return self - - def make_target(self, target: dict) -> Target: - """ - Make a notification target from a dictionary. - - Args: - target (dict): The target details. - - Returns: - Target: The notification target. - - """ - return Target( - id=target.get("id"), - name=target.get("name"), - on=target.get("on", []), - presets=target.get("presets", []), - enabled=target.get("enabled", True), - request=TargetRequest( - type=target.get("request", {}).get("type", "json"), - method=target.get("request", {}).get("method", "POST"), - url=target.get("request", {}).get("url"), - data_key=target.get("request", {}).get("data_key", "data"), - headers=[ - TargetRequestHeader( - key=str(h.get("key", "")).strip(), - value=str(h.get("value", "")).strip(), - ) - for h in target.get("request", {}).get("headers", []) - ], - ), - ) - - @staticmethod - def validate(target: Target | dict) -> bool: - """ - Validate a notification target. - - Args: - target (Target|dict): The target to validate. - - Returns: - bool: True if the target is valid, False otherwise. - - """ - if not isinstance(target, dict): - target = target.serialize() - - if "id" not in target or validate_uuid(target["id"], version=4) is False: - msg = "Invalid notification target. No ID found." - raise ValueError(msg) - - if "name" not in target: - msg = "Invalid notification target. No name found." - raise ValueError(msg) - - if "request" not in target: - msg = "Invalid notification target. No request details found." - raise ValueError(msg) - - if "url" not in target["request"]: - msg = "Invalid notification target. No URL found." - raise ValueError(msg) - - if "data_key" not in target["request"]: - target["request"]["data_key"] = "data" - - if "enabled" not in target: - target["enabled"] = True - - if target.get("enabled") is not None and not isinstance(target.get("enabled"), bool): - msg = "Enabled must be a boolean." - raise ValueError(msg) - - if "method" in target["request"] and target["request"]["method"].upper() not in ["POST", "PUT"]: - msg = "Invalid notification target. Invalid method found." - raise ValueError(msg) - - if "type" in target["request"] and target["request"]["type"].lower() not in ["json", "form"]: - msg = "Invalid notification target. Invalid type found." - raise ValueError(msg) - - if "on" in target: - if not isinstance(target["on"], list): - msg = "Invalid notification target. Invalid 'on' event list found." - raise ValueError(msg) - - removed_events: list = [] - all_events: dict[str, str] = NotificationEvents.get_events().values() - for e in target["on"]: - if e not in all_events: - removed_events.append(e) - target["on"].remove(e) - continue - - if len(removed_events) > 0 and len(target["on"]) < 1: - msg: str = f"Invalid notification target. Invalid events '{', '.join(removed_events)}' found." - raise ValueError(msg) - - if "presets" in target: - if not isinstance(target["presets"], list): - msg = "Invalid notification target. Invalid 'presets' list found." - raise ValueError(msg) - - removed_presets: list = [] - all_presets: list[Preset] = Presets.get_instance().get_all() - - for p in target["presets"]: - if p not in [ap.name for ap in all_presets]: - removed_presets.append(p) - target["presets"].remove(p) - continue - - if len(removed_presets) > 0 and len(target["presets"]) < 1: - msg: str = f"Invalid notification target. Invalid presets '{', '.join(removed_presets)}' found." - raise ValueError(msg) - - if "headers" in target["request"]: - if not isinstance(target["request"]["headers"], list): - msg = "Invalid notification target. Invalid headers list found." - raise ValueError(msg) - - for h in target["request"]["headers"]: - if "key" not in h: - msg = "Invalid notification target. No header key found." - raise ValueError(msg) - if "value" not in h: - msg = "Invalid notification target. No header value found." - raise ValueError(msg) - - return True - - async def send(self, ev: Event, wait: bool = True) -> list[dict] | Awaitable[list[dict]]: - if len(self._targets) < 1: - return [] - - if not isinstance(ev.data, ItemDTO) and not isinstance(ev.data, dict): - LOG.debug(f"Received invalid item type '{type(ev.data)}' with event '{ev.event}'.") - return [] - - tasks = [] - - apprise_targets: list[Target] = [] - - for target in self._targets: - if not target.enabled: - continue - - if len(target.on) > 0 and ev.event not in target.on and "test" != ev.event: - continue - - if "test" != ev.event and not self._check_preset(target, ev): - continue - - if not target.request.url.startswith("http"): - apprise_targets.append(target) - else: - tasks.append(self._send(target, ev)) - - if len(apprise_targets) > 0: - tasks.append(self._apprise(apprise_targets, ev)) - - if wait: - return await asyncio.gather(*tasks) - - return tasks - - def _check_preset(self, target: Target, ev: Event) -> bool: - if len(target.presets) < 1: - return True - - if not isinstance(ev.data, (Item, ItemDTO, dict)): - return False - - preset_name: str | None = None - - if isinstance(ev.data, Item): - preset_name = ev.data.preset - - if isinstance(ev.data, ItemDTO): - preset_name = ev.data.preset - - if isinstance(ev.data, dict): - preset_name = ev.data.get("preset", None) - - if not preset_name: - return False - - return preset_name in target.presets - - async def _apprise(self, target: list[Target], ev: Event) -> dict: - if not target or not isinstance(target, list): - return {} - - import apprise - - try: - notify = apprise.Apprise(debug=self._debug) - apr_config = Path(Config.get_instance().apprise_config) - if apr_config.exists(): - apprise_config = notify.AppriseConfig() - apprise_config.add(apr_config) - notify.add(apprise_config) - - for t in target: - notify.add(t.request.url) - - status = await notify.async_notify( - body=ev.message or self._encoder.encode(ev.serialize()), - title=ev.title or f"YTPTube Event: {ev.event}", - ) - - if not status: - msg = "Apprise failed to send notification." - raise RuntimeError(msg) # noqa: TRY301 - except Exception as e: - LOG.exception(e) - LOG.error(f"Error sending Apprise notification: {e!s}") - return {"error": str(e), "event": ev.event, "id": ev.id, "targets": [t.name for t in target]} - - return {} - - async def _send(self, target: Target, ev: Event) -> dict: - try: - LOG.info(f"Sending Notification event '{ev.event}: {ev.id}' to '{target.name}'.") - - headers: dict[str, str] = { - "User-Agent": f"YTPTube/{self._version}", - "X-Event-Id": ev.id, - "X-Event": ev.event, - "Content-Type": "application/json" - if "json" == target.request.type.lower() - else "application/x-www-form-urlencoded", - } - - if len(target.request.headers) > 0: - headers.update({h.key: h.value for h in target.request.headers if h.key and h.value}) - - payload: dict = ev.serialize() - - if "data" != target.request.data_key: - payload[target.request.data_key] = payload["data"] - payload.pop("data", None) - - if "form" == target.request.type.lower(): - payload[target.request.data_key] = self._encoder.encode(payload[target.request.data_key]) - else: - payload = self._encoder.encode(payload) - - response = await self._client.request( - method=target.request.method.upper(), - url=target.request.url, - headers=headers, - data=payload if "form" == target.request.type.lower() else None, - content=payload if "json" == target.request.type.lower() else None, - ) - - respData: dict[str, Any] = { - "url": target.request.url, - "status": response.status_code, - "text": response.text, - } - - msg: str = f"Notification target '{target.name}' Responded to event '{ev.event}: {ev.id}' with status '{response.status_code}'." - if self._debug and respData.get("text"): - msg += f" body '{respData.get('text', '??')}'." - - LOG.info(msg) - - return respData - except Exception as e: - err_msg = str(e) - if not err_msg: - err_msg: str = type(e).__name__ - - tb = "".join(traceback.format_exception(type(e), e, e.__traceback__)) - LOG.error(f"Error sending Notification event '{ev.event}: {ev.id}' to '{target.name}'. '{err_msg!s}'. {tb}") - return {"url": target.request.url, "status": 500, "text": str(ev)} - - def emit(self, e: Event, _, **__) -> None: - if len(self._targets) < 1 or not NotificationEvents.is_valid(e.event): - return - - self._offload.submit(self.send, e) - return - - async def noop(self) -> None: - return None diff --git a/app/library/Presets.py b/app/library/Presets.py deleted file mode 100644 index 7f68346c..00000000 --- a/app/library/Presets.py +++ /dev/null @@ -1,419 +0,0 @@ -import json -import logging -import uuid -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from aiohttp import web - -from .config import Config -from .Events import EventBus, Events -from .Singleton import Singleton -from .Utils import arg_converter, create_cookies_file, init_class - -LOG = logging.getLogger("presets") - -DEFAULT_PRESETS: list[dict[int, dict[str, str | bool]]] = [ - { - "id": "3e163c6c-64eb-4448-924f-814b629b3810", - "name": "default", - "default": True, - "cli": "--socket-timeout 30 --download-archive %(archive_file)s", - "description": "Default preset for yt-dlp. It will download whatever yt-dlp decides is the best quality for the video and audio.", - }, - { - "id": "441675ed-b739-40f0-a0b0-1ecfcb9dc48d", - "name": "Mobile", - "cli": '--socket-timeout 30 --download-archive %(archive_file)s\n-t mp4 --merge-output-format mp4 --add-chapters --remux-video mp4 \n--embed-metadata --embed-thumbnail \n--postprocessor-args "-movflags +faststart"', - "default": True, - "description": "This preset is designed for mobile devices. It will download the best quality video and audio in mp4 format, merge them, and add chapters, metadata, and thumbnail.", - }, - { - "id": "441675ed-b739-40f0-a0b0-1ecfcb9dc48b", - "name": "1080p", - "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n-S vcodec:h264 --format 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'", - "default": True, - "description": "Download the best quality video and audio in mp4 format for 1080p resolution.", - }, - { - "id": "9719fcc3-4cf2-4d88-b1e4-74dff3dba00e", - "name": "720p", - "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n-S vcodec:h264 --format 'bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'", - "default": True, - "description": "Download the best quality video and audio in mp4 format for 720p resolution.", - }, - { - "id": "a6fd4b25-2b3e-458d-bb57-b75e41cc4330", - "name": "Audio Only", - "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n--extract-audio --add-chapters --embed-metadata --embed-thumbnail --format 'bestaudio/best'", - "default": True, - "description": "This preset is designed to download only the audio of the video. It will extract the audio, add chapters, metadata, and thumbnail.", - }, - { - "id": "2ade2c28-cad4-4a06-b7eb-2439fdf46f60", - "name": "Info Reader Plugin", - "description": 'This preset generate specific filename format and metadata to work with yt-dlp info reader plugins for jellyfin/emby and plex and to make play state sync work for WatchState.\n\nThere is one more step you need to do via Other > Terminal if you have it enabled or directly from container shell\n\nyt-dlp -I0 --write-info-json --write-thumbnail --convert-thumbnails jpg --paths /downloads/youtube -o "%(channel|Unknown_title)s [%(channel_id|Unknown_id)s]/%(title).180B [%(channel_id|Unknown_id)s].%(ext)s" -- https://www.youtube.com/channel/UCClfFsWcT3N2I7VTXXyt84A\n\nChange the url to the channel you want to download.\n\nFor more information please visit \nhttps://github.com/arabcoders/watchstate/blob/master/FAQ.md#how-to-get-watchstate-working-with-youtube-contentlibrary', - "template": "%(channel)s [%(channel_id|Unknown_id)s]/Season %(release_date>%Y,upload_date>%Y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(id)s].%(ext)s", - "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n--windows-filenames --write-info-json --embed-info-json \n--convert-thumbnails jpg --write-thumbnail --embed-metadata", - "default": True, - }, - { - "id": "2ade2c28-cad4-4a06-b7eb-2439fdf46f61", - "name": "NFO Maker TV", - "description": "This preset generate specific filename format and metadata to work with media servers like jellyfin/emby and plex for episodes.", - "template": "%(channel)s [%(channel_id|Unknown_id)s]/Season %(release_date>%Y,upload_date>%Y|Unknown)s/S%(release_date>%Y,upload_date>%Y)sE%(release_date>%m%d,upload_date>%m%d)s - %(title).100B [%(id)s].%(ext)s", - "cli": '--socket-timeout 30 --download-archive %(archive_file)s\n--use-postprocessor "NFOMakerPP:when=after_move" \n--windows-filenames --convert-thumbnails jpg --write-thumbnail', - "default": True, - }, - { - "id": "2ade2c28-cad4-4a06-b7eb-2439fdf46f61", - "name": "NFO Maker Movie", - "description": "This preset generate specific filename format and metadata to work with media servers like jellyfin/emby and plex for movies.", - "template": "%(channel)s [%(channel_id|Unknown_id)s]/Season %(release_date>%Y,upload_date>%Y|Unknown)s/S%(release_date>%Y,upload_date>%Y)sE%(release_date>%m%d,upload_date>%m%d)s - %(title).100B [%(id)s].%(ext)s", - "cli": '--socket-timeout 30 --download-archive %(archive_file)s\n--use-postprocessor "NFOMakerPP:when=after_move;mode=movie" \n--windows-filenames --convert-thumbnails jpg --write-thumbnail', - "default": True, - }, -] - - -@dataclass(kw_only=True) -class Preset: - id: str = field(default_factory=lambda: str(uuid.uuid4())) - """The id of the preset.""" - - name: str - """The name of the preset.""" - - description: str = "" - """The description of the preset.""" - - folder: str = "" - """The default download folder to use if non is given.""" - - template: str = "" - """The default template to use if non is given.""" - - cookies: str = "" - """The default cookies to use if non is given.""" - - cli: str = "" - """command options for yt-dlp.""" - - default: bool = False - """If True, the preset is a default preset.""" - - priority: int = 0 - """Priority of the preset.""" - - _cookies_file: Path | None = field(init=False, default=None) - """The path to the cookies file.""" - - def serialize(self) -> dict: - return {k: v for k, v in self.__dict__.items() if not k.startswith("_")} - - def json(self) -> str: - from .encoder import Encoder - - return Encoder().encode(self.serialize()) - - def get(self, key: str, default: Any = None) -> Any: - return self.__dict__.get(key, default) - - def get_cookies_file(self, config: Config | None = None) -> Path | None: - """ - Get the path to the cookies file. - - Args: - config (Config|None): The config instance. - - Returns: - Path|None: The path to the cookies file. - - """ - if self._cookies_file: - return self._cookies_file - - if not self.cookies or not self.id: - return None - - if not config: - config = Config.get_instance() - - self._cookies_file = create_cookies_file(self.cookies, Path(config.config_path) / "cookies" / f"{self.id}.txt") - - return self._cookies_file - - -class Presets(metaclass=Singleton): - """ - This class is used to manage the presets. - """ - - def __init__(self, file: str | Path | None = None, config: Config | None = None): - self._items: list[Preset] = [] - "The list of presets." - - self._config: Config = None - "The config instance." - - self._default: list[Preset] = [] - "The list of default presets." - - self._config = config or Config.get_instance() - - self._file: Path = Path(file) if file else Path(self._config.config_path).joinpath("presets.json") - "The path to the presets file." - - if self._file.exists() and "600" != self._file.stat().st_mode: - try: - self._file.chmod(0o600) - except Exception: - pass - - for i, preset in enumerate(DEFAULT_PRESETS): - try: - self.validate(preset) - self._default.append(init_class(Preset, preset)) - except Exception as e: - LOG.error(f"Failed to parse default preset ':{i}'. '{e!s}'.") - continue - - @staticmethod - def get_instance(file: str | Path | None = None, config: Config | None = None) -> "Presets": - """ - Get the instance of the class. - - Args: - file (str|Path|None): The path to the presets file. - config (Config|None): The config instance. - - Returns: - Presets: The instance of the class - - """ - return Presets(file=file, config=config) - - async def on_shutdown(self, _: web.Application): - pass - - def attach(self, _: web.Application): - """ - Attach the class to the aiohttp application. - - Args: - _ (web.Application): The aiohttp application. - - Returns: - None - - """ - self.load() - - if not self.get(self._config.default_preset): - LOG.error(f"Default preset '{self._config.default_preset}' not found, using 'default' preset.") - self._config.default_preset = "default" - - async def event_handler(_, __): - msg = "Not implemented" - raise Exception(msg) - - EventBus.get_instance().subscribe(Events.PRESETS_ADD, event_handler, f"{__class__.__name__}.add") - - def get_all(self) -> list[Preset]: - """Return the items.""" - return sorted(self._default + self._items, key=lambda x: x.priority, reverse=True) - - def load(self) -> "Presets": - """ - Load the items. - - Returns: - Presets: The current instance. - - """ - has: int = len(self._items) - self.clear() - - if not self._file.exists() or self._file.stat().st_size < 10: - return self - - try: - LOG.info(f"{'Reloading' if has else 'Loading'} '{self._file}'.") - presets: dict = json.loads(self._file.read_text()) - except Exception as e: - LOG.error(f"Failed to parse '{self._file}'. '{e}'.") - return self - - if not presets or len(presets) < 1: - return self - - need_save = False - - for i, preset in enumerate(presets): - try: - if "priority" not in preset: - preset["priority"] = 0 - need_save = True - - if "id" not in preset: - preset["id"] = str(uuid.uuid4()) - need_save = True - - if preset.get("format"): - if not preset.get("cli"): - preset.update({"cli": f"--format {preset['format']}"}) - else: - preset["cli"] = f"--format '{preset['format']}'\n" + preset["cli"] - - preset["cli"] = str(preset["cli"]).strip() - - preset.pop("format") - need_save = True - - preset: Preset = init_class(Preset, preset) - - self._items.append(preset) - except Exception as e: - LOG.error(f"Failed to parse '{self._file}:{i}'. '{e!s}'.") - continue - - if need_save: - LOG.warning("Saving presets due to schema changes.") - self.save(self._items) - - return self - - def clear(self) -> "Presets": - """ - Clear all items. - - Returns: - Presets: The current instance. - - """ - if len(self._items) < 1: - return self - - self._items.clear() - - return self - - def validate(self, item: Preset | dict) -> bool: - """ - Validate the item. - - Args: - item (Preset|dict): The item to validate. - - Returns: - bool: True if valid - - Raises: - ValueError: If the item is not valid. - - """ - if not isinstance(item, dict): - if not isinstance(item, Preset): - msg = f"Unexpected '{type(item).__name__}' type was given." - raise ValueError(msg) - - item = item.serialize() - - if not item.get("id"): - msg = "No id found." - raise ValueError(msg) - - if not item.get("name"): - msg = "No name found." - raise ValueError(msg) - - if item.get("cli"): - try: - arg_converter(args=item.get("cli")) - except Exception as e: - msg = f"Invalid command options for yt-dlp. '{e!s}'." - raise ValueError(msg) from e - - if item.get("priority") is not None: - priority = item.get("priority") - if not isinstance(priority, int): - msg = "Priority must be an integer." - raise ValueError(msg) - if priority < 0: - msg = "Priority must be >= 0." - raise ValueError(msg) - - return True - - def save(self, items: list[Preset | dict]) -> "Presets": - """ - Save the items. - - Args: - items (list[Preset]): The items to save. - - Returns: - Presets: The current instance. - - """ - for i, preset in enumerate(items): - try: - if not isinstance(preset, Preset): - preset: Preset = init_class(Preset, preset) - items[i] = preset - except Exception as e: - LOG.error(f"Failed to save item '{i}' due to parsing error. '{e!s}'.") - continue - - try: - self.validate(preset) - except ValueError as e: - LOG.error(f"Failed to validate item '{i}: {preset.name}'. '{e}'.") - continue - - try: - self._file.write_text( - json.dumps(obj=[preset.serialize() for preset in items if preset.default is False], indent=4) - ) - - LOG.info(f"Saved '{self._file}'.") - except Exception as e: - LOG.error(f"Failed to save '{self._file}'. '{e!s}'.") - - return self - - def has(self, id_or_name: str) -> bool: - """ - Check if the item exists by id or name. - - Args: - id_or_name (str): The id or name of the item. - - Returns: - bool: True if exists, False otherwise. - - """ - return self.get(id_or_name) is not None - - def get(self, id_or_name: str) -> Preset | None: - """ - Get the item by id or name. - - Args: - id_or_name (str): The id or name of the item. - - Returns: - Preset|None: The item if found, None otherwise. - - """ - if not id_or_name: - return None - - for preset in self.get_all(): - if id_or_name not in (preset.id, preset.name): - continue - - return preset - - return None diff --git a/app/library/Scheduler.py b/app/library/Scheduler.py index b3230454..7d09b1ef 100644 --- a/app/library/Scheduler.py +++ b/app/library/Scheduler.py @@ -19,7 +19,7 @@ class Scheduler(metaclass=Singleton): self._jobs: dict[str, Cron] = {} "The scheduled jobs." - self._loop = loop or asyncio.get_event_loop() + self._loop: asyncio.AbstractEventLoop | None = loop "The event loop to use." @staticmethod @@ -114,6 +114,13 @@ class Scheduler(metaclass=Singleton): if id and id in self._jobs: self.remove(id) + if not self._loop: + try: + self._loop = asyncio.get_running_loop() + except RuntimeError: + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + job = Cron(spec=timer, func=func, args=args, kwargs=kwargs, uuid=id, start=True, loop=self._loop) job_id = str(job.uuid) @@ -126,7 +133,7 @@ class Scheduler(metaclass=Singleton): def remove(self, id: str | list[str]) -> bool: """ - Remove a job from the schedule. + Remove a job from the scheduler. Args: id (str|list[str]): The id of the job to remove. @@ -149,7 +156,7 @@ class Scheduler(metaclass=Singleton): return False del self._jobs[id] - LOG.debug(f"Removed job '{id}' from the schedule.") + LOG.debug(f"Removed job '{id}' from the scheduler.") return True return False diff --git a/app/library/Services.py b/app/library/Services.py index 894c66aa..eaecd529 100644 --- a/app/library/Services.py +++ b/app/library/Services.py @@ -1,6 +1,7 @@ import inspect import logging -from typing import Any, TypeVar +from dataclasses import dataclass +from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints from app.library.Singleton import Singleton @@ -8,59 +9,170 @@ T = TypeVar("T") LOG: logging.Logger = logging.getLogger(__name__) +def _unwrap_annotation(ann: Any) -> Any: + if ann is inspect._empty: + return inspect._empty + + origin = get_origin(ann) + + # Annotated[T, ...] -> T + if origin is Annotated: + args = get_args(ann) + return args[0] if args else ann + + # Optional[T] / Union[T, None] / T | None -> T + if origin is None: + return ann + + if str(origin) in ("typing.Union", "types.UnionType"): + args = [a for a in get_args(ann) if a is not type(None)] + if len(args) == 1: + return args[0] + return ann + + return ann + + +@dataclass(frozen=True) +class ServiceEntry: + name: str + declared_type: type | None + instance: Any + + class Services(metaclass=Singleton): def __init__(self): - self._services: dict[str, T] = {} + self._services: list[ServiceEntry] = [] @staticmethod def get_instance() -> "Services": return Services() - def add(self, name: str, service: T): - self._services[name] = service + def add(self, name: str, service: Any, declared_type: type | None = None) -> "Services": + """ + Add a service by name. - def add_all(self, services: dict[str, T]): - for name, service in services.items(): - self.add(name, service) + Args: + name: The name of the service. + service: The service instance. + declared_type: The declared type of the service (optional). - def get(self, name: str) -> T | None: - return self._services.get(name) + Returns: + Services: The Services instance (for chaining). - def has(self, name: str) -> bool: - return name in self._services + """ + if declared_type is None and service is not None: + declared_type = type(service) + + self.remove(name) + self._services.append(ServiceEntry(name=name, declared_type=declared_type, instance=service)) + return self + + def add_all(self, services: dict[str, Any]): + for name, svc in services.items(): + self.add(name, svc) def remove(self, name: str): - if name not in self._services: - return - - self._services.pop(name, None) + self._services = [e for e in self._services if e.name != name] def clear(self): self._services.clear() - def get_all(self) -> dict[str, T]: + def get(self, name: str) -> Any | None: + for e in reversed(self._services): + if e.name == name: + return e.instance + return None + + def has(self, name: str) -> bool: + return any(e.name == name for e in self._services) + + def get_all(self) -> list[ServiceEntry]: return self._services.copy() + def get_by_type(self, expected_type: Any) -> Any | None: + expected_type = _unwrap_annotation(expected_type) + if expected_type is inspect._empty or not isinstance(expected_type, type): + return None + + exact: list[Any] = [e.instance for e in self._services if e.declared_type is expected_type] + if len(exact) == 1: + return exact[0] + if len(exact) > 1: + msg: str = ( + f"Ambiguous dependency for type {expected_type.__name__}: {len(exact)} exact matches. Resolve by name." + ) + raise LookupError(msg) + + candidates: list[Any] = [e.instance for e in self._services if isinstance(e.instance, expected_type)] + if len(candidates) == 0: + return None + + if len(candidates) > 1: + msg: str = ( + f"Ambiguous dependency for type {expected_type.__name__}: " + f"{len(candidates)} candidates. Resolve by name." + ) + raise LookupError(msg) + + return candidates[0] + + def _build_call_args(self, handler: callable, overrides: dict[str, Any]) -> dict[str, Any]: + sig: inspect.Signature = inspect.signature(handler) + + try: + type_hints: dict[str, Any] = get_type_hints(handler) + except Exception: + type_hints = {} + + resolved: dict[str, Any] = {} + + for name, param in sig.parameters.items(): + if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD): + continue + + if name in overrides: + resolved[name] = overrides[name] + continue + + by_name: Any | None = self.get(name) + if by_name is not None: + resolved[name] = by_name + continue + + ann: Any | None = type_hints.get(name, param.annotation) + by_type: Any | None = self.get_by_type(ann) + if by_type is not None: + resolved[name] = by_type + continue + + if param.default is not inspect._empty: + continue + + missing_required: list[str] = [] + for name, param in sig.parameters.items(): + if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD): + continue + + if param.default is not inspect._empty: + continue + + if name not in resolved: + missing_required.append(name) + + if missing_required: + LOG.error( + "Missing arguments for handler '%s': %s", + getattr(handler, "__name__", str(handler)), + missing_required, + ) + + return resolved + async def handle_async(self, handler: callable, **kwargs) -> Any: - context = {**self.get_all(), **kwargs} - - sig = inspect.signature(handler) - expected_args = sig.parameters.keys() - filtered = {k: v for k, v in context.items() if k in expected_args} - - if missing_args := expected_args - filtered.keys(): - LOG.error(f"Missing arguments for handler '{handler.__name__}': {missing_args}") - - return await handler(**filtered) + resolved: dict[str, Any] = self._build_call_args(handler, kwargs) + return await handler(**resolved) def handle_sync(self, handler: callable, **kwargs) -> Any: - context = {**self.get_all(), **kwargs} - - sig = inspect.signature(handler) - expected_args = sig.parameters.keys() - filtered = {k: v for k, v in context.items() if k in expected_args} - - if missing_args := expected_args - filtered.keys(): - LOG.error(f"Missing arguments for handler '{handler.__name__}': {missing_args}") - - return handler(**filtered) + resolved: dict[str, Any] = self._build_call_args(handler, kwargs) + return handler(**resolved) diff --git a/app/library/TaskDefinitions.py b/app/library/TaskDefinitions.py deleted file mode 100644 index 189aa347..00000000 --- a/app/library/TaskDefinitions.py +++ /dev/null @@ -1,435 +0,0 @@ -import json -import logging -import uuid -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from aiohttp import web -from jsonschema import Draft7Validator, SchemaError, ValidationError - -from .config import Config -from .encoder import Encoder -from .Services import Services -from .Singleton import Singleton - -LOG: logging.Logger = logging.getLogger("task_definitions") - - -@dataclass(slots=True) -class TaskDefinitionRecord: - identifier: str - """UUID identifier of the task definition.""" - filename: str - """Filename of the task definition JSON file.""" - name: str - """Human-readable name of the task definition.""" - priority: int - """Priority of the task definition.""" - path: Path - """Path to the task definition JSON file.""" - data: dict[str, Any] - """The task definition data.""" - updated_at: float - """Last modified timestamp of the task definition file.""" - - def serialize(self, *, include_definition: bool = False) -> dict[str, Any]: - """ - Serialize the task definition record to a dictionary. - - Args: - include_definition (bool): Whether to include the full task definition data. - - Returns: - dict[str, Any]: The serialized task definition record. - - """ - payload: dict[str, Any] = { - "id": self.identifier, - "name": self.name, - "priority": self.priority, - "updated_at": self.updated_at, - } - - if include_definition: - payload["definition"] = self.data - - return payload - - def json(self, *, include_definition: bool = False) -> str: - """ - Serialize the task definition record to a JSON string. - - Args: - include_definition (bool): Whether to include the full task definition data. - - Returns: - str: The JSON string representation of the task definition record. - - """ - return Encoder().encode(self.serialize(include_definition=include_definition)) - - -class TaskDefinitions(metaclass=Singleton): - def __init__( - self, - directory: str | Path | None = None, - config: Config | None = None, - validator: Draft7Validator | None = None, - ): - self._config: Config = config or Config.get_instance() - "Instance of Config to use." - self._directory: Path = Path(directory) if directory else Path(self._config.config_path) / "tasks" - "Directory where task definition files are stored." - self._validator: Draft7Validator | None = validator - "JSON schema validator instance." - self._items: dict[str, TaskDefinitionRecord] = {} - "Mapping of task definition ID to TaskDefinitionRecord." - self._schema: Path = Path(self._config.app_path) / "schema" / "task_definition.json" - "Path to the JSON schema file for task definitions." - - try: - self._directory.mkdir(parents=True, exist_ok=True) - except Exception as e: - LOG.error(f"Failed to create tasks directory '{self._directory}': {e}") - - self.load() - - @staticmethod - def get_instance( - directory: str | Path | None = None, - config: Config | None = None, - validator: Draft7Validator | None = None, - ) -> "TaskDefinitions": - """ - Get the singleton instance of TaskDefinitions. - - Args: - directory (str | Path | None): Optional directory to store task definitions. - config (Config | None): Optional Config instance to use. - validator (Draft7Validator | None): Optional JSON schema validator to use. - - Returns: - TaskDefinitions: The singleton instance of TaskDefinitions. - - """ - return TaskDefinitions(directory=directory, config=config, validator=validator) - - def attach(self, _: web.Application) -> None: - """ - Attach the TaskDefinitions service to the application. - - Args: - _ (web.Application): The aiohttp web application instance. - - """ - Services.get_instance().add("task_definitions", self) - - async def on_shutdown(self, _: web.Application) -> None: - """ - Handle application shutdown event. - - Args: - _ (web.Application): The aiohttp web application instance. - - """ - return - - def _get_validator(self) -> Draft7Validator: - """ - Get or create the JSON schema validator for task definitions. - - Returns: - Draft7Validator: The JSON schema validator instance. - - """ - if self._validator: - return self._validator - - try: - contents: str = self._schema.read_text(encoding="utf-8") - schema = json.loads(contents) - except Exception as e: - LOG.error(f"Failed to read task definition schema '{self._schema}': {e}") - raise - - try: - self._validator = Draft7Validator(schema) - except SchemaError as e: - LOG.error(f"Invalid task definition schema '{self._schema}': {e}") - raise - - return self._validator - - def validate(self, definition: dict[str, Any]) -> None: - """ - Validate a task definition against the JSON schema. - - Args: - definition (dict[str, Any]): The task definition to validate. - - Raises: - ValueError: If the task definition is invalid. - - """ - try: - self._get_validator().validate(definition) - except ValidationError as e: - path: str = " ".join(str(part) for part in e.path) - error_path: str = f" ({path})" if path else "" - message: str = f"Task definition validation failed{error_path}: {e.message}" - raise ValueError(message) from e - - def load(self) -> "TaskDefinitions": - """ - Load all task definitions from the directory. - - Returns: - TaskDefinitions: The current instance for chaining. - - """ - self._items.clear() - - if not self._directory.exists(): - return self - - for file_path in sorted(self._directory.glob("*.json")): - stem: str = file_path.stem - - try: - identifier_uuid = uuid.UUID(stem) - except ValueError: - LOG.warning(f"Skipping task definition with invalid UUID filename '{file_path.name}'.") - continue - - if 4 != identifier_uuid.version: - LOG.warning(f"Skipping task definition '{file_path.name}', Name is not UUIDv4.") - continue - - try: - contents: str = file_path.read_text(encoding="utf-8") - except Exception as e: - LOG.error(f"Failed to load task definition '{file_path}': {e!s}") - continue - - try: - parsed = json.loads(contents) - except Exception as e: - LOG.error(f"Failed to parse task definition '{file_path}': {e!s}") - continue - - if not isinstance(parsed, dict): - LOG.error(f"Invalid task definition file '{file_path}': must be a JSON object.") - continue - - data: dict[str, Any] = parsed - - identifier: str = str(identifier_uuid) - - name_value: str = str(data.get("name") or identifier) - priority: int = self._normalize_priority(data.get("priority", 0)) - data["priority"] = priority - - record = TaskDefinitionRecord( - identifier=identifier, - filename=file_path.name, - name=name_value, - priority=priority, - path=file_path, - data=data, - updated_at=file_path.stat().st_mtime, - ) - self._items[record.identifier] = record - - return self - - def list(self) -> list[TaskDefinitionRecord]: - """ - List all task definitions, sorted by priority and name. - - Returns: - list[TaskDefinitionRecord]: List of task definitions sorted by priority and name. - - """ - return sorted( - self._items.values(), - key=lambda record: (record.priority, record.name.lower()), - ) - - def get(self, identifier: str) -> TaskDefinitionRecord | None: - """ - Get a task definition by its identifier. - - Args: - identifier (str): The UUID identifier of the task definition. - - Returns: - TaskDefinitionRecord | None: The task definition record, or None if not found. - - """ - return self._items.get(identifier) - - def _path_for(self, identifier: str) -> Path: - """ - Get the file path for a given task definition identifier. - - Args: - identifier (str): The UUID identifier of the task definition. - - Returns: - Path: The file path for the task definition JSON file. - - """ - return self._directory / f"{identifier}.json" - - def _write_file(self, path: Path, payload: dict[str, Any]) -> None: - """ - Write a task definition to a file. - - Args: - path (Path): The file path to write to. - payload (dict[str, Any]): The task definition data to write. - - Raises: - Exception: If writing to the file fails. - - """ - path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") - try: - path.chmod(0o600) - except Exception: - pass - - def _normalize_priority(self, value: Any) -> int: - """ - Normalize the priority value to an integer. - - Args: - value (Any): The priority value to normalize. - - Returns: - int: The normalized priority value. - - """ - try: - priority = int(value) - except Exception: - priority = 0 - - return priority - - def _refresh_generic_handler(self) -> None: - """ - Refresh the generic task handler definitions. - """ - try: - from .task_handlers.generic import GenericTaskHandler - - GenericTaskHandler.refresh_definitions(force=True) - except Exception as e: - LOG.error(f"Failed to refresh generic task handler: {e}") - - def create(self, definition: dict[str, Any]) -> TaskDefinitionRecord: - """ - Create a new task definition. - - Args: - definition (dict[str, Any]): The task definition data. - - Returns: - TaskDefinitionRecord: The created task definition record. - - """ - self.validate(definition) - - identifier: str = str(uuid.uuid4()) - path: Path = self._path_for(identifier) - - while path.exists(): - identifier = str(uuid.uuid4()) - path = self._path_for(identifier) - - priority: int = self._normalize_priority(definition.get("priority", 0)) - definition["priority"] = priority - - self._write_file(path, definition) - - record = TaskDefinitionRecord( - identifier=identifier, - filename=path.name, - name=str(definition.get("name", identifier)), - priority=priority, - path=path, - data=definition, - updated_at=path.stat().st_mtime, - ) - - self._items[identifier] = record - self._refresh_generic_handler() - return record - - def update(self, identifier: str, definition: dict[str, Any]) -> TaskDefinitionRecord: - """ - Update an existing task definition. - - Args: - identifier (str): The UUID identifier of the task definition to update. - definition (dict[str, Any]): The updated task definition data. - - Returns: - TaskDefinitionRecord: The updated task definition record. - - Raises: - ValueError: If the task definition does not exist or is invalid. - - """ - record: TaskDefinitionRecord | None = self.get(identifier) - if not record: - message: str = f"Task definition '{identifier}' does not exist." - raise ValueError(message) - - self.validate(definition) - priority: int = self._normalize_priority(definition.get("priority", record.priority)) - definition["priority"] = priority - self._write_file(record.path, definition) - - updated_record = TaskDefinitionRecord( - identifier=identifier, - filename=record.filename, - name=str(definition.get("name", identifier)), - priority=priority, - path=record.path, - data=definition, - updated_at=record.path.stat().st_mtime, - ) - - self._items[identifier] = updated_record - self._refresh_generic_handler() - return updated_record - - def delete(self, identifier: str) -> None: - """ - Delete a task definition. - - Args: - identifier (str): The UUID identifier of the task definition to delete. - - Raises: - ValueError: If the task definition does not exist. - - """ - record: TaskDefinitionRecord | None = self.get(identifier) - if not record: - message: str = f"Task definition '{identifier}' does not exist." - raise ValueError(message) - - try: - record.path.unlink(missing_ok=False) - except FileNotFoundError: - LOG.warning(f"Task definition file '{record.path}' already removed.") - except Exception as exc: - LOG.error(f"Failed to delete task definition '{identifier}': {exc}") - raise - - self._items.pop(identifier, None) - self._refresh_generic_handler() diff --git a/app/library/Tasks.py b/app/library/Tasks.py deleted file mode 100644 index f1a78dcc..00000000 --- a/app/library/Tasks.py +++ /dev/null @@ -1,1067 +0,0 @@ -import asyncio -import inspect -import json -import logging -import pkgutil -import random -import time -import uuid -from dataclasses import asdict, dataclass, field -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from aiohttp import web - -from .config import Config -from .downloads import DownloadQueue -from .encoder import Encoder -from .Events import EventBus, Events -from .ItemDTO import Item, ItemDTO -from .Scheduler import Scheduler -from .Services import Services -from .Singleton import Singleton -from .Utils import archive_add, archive_delete, archive_read, extract_info, init_class, validate_url -from .YTDLPOpts import YTDLPOpts - -LOG: logging.Logger = logging.getLogger("tasks") - - -@dataclass(kw_only=True) -class Task: - id: str - name: str - url: str - folder: str = "" - preset: str = "" - timer: str = "" - template: str = "" - cli: str = "" - auto_start: bool = True - handler_enabled: bool = True - enabled: bool = True - - def serialize(self) -> dict: - return self.__dict__ - - def json(self) -> str: - return Encoder().encode(self.serialize()) - - def get(self, key: str, default: Any = None) -> Any: - """ - Get a value from the task by key. - - Args: - key (str): The key to get. - default (Any): The default value if the key is not found. - - Returns: - Any: The value of the key or the default value. - - """ - return self.serialize().get(key, default) - - def get_ytdlp_opts(self) -> YTDLPOpts: - """ - Get the yt-dlp options for the task. - - Returns: - YTDLPOpts: The yt-dlp options. - - """ - params: YTDLPOpts = YTDLPOpts.get_instance() - - if self.preset: - params = params.preset(name=self.preset) - - if self.cli: - params = params.add_cli(self.cli, from_user=True) - - if self.template: - params = params.add({"outtmpl": {"default": self.template}}, from_user=False) - - return params - - def mark(self) -> tuple[bool, str]: - """ - Mark the task's items as downloaded in the archive file. - - Returns: - tuple[bool, str]: A tuple indicating success and a message. - - """ - ret = self._mark_logic() - if isinstance(ret, tuple): - return ret - - archive_file: Path = ret.get("file") - items: set[str] = ret.get("items", set()) - - if len(items) < 1 or not archive_add(archive_file, list(items)): - return (True, "No new items to mark as downloaded.") - - return (True, f"Task '{self.name}' items marked as downloaded.") - - def unmark(self) -> tuple[bool, str]: - """ - Unmark the task's items from the archive file. - - Returns: - tuple[bool, str]: A tuple indicating success and a message. - - """ - ret: tuple[bool, str] | set[tuple[Path, set[str]]] = self._mark_logic() - if isinstance(ret, tuple): - return ret - - archive_file: Path = ret.get("file") - items: set[str] = ret.get("items", set()) - - if len(items) < 1 or not archive_delete(archive_file, list(items)): - return (True, "No items to remove from archive file.") - - return (True, f"Removed '{self.name}' items from archive file.") - - def fetch_metadata(self, full: bool = False) -> tuple[dict[str, Any] | None, bool, str]: - """ - Fetch metadata for the task's URL. - - Args: - full (bool): Whether to fetch full metadata including all entries for playlists. - - Returns: - tuple[dict[str, Any]|None, bool, str]: A tuple containing the metadata (or None on failure), a boolean - indicating if the operation was successful, and a message. - - """ - if not self.url: - return ({}, False, "No URL found in task parameters.") - - params = self.get_ytdlp_opts() - if not full: - params.add_cli("-I0", from_user=False) - - params = params.get_all() - - ie_info: dict | None = extract_info( - params, self.url, no_archive=True, follow_redirect=False, sanitize_info=True - ) - if not ie_info or not isinstance(ie_info, dict): - return ({}, False, "Failed to extract information from URL.") - - return (ie_info, True, "") - - def _mark_logic(self) -> tuple[bool, str] | set[tuple[Path, set[str]]]: - if not self.url: - return (False, "No URL found in task parameters.") - - params: dict = self.get_ytdlp_opts().get_all() - if not (archive_file := params.get("download_archive")): - return (False, "No archive file found.") - - archive_file: Path = Path(archive_file) - - ie_info: dict | None = extract_info(params, self.url, no_archive=True, follow_redirect=True) - if not ie_info or not isinstance(ie_info, dict): - return (False, "Failed to extract information from URL.") - - if "playlist" != ie_info.get("_type"): - return (False, "Expected a playlist type from extract_info.") - - items: set[str] = set() - - def _process(item: dict): - for entry in item.get("entries", []): - if not isinstance(entry, dict): - continue - - if "playlist" == entry.get("_type"): - _process(entry) - continue - - if entry.get("_type") not in ("video", "url"): - continue - - if not entry.get("id") or not entry.get("ie_key"): - continue - - archive_id: str = f"{entry.get('ie_key', '').lower()} {entry.get('id')}" - - items.add(archive_id) - - _process(ie_info) - - return {"file": archive_file, "items": items} - - -def _split_inspect_metadata(metadata: dict[str, Any] | None) -> tuple[dict[str, Any], dict[str, Any]]: - """ - Split commonly consumed metadata keys from the rest. - - Args: - metadata (dict[str, Any]|None): The metadata to split. - - Returns: - tuple[dict[str, Any], dict[str, Any]]: The primary and extra metadata. - - """ - metadata = dict(metadata or {}) - primary: dict[str, Any] = {} - - for key in ("matched", "handler", "supported"): - if key in metadata: - primary[key] = metadata.pop(key) - - return primary, metadata - - -@dataclass(slots=True) -class TaskItem: - url: str - "The URL of the item." - title: str | None = None - "The title of the item." - archive_id: str | None = None - "The archive ID of the item." - metadata: dict[str, Any] = field(default_factory=dict) - "Additional metadata related to the item." - - -@dataclass(slots=True) -class TaskResult: - items: list[TaskItem] = field(default_factory=list) - "The list of items." - metadata: dict[str, Any] = field(default_factory=dict) - "Additional metadata related to the result." - - def serialize(self) -> dict[str, Any]: - """ - Serialize the task result. - - Returns: - dict[str, Any]: The serialized task result. - - """ - primary, extra = _split_inspect_metadata(self.metadata) - payload: dict[str, Any] = {**primary, "items": [asdict(item) for item in self.items]} - - if extra: - payload["metadata"] = extra - - return payload - - -@dataclass(slots=True) -class TaskFailure: - message: str - "A human-readable message describing the failure." - error: str | None = None - "An optional error code or string." - metadata: dict[str, Any] = field(default_factory=dict) - "Additional metadata related to the failure." - - def serialize(self) -> dict[str, Any]: - """ - Serialize the task failure. - - Returns: - dict[str, Any]: The serialized task failure. - - """ - primary, extra = _split_inspect_metadata(self.metadata) - payload: dict[str, Any] = dict(primary) - - if self.error: - payload["error"] = self.error - - if self.message and (not self.error or self.message != self.error): - payload["message"] = self.message - - if extra: - payload["metadata"] = extra - - return payload - - -class Tasks(metaclass=Singleton): - """ - This class is used to manage the tasks. - """ - - def __init__( - self, - file: str | None = None, - loop: asyncio.AbstractEventLoop | None = None, - config: Config | None = None, - encoder: Encoder | None = None, - scheduler: Scheduler | None = None, - ): - self._tasks: list[Task] = [] - "The tasks." - config = config or Config.get_instance() - - self._debug: bool = config.debug - "Debug mode." - self._default_preset: str = config.default_preset - "The default preset." - self._file: Path = Path(file) if file else Path(config.config_path).joinpath("tasks.json") - "The tasks file." - self._encoder: Encoder = encoder or Encoder() - "The JSON encoder." - self._loop: asyncio.AbstractEventLoop = loop or asyncio.get_event_loop() - "The event loop." - self._scheduler: Scheduler = scheduler or Scheduler.get_instance() - "The scheduler." - self._notify: EventBus = EventBus.get_instance() - "The event bus." - self._task_handler = HandleTask(self._scheduler, self, config) - "The task handler." - self._downloadQueue = DownloadQueue.get_instance() - "The download queue." - - if self._file.exists() and "600" != self._file.stat().st_mode: - try: - self._file.chmod(0o600) - except Exception: - pass - - @staticmethod - def get_instance( - file: str | None = None, - loop: asyncio.AbstractEventLoop | None = None, - config: Config | None = None, - encoder: Encoder | None = None, - scheduler: Scheduler | None = None, - ) -> "Tasks": - """ - Get the instance of the class. - - Returns: - Tasks: The instance of the class. - - """ - return Tasks(file=file, loop=loop, config=config, encoder=encoder, scheduler=scheduler) - - async def on_shutdown(self, _: web.Application): - self.clear(shutdown=True) - self._task_handler.on_shutdown(_) - - def attach(self, _: web.Application): - """ - Attach the tasks to the aiohttp application. - - Args: - _ (web.Application): The aiohttp application. - - """ - self.load() - Services.get_instance().add("tasks", self) - - async def event_handler(data, _): - if data and data.data: - self.save(data.data) - - self._notify.subscribe(Events.TASKS_ADD, event_handler, f"{__class__.__name__}.add") - self._task_handler.load() - - def get_all(self) -> list[Task]: - """Return the tasks.""" - return self._tasks - - def get_handler(self) -> "HandleTask": - """Expose the handle task helper.""" - return self._task_handler - - def get(self, task_id: str) -> Task | None: - """ - Get a task by its ID. - - Args: - task_id (str): The ID of the task. - - Returns: - Task | None: The task if found, otherwise None. - - """ - return next((task for task in self._tasks if task.id == task_id), None) - - def load(self) -> "Tasks": - """ - Load the tasks. - - Returns: - Tasks: The current instance. - - """ - has_tasks: bool = len(self._tasks) > 0 - self.clear() - - if not self._file.exists() or self._file.stat().st_size < 1: - return self - - try: - LOG.info(f"{'Reloading' if has_tasks else 'Loading'} '{self._file}'.") - tasks = json.loads(self._file.read_text()) - except Exception as e: - LOG.error(f"Error loading '{self._file}'. '{e!s}'.") - return self - - if not tasks or len(tasks) < 1: - return self - - needs_update: bool = False - for i, task in enumerate(tasks): - try: - Tasks.validate(task) - if "enabled" not in task: - task["enabled"] = True - needs_update = True - - task: Task = init_class(Task, task) - except Exception as e: - LOG.error(f"Failed to parse task at list position '{i}'. '{e!s}'.") - continue - - self._tasks.append(task) - - if not task.timer: - continue - - try: - self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=task.id) - - try: - from cronsim import CronSim - - cs = CronSim(task.timer, datetime.now(UTC)) - schedule_time: str = cs.explain() - except Exception: - schedule_time = task.timer - - if not has_tasks: - LOG.info(f"Task '{task.name}' queued to be executed '{schedule_time}'.") - except Exception as e: - LOG.exception(e) - LOG.error(f"Failed to queue '{i}: {task.name}'. '{e!s}'.") - - if needs_update: - self.save(self._tasks) - - return self - - def clear(self, shutdown: bool = False) -> "Tasks": - """ - Clear all tasks. - - Returns: - Tasks: The current instance. - - """ - if len(self._tasks) < 1: - return self - - for task in self._tasks: - if not self._scheduler.has(task.id): - continue - - try: - LOG.debug(f"Stopping '{task.name}'.") - self._scheduler.remove(task.id) - except Exception as e: - if not shutdown: - LOG.exception(e) - LOG.error(f"Failed to stop '{task.name}'. '{e!s}'.") - - self._tasks.clear() - - return self - - @staticmethod - def validate(task: Task | dict) -> bool: - """ - Validate the task. - - Args: - task (Task|dict): The task to validate. - - Returns: - bool: True if the task is valid, False otherwise. - - """ - if not isinstance(task, dict): - if not isinstance(task, Task): - msg = "Invalid task type." - raise ValueError(msg) - - task = task.serialize() - - if not task.get("name"): - msg = "No name found." - raise ValueError(msg) - - task["name"] = task["name"].strip() - - if not task.get("url"): - msg = "No URL found." - raise ValueError(msg) - - task["url"] = task["url"].strip() - try: - validate_url(task["url"], allow_internal=True) - except ValueError as e: - msg = f"Invalid URL format. '{e!s}'." - raise ValueError(msg) from e - - if task.get("timer"): - try: - from cronsim import CronSim - - CronSim(task.get("timer"), datetime.now(UTC)) - task["timer"] = str(task["timer"]).strip() - except Exception as e: - msg = f"Invalid timer format. '{e!s}'." - raise ValueError(msg) from e - - if task.get("cli"): - try: - from .Utils import arg_converter - - arg_converter(args=task.get("cli")) - task["cli"] = str(task["cli"]).strip() - except Exception as e: - msg = f"Invalid command options for yt-dlp. '{e!s}'." - raise ValueError(msg) from e - - return True - - def save(self, tasks: list[Task | dict]) -> "Tasks": - """ - Save the tasks. - - Args: - tasks (list[Task]): The tasks to save. - - Returns: - Tasks: The current instance. - - """ - for i, task in enumerate(tasks): - try: - self.validate(task) - - if not isinstance(task, Task): - task: Task = init_class(Task, task) - tasks[i] = task - except ValueError as e: - LOG.error(f"Failed to validate item '{i}: {task.name}'. '{e}'.") - continue - except Exception as e: - LOG.error(f"Failed to save task '{i}'. '{e!s}'.") - continue - - try: - self._file.write_text(json.dumps([i.serialize() for i in tasks], indent=4)) - LOG.info(f"Updated '{self._file}'.") - except Exception as e: - LOG.error(f"Error saving '{self._file}'. '{e!s}'.") - - return self - - async def _runner(self, task: Task) -> None: - """ - Run the task. - - Args: - task (Task): The task to run. - - Returns: - None - - """ - timeNow: str = datetime.now(UTC).isoformat() - try: - if not self.get(task.id): - LOG.info(f"Task '{task.name}' no longer exists.") - if self._scheduler.has(task.id): - self._scheduler.remove(task.id) - return - - started: float = time.time() - - if not task.enabled: - LOG.debug(f"Task '{task.name}' is disabled. Skipping execution.") - return - - if not task.url: - LOG.error(f"Failed to dispatch '{task.name}'. No URL found.") - return - - preset: str = str(task.preset or self._default_preset) - folder: str = task.folder if task.folder else "" - template: str = task.template if task.template else "" - cli: str = task.cli if task.cli else "" - - status = await self._downloadQueue.add( - item=Item.format( - { - "url": task.url, - "preset": preset, - "folder": folder, - "template": template, - "cli": cli, - "auto_start": task.auto_start, - "extras": { - "source_name": task.name, - "source_id": task.id, - "source_handler": __class__.__name__, - }, - } - ) - ) - - timeNow = datetime.now(UTC).isoformat() - ended: float = time.time() - LOG.info(f"Task '{task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.") - self._notify.emit( - Events.TASK_DISPATCHED, - data={**status, "preset": task.preset} if status else {"preset": task.preset}, - title=f"Task '{task.name}' dispatched", - message=f"Task '{task.name}' dispatched at '{timeNow}'.", - ) - self._notify.emit( - Events.LOG_SUCCESS, - data={"preset": task.preset, "lowPriority": True}, - title="Task completed", - message=f"Task '{task.name}' completed in '{ended - started:.2f}'.", - ) - except Exception as e: - LOG.error(f"Failed to execute '{task.name}' at '{timeNow}'. '{e!s}'.") - self._notify.emit( - Events.LOG_ERROR, - data={"preset": task.preset}, - title="Task failed", - message=f"Failed to execute '{task.name}'. '{e!s}'", - ) - - -class HandleTask: - def __init__(self, scheduler: Scheduler, tasks: Tasks, config: Config) -> None: - self._handlers: list[type] = [] - "The available handlers." - self._tasks: Tasks = tasks - "The tasks manager." - self._scheduler: Scheduler = scheduler - "The scheduler." - self._config: Config = config - "The configuration." - self._task_name: str = f"{__class__.__name__}._dispatcher" - "The task name for the scheduler." - self._queued: dict[str, set[str]] = {} - "Queued archive IDs per handler." - self._failure_count: dict[str, dict[str, int]] = {} - "Failure counts per handler and archive ID." - - EventBus.get_instance().subscribe( - Events.ITEM_ERROR, - self._handle_item_error, - f"{__class__.__name__}.item_error", - ) - - def load(self) -> None: - """ - Load the available handlers and schedule the dispatcher. - """ - self._handlers: list[type] = self._discover() - - timer: str = self._config.tasks_handler_timer - try: - from cronsim import CronSim - - CronSim(timer, datetime.now(UTC)) - except Exception as e: - timer = "15 */1 * * *" - LOG.error(f"Invalid timer format. '{e!s}'. Defaulting to '{timer}'.") - - self._scheduler.add( - timer=timer, - func=self._dispatcher, - id=f"{__class__.__name__}._dispatcher", - ) - - def on_shutdown(self, _: web.Application) -> None: - """ - Handle shutdown event. - - Args: - _: web.Application: The aiohttp application. - - """ - if self._scheduler.has(self._task_name): - self._scheduler.remove(self._task_name) - - def _dispatcher(self): - s: dict[list[str]] = {"h": [], "d": [], "u": [], "f": []} - - handler_groups: dict[str, list[tuple[Task, type]]] = {} - - for task in self._tasks.get_all(): - if not task.enabled or not task.handler_enabled: - s["d"].append(task.name) - continue - - if not task.get_ytdlp_opts().get_all().get("download_archive"): - LOG.debug(f"Task '{task.name}' does not have an archive file configured.") - s["f"].append(task.name) - continue - - try: - handler = self._find_handler(task) - if handler is None: - s["u"].append(task.name) - continue - - handler_name = handler.__name__ - if handler_name not in handler_groups: - handler_groups[handler_name] = [] - handler_groups[handler_name].append((task, handler)) - s["h"].append(task.name) - except Exception as e: - LOG.error(f"Failed to handle task '{task.name}'. '{e!s}'.") - s["f"].append(task.name) - - for tasks_with_handlers in handler_groups.values(): - for idx, (task, handler) in enumerate(tasks_with_handlers): - try: - t = asyncio.create_task( - coro=self._dispatch( - task, - handler, - delay=0.0 if 0 == idx else random.uniform(1.0, self._config.task_handler_random_delay), - ), - name=f"task-{task.id}", - ) - t.add_done_callback(lambda fut, t=task: self._handle_exception(fut, t)) - except Exception as e: - LOG.error(f"Failed to dispatch task '{task.name}'. '{e!s}'.") - - if len(self._tasks.get_all()) > 0: - LOG.info( - f"Tasks handler summary: Handled: {len(s['h'])}, Unhandled: {len(s['u'])}, Disabled: {len(s['d'])}, Failed: {len(s['f'])}." - ) - - async def _dispatch(self, task: Task, handler: type, delay: float) -> TaskResult | TaskFailure | None: - """ - Dispatch a task after a random delay to avoid rate limiting. - - Args: - task (Task): The task to dispatch. - handler (type): The handler to use. - delay (float): The delay in seconds before dispatching. - - Returns: - TaskResult|TaskFailure|None: The dispatch result. - - """ - if delay > 0: - LOG.debug(f"Delaying dispatch of task '{task.name}' by {delay:.1f} seconds.") - await asyncio.sleep(delay) - return await self.dispatch(task, handler=handler) - - def _handle_exception(self, fut: asyncio.Task, task: Task) -> None: - if fut.cancelled(): - return - - if exc := fut.exception(): - LOG.error(f"Exception while handling task '{task.name}': {exc}") - - def _find_handler(self, task: Task) -> type | None: - for cls in self._handlers: - try: - if Services.get_instance().handle_sync(handler=cls.can_handle, task=task): - return cls - except Exception as e: - LOG.exception(e) - continue - - return None - - async def dispatch(self, task: Task, handler: type | None = None, **kwargs) -> TaskResult | TaskFailure | None: # noqa: ARG002 - """ - Dispatch a task to the appropriate handler. - - Args: - task (Task): The task to dispatch. - handler (type|None): Optional specific handler to use instead of finding one. - **kwargs: Additional context to pass to the handler. - - Returns: - TaskResult|TaskFailure|None: The extraction outcome, or None if no handler matched. - - """ - if not handler: - handler = self._find_handler(task) - if handler is None: - return None - - services: Services = Services.get_instance() - - try: - extraction: TaskResult | TaskFailure = await services.handle_async( - handler=handler.extract, task=task, config=self._config - ) - except NotImplementedError: - LOG.error(f"Handler '{handler.__name__}' does not implement extract().") - return TaskFailure(message="Handler does not support extraction.") - except Exception as exc: - LOG.exception(exc) - raise - - if isinstance(extraction, TaskFailure): - LOG.error(f"Handler '{handler.__name__}' failed to extract items: {extraction.message}") - return extraction - - if not isinstance(extraction, TaskResult): - LOG.error( - f"Handler '{handler.__name__}' returned unexpected result type '{type(extraction).__name__}'.", - ) - return TaskFailure( - message="Handler returned invalid result type.", metadata={"type": type(extraction).__name__} - ) - - raw_items: list[TaskItem] = extraction.items or [] - metadata: dict[str, Any] = extraction.metadata or {} - - handler_name: str = handler.__name__ - queued: set[str] = self._queued.setdefault(handler_name, set()) - failures: dict[str, int] = self._failure_count.setdefault(handler_name, {}) - - params: dict = task.get_ytdlp_opts().get_all() - archive_file: str | None = params.get("download_archive") - - download_queue: DownloadQueue = services.get("queue") or DownloadQueue.get_instance() - notify: EventBus = services.get("notify") or EventBus.get_instance() - - archive_ids: list[str] = [ - item.archive_id for item in raw_items if isinstance(item, TaskItem) and item.archive_id - ] - downloaded: list[str] = archive_read(archive_file, archive_ids) if archive_file else [] - - filtered: list[TaskItem] = [] - - for item in raw_items: - if not isinstance(item, TaskItem): - LOG.warning("Handler '{handler.__name__}' produced non-TaskItem entry: {item!r}") - continue - - url: str = item.url - if not url: - continue - - archive_id: str | None = item.archive_id - if not archive_id: - LOG.warning(f"'{task.name}': Item with URL '{url}' is missing an archive ID. Skipping.") - continue - - if archive_id in queued: - continue - - queued.add(archive_id) - - if archive_file and archive_id in downloaded: - continue - - if await download_queue.queue.exists(url=url): - continue - - try: - done = await download_queue.done.get(url=url) - if "error" != done.info.status: - continue - except KeyError: - pass - - if archive_id not in failures: - failures[archive_id] = 0 - - filtered.append(item) - - if not filtered: - if raw_items: - LOG.debug( - f"Handler '{handler.__name__}' produced '{len(raw_items)}' for '{task.name}' items, none queued after filtering." - ) - return TaskResult(items=[], metadata=metadata) - - LOG.info( - f"Handler '{handler.__name__}' Found '{len(filtered)}' new items for '{task.name}' (raw={len(raw_items)})." - ) - - base_item = Item.format( - { - "url": task.url, - "preset": task.preset or self._config.default_preset, - "folder": task.folder or "", - "template": task.template or "", - "cli": task.cli or "", - "auto_start": task.auto_start, - "extras": {"source_name": task.name, "source_id": task.id, "source_handler": handler.__name__}, - } - ) - - for item in filtered: - metadata_entry: dict[str, Any] = item.metadata if isinstance(item.metadata, dict) else {} - extras: dict[str, Any] = base_item.extras.copy() - if metadata_entry: - extras["metadata"] = metadata_entry - - notify.emit( - Events.ADD_URL, - data=base_item.new_with(url=item.url, extras=extras).serialize(), - ) - - return TaskResult(items=filtered, metadata=metadata) - - async def inspect( - self, - url: str, - preset: str | None = None, - handler_name: str | None = None, - static_only: bool = False, - ) -> TaskResult | TaskFailure: - if not self._handlers: - self._handlers = self._discover() - - task = Task( - id=str(uuid.uuid4()), - name="Inspector", - url=url, - preset=preset or self._config.default_preset, - auto_start=False, - ) - - services = Services.get_instance() - - handler_cls: type | None - if handler_name: - handler_cls = next((cls for cls in self._handlers if cls.__name__.lower() == handler_name.lower()), None) - if handler_cls is None: - message: str = f"Handler '{handler_name}' not found." - return TaskFailure( - message=message, - error=message, - metadata={"matched": False, "handler": handler_name}, - ) - - try: - matched = services.handle_sync(handler=handler_cls.can_handle, task=task) - except Exception as exc: # pragma: no cover - defensive - LOG.exception(exc) - message = str(exc) - return TaskFailure( - message=message, - error=message, - metadata={"matched": False, "handler": handler_cls.__name__}, - ) - - if not matched: - return TaskFailure( - message="Handler cannot process the supplied URL.", - metadata={"matched": False, "handler": handler_cls.__name__}, - ) - else: - handler_cls = self._find_handler(task) - if handler_cls is None: - message = "No handler matched the supplied URL." - return TaskFailure( - message=message, - error=message, - metadata={"matched": False, "handler": None}, - ) - - base_metadata: dict[str, Any] = {"matched": True, "handler": handler_cls.__name__} - - if static_only: - return TaskResult(items=[], metadata=base_metadata) - - try: - extraction: TaskResult | TaskFailure = await services.handle_async( - handler=handler_cls.extract, task=task, config=self._config - ) - except NotImplementedError: - return TaskFailure( - message="Handler does not support manual inspection.", - metadata={**base_metadata, "supported": False}, - ) - except Exception as exc: - LOG.exception(exc) - message = str(exc) - return TaskFailure( - message=message, - error=message, - metadata={**base_metadata, "supported": True}, - ) - - if isinstance(extraction, TaskFailure): - combined_failure_metadata: dict[str, Any] = {**base_metadata, "supported": True} - if extraction.metadata: - combined_failure_metadata.update(extraction.metadata) - - return TaskFailure( - message=extraction.message, - error=extraction.error if extraction.error else extraction.message, - metadata=combined_failure_metadata, - ) - - if not isinstance(extraction, TaskResult): - LOG.error( - f"Handler '{handler_cls.__name__}' returned unexpected result type '{type(extraction).__name__}' during inspection.", - ) - extraction = TaskResult() - - combined_metadata: dict[str, Any] = {**base_metadata, "supported": True} - if extraction.metadata: - combined_metadata.update(extraction.metadata) - - return TaskResult(items=list(extraction.items), metadata=combined_metadata) - - def _discover(self) -> list[type]: - import importlib - - import app.library.task_handlers as handlers_pkg - - handlers: list[type] = [] - - for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__): - if module_name.startswith("_"): - continue - - module = importlib.import_module(f"{handlers_pkg.__name__}.{module_name}") - for _, cls in inspect.getmembers(module, inspect.isclass): - if cls.__module__ != module.__name__: - continue - - if callable(getattr(cls, "can_handle", None)) and callable(getattr(cls, "extract", None)): - handlers.append(cls) - - return handlers - - async def _handle_item_error(self, event, _name, **_kwargs): - item: ItemDTO | None = getattr(event, "data", None) - if not isinstance(item, ItemDTO): - return - - extras: dict[Any, Any] = getattr(item, "extras", {}) or {} - handler_name: Any | None = extras.get("source_handler") - if not handler_name: - return - - archive_id: str | None = item.archive_id - if not archive_id: - return - - queued: set[str] | None = self._queued.get(handler_name) - if queued: - queued.discard(archive_id) - - failures: dict[str, int] = self._failure_count.setdefault(handler_name, {}) - failures[archive_id] = failures.get(archive_id, 0) + 1 diff --git a/app/library/UpdateChecker.py b/app/library/UpdateChecker.py index a6c939e7..72d3cdac 100644 --- a/app/library/UpdateChecker.py +++ b/app/library/UpdateChecker.py @@ -1,7 +1,7 @@ import asyncio import logging import re -from typing import TYPE_CHECKING +from typing import Any from aiohttp import web @@ -13,9 +13,6 @@ from .Scheduler import Scheduler from .Singleton import Singleton from .version import APP_VERSION -if TYPE_CHECKING: - from app.library.dl_fields import Any - LOG: logging.Logger = logging.getLogger("update_checker") @@ -27,12 +24,18 @@ class UpdateChecker(metaclass=Singleton): GITHUB_API_URL: str = "https://api.github.com/repos/arabcoders/ytptube/releases/latest" "GitHub API endpoint for latest release" + YTDLP_API_URL: str = "https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest" + "GitHub API endpoint for yt-dlp latest release" + CACHE_DURATION: int = 300 "Cache duration in seconds (5 minutes)" CACHE_KEY: str = "update_checker:result" "Cache key for storing check results" + YTDLP_CACHE_KEY: str = "update_checker:ytdlp" + "Cache key for storing yt-dlp check results" + def __init__( self, config: Config | None = None, scheduler: Scheduler | None = None, notify: EventBus | None = None ) -> None: @@ -55,28 +58,9 @@ class UpdateChecker(metaclass=Singleton): def get_instance( config: Config | None = None, scheduler: Scheduler | None = None, notify: EventBus | None = None ) -> "UpdateChecker": - """ - Get the singleton instance of UpdateChecker. - - Args: - config (Config | None): Optional Config instance to use. - scheduler (Scheduler | None): Optional Scheduler instance to use. - notify (EventBus | None): Optional EventBus instance to use. - - Returns: - UpdateChecker: The singleton instance of UpdateChecker. - - """ return UpdateChecker(config=config, scheduler=scheduler, notify=notify) def attach(self, _: web.Application) -> None: - """ - Attach the UpdateChecker to the application. - - Args: - _ (web.Application): The aiohttp web application instance. - - """ from .Services import Services Services.get_instance().add("update_checker", self) @@ -100,7 +84,7 @@ class UpdateChecker(metaclass=Singleton): self._schedule_check() - async def on_shutdown(self, _: web.Application) -> None: + async def on_shutdown(self, _: web.Application | None) -> None: """ Handle application shutdown event. @@ -113,89 +97,129 @@ class UpdateChecker(metaclass=Singleton): self._scheduler.remove(self._job_id) self._job_id = None - LOG.debug("Stopped update check scheduled task.") def _schedule_check(self) -> None: - """Schedule the update check task to run daily at 3 AM.""" if not self._config.check_for_updates: - LOG.debug("Update checking is disabled, skipping scheduling.") return - # Run daily at 3 AM timer: str = "0 3 * * *" self._job_id = self._scheduler.add( timer=timer, func=lambda: asyncio.create_task(self.check_for_updates()), - id="update_checker", + id=f"{__class__.__name__}.{self.check_for_updates.__name__}", ) - LOG.info(f"Scheduled update check to run daily at 3 AM (cron: {timer}).") - - async def check_for_updates(self) -> tuple[str, str | None]: + async def check_for_updates(self) -> tuple[tuple[str, str | None], tuple[str, str | None]]: """ - Check for updates from GitHub releases. - Updates config.new_version if a newer version is available. - Stops the scheduled task if an update is found. + Check for updates from GitHub releases for both the app and yt-dlp. + Updates config.new_version and config.yt_new_version if newer versions are available. + Stops the scheduled task if an app update is found. Returns: - tuple[str, str | None]: (status, new_version) + tuple[tuple[str, str | None], tuple[str, str | None]]: ((app_status, app_version), (ytdlp_status, ytdlp_version)) status: "disabled", "error", "up_to_date", or "update_available" - new_version: The new version tag if available, None otherwise + version: The new version tag if available, None otherwise """ if not self._config.check_for_updates: - LOG.debug("Update checking is disabled, skipping check.") - return ("disabled", None) + return (("disabled", None), ("disabled", None)) - # Check cache - cached = await self._cache.aget(self.CACHE_KEY) - if cached: - ttl = await self._cache.attl(self.CACHE_KEY) - LOG.debug(f"Returning cached result (TTL: {ttl:.0f}s)") - return cached + app_cached = await self._cache.aget(self.CACHE_KEY) + ytdlp_cached = await self._cache.aget(self.YTDLP_CACHE_KEY) + if app_cached and ytdlp_cached: + return (app_cached, ytdlp_cached) + + app_result = app_cached if app_cached else await self._check_app_version() + ytdlp_result = ytdlp_cached if ytdlp_cached else await self._check_ytdlp_version() + + return (app_result, ytdlp_result) + + async def _check_github_version( + self, + name: str, + api_url: str, + current_version: str, + cache_key: str, + strip_v_prefix: bool = False, + ) -> tuple[str, str | None]: try: - LOG.info("Checking for application updates...") - - current_version: str = APP_VERSION.lstrip("v") + LOG.info(f"Checking for {name} updates...") async with async_client(timeout=10.0) as client: response = await client.get( - self.GITHUB_API_URL, + api_url, headers={"Accept": "application/vnd.github+json"}, ) if 200 != response.status_code: - LOG.warning(f"Failed to check for updates: HTTP {response.status_code}") + LOG.warning(f"Failed to check for {name} updates: HTTP {response.status_code}") return ("error", None) data: dict[str, Any] = response.json() - latest_tag: str = data.get("tag_name", "").lstrip("v") + latest_tag: str = data.get("tag_name", "") if not latest_tag: - LOG.warning("No tag_name found in GitHub release data.") + LOG.warning(f"No tag_name found in {name} GitHub release data.") return ("error", None) - if self._compare_versions(current_version, latest_tag): - LOG.warning(f"Update available: {current_version} → {latest_tag}") - new_version_tag = data.get("tag_name", "") - self._config.new_version = new_version_tag - await self.on_shutdown(None) - result = ("update_available", new_version_tag) - await self._cache.aset(self.CACHE_KEY, result, self.CACHE_DURATION) + compare_current: str = current_version.lstrip("v") if strip_v_prefix else current_version + compare_latest: str = latest_tag.lstrip("v") if strip_v_prefix else latest_tag + + if self._compare_versions(compare_current, compare_latest): + LOG.warning(f"{name} update available: {current_version} -> {latest_tag}") + result: tuple[str, str] = ("update_available", latest_tag) + await self._cache.aset(cache_key, result, self.CACHE_DURATION) return result - LOG.info("No updates available.") - self._config.new_version = "" - result = ("up_to_date", None) - await self._cache.aset(self.CACHE_KEY, result, self.CACHE_DURATION) + LOG.info(f"No {name} updates available.") + result: tuple[str, None] = ("up_to_date", None) + await self._cache.aset(cache_key, result, self.CACHE_DURATION) return result except Exception as e: LOG.exception(e) - LOG.error(f"Error checking for updates: {e!s}") + LOG.error(f"Error checking for {name} updates: {e!s}") return ("error", None) + async def _check_app_version(self) -> tuple[str, str | None]: + status, new_version = await self._check_github_version( + name="application", + api_url=self.GITHUB_API_URL, + current_version=APP_VERSION, + cache_key=self.CACHE_KEY, + strip_v_prefix=True, + ) + + if "update_available" == status: + self._config.new_version = new_version or "" + await self.on_shutdown(None) + elif "up_to_date" == status: + self._config.new_version = "" + + return (status, new_version) + + async def _check_ytdlp_version(self) -> tuple[str, str | None]: + current_version: str = self._config._ytdlp_version() + if not current_version or "0.0.0" == current_version: + LOG.warning("Could not determine yt-dlp version, skipping yt-dlp update check.") + return ("error", None) + + status, new_version = await self._check_github_version( + name="yt-dlp", + api_url=self.YTDLP_API_URL, + current_version=current_version, + cache_key=self.YTDLP_CACHE_KEY, + strip_v_prefix=False, + ) + + if "update_available" == status: + self._config.yt_new_version = new_version or "" + elif "up_to_date" == status: + self._config.yt_new_version = "" + + return (status, new_version) + def _compare_versions(self, current: str, latest: str) -> bool: """ Compare version strings to determine if an update is available. diff --git a/app/library/Utils.py b/app/library/Utils.py index 37f541f3..b62ce94f 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -1,5 +1,7 @@ +import asyncio import base64 import copy +import functools import glob import ipaddress import json @@ -86,6 +88,9 @@ TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c") DT_PATTERN: re.Pattern[str] = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:\d{2}))\s?") "Regex to match ISO 8601 datetime strings." +EXTRACTORS_SEMAPHORE: asyncio.Semaphore | None = None +"Global semaphore for limiting concurrent extractor operations." + class StreamingError(Exception): """Raised when an error occurs during streaming.""" @@ -416,6 +421,58 @@ def extract_info( return YTDLP.sanitize_info(data, remove_private_keys=True) if sanitize_info else data +async def fetch_info( + config: dict, + url: str, + debug: bool = False, + no_archive: bool = False, + follow_redirect: bool = False, + sanitize_info: bool = False, + **kwargs, +) -> dict: + """ + Extracts video information from the given URL. + + Args: + config (dict): Configuration options. + url (str): URL to extract information from. + debug (bool): Enable debug logging. + no_archive (bool): Disable download archive. + follow_redirect (bool): Follow URL redirects. + sanitize_info (bool): Sanitize the extracted information + **kwargs: Additional arguments. + + Returns: + dict: Video information. + + """ + from .config import Config + + global EXTRACTORS_SEMAPHORE # noqa: PLW0603 + + conf = Config.get_instance() + if EXTRACTORS_SEMAPHORE is None: + EXTRACTORS_SEMAPHORE = asyncio.Semaphore(conf.extract_info_concurrency) + + async with EXTRACTORS_SEMAPHORE: + return await asyncio.wait_for( + fut=asyncio.get_running_loop().run_in_executor( + None, + functools.partial( + extract_info, + config=config, + url=url, + debug=debug, + no_archive=no_archive, + follow_redirect=follow_redirect, + sanitize_info=sanitize_info, + **kwargs, + ), + ), + timeout=conf.extract_info_timeout, + ) + + def _is_safe_key(key: any) -> bool: """ Check if a dictionary key is safe for merging. @@ -1590,6 +1647,9 @@ def load_modules(root_path: Path, directory: Path): import pkgutil package_name: str = str(directory.relative_to(root_path).as_posix()).replace("/", ".") + # Ensure package name starts with 'app.' for proper module resolution + if not package_name.startswith("app."): + package_name = f"app.{package_name}" for _, name, _ in pkgutil.iter_modules([directory]): full_name: str = f"{package_name}.{name}" diff --git a/app/library/YTDLPOpts.py b/app/library/YTDLPOpts.py index 8b678d12..d846e6a9 100644 --- a/app/library/YTDLPOpts.py +++ b/app/library/YTDLPOpts.py @@ -3,8 +3,9 @@ import shlex from pathlib import Path from typing import Any +from app.features.presets.schemas import Preset + from .config import Config -from .Presets import Preset, Presets from .Utils import REMOVE_KEYS, arg_converter, calc_download_path, create_cookies_file, merge_dict LOG: logging.Logger = logging.getLogger("YTDLPOpts") @@ -129,9 +130,16 @@ class YTDLPCli: raise ValueError(msg) self.item = item - self.preset: Preset = item.get_preset() + preset_name = item.preset if item.preset else self._config.default_preset + self.preset: Preset | None = YTDLPCli._get_presets().get(preset_name) self._config: Config = config or Config.get_instance() + @staticmethod + def _get_presets(): + from app.features.presets.service import Presets + + return Presets.get_instance() + def build(self) -> tuple[str, dict]: """ Build the CLI command following make_command logic. @@ -306,7 +314,7 @@ class YTDLPOpts: YTDLPOpts: The instance of the class """ - preset: Preset | None = Presets.get_instance().get(name) + preset: Preset | None = YTDLPCli._get_presets().get(name) if not preset: return self diff --git a/app/library/cache.py b/app/library/cache.py index 34aa7bbe..cce18362 100644 --- a/app/library/cache.py +++ b/app/library/cache.py @@ -1,10 +1,18 @@ import hashlib +import logging import threading import time from typing import Any +from aiohttp import web + +from app.library.Services import Services + +from .Scheduler import Scheduler from .Singleton import ThreadSafe +LOG = logging.getLogger("cache") + class Cache(metaclass=ThreadSafe): def __init__(self) -> None: @@ -14,6 +22,25 @@ class Cache(metaclass=ThreadSafe): self._cache: dict[str, tuple[Any, float | None]] = {} self._lock = threading.Lock() + @staticmethod + def get_instance() -> "Cache": + """ + Get the singleton instance of Cache. + + Returns: + Cache: The singleton instance of Cache. + + """ + return Cache() + + def attach(self, _: web.Application) -> None: + Services.get_instance().add("cache", self) + Scheduler.get_instance().add( + timer="* * * * *", + func=self.cleanup, + id=f"{__class__.__name__}.{__class__.cleanup.__name__}", + ) + def set(self, key: str, value: Any, ttl: float | None = None) -> None: """ Synchronously set a value in the cache with an optional time-to-live. @@ -132,3 +159,20 @@ class Cache(metaclass=ThreadSafe): Asynchronously generate a SHA-256 hash for the given input string. """ return self.hash(key=key) + + async def cleanup(self) -> None: + """ + Remove all expired entries from the cache. + Called periodically by the scheduler. + """ + with self._lock: + now = time.time() + expired_keys: list[str] = [ + key for key, (_, expire_at) in self._cache.items() if expire_at is not None and now >= expire_at + ] + + for key in expired_keys: + del self._cache[key] + + if expired_keys: + LOG.debug(f"Cleaned up {len(expired_keys)} expired cache entries.") diff --git a/app/library/conditions.py b/app/library/conditions.py deleted file mode 100644 index 281e2657..00000000 --- a/app/library/conditions.py +++ /dev/null @@ -1,377 +0,0 @@ -import json -import logging -import uuid -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from aiohttp import web - -from .config import Config -from .encoder import Encoder -from .Events import EventBus, Events -from .mini_filter import match_str -from .Singleton import Singleton -from .Utils import arg_converter, init_class - -LOG: logging.Logger = logging.getLogger("conditions") - - -@dataclass(kw_only=True) -class Condition: - id: str = field(default_factory=lambda: str(uuid.uuid4())) - """The id of the condition.""" - - name: str - """The name of the condition.""" - - filter: str - """The filter to run on info dict.""" - - cli: str = "" - """If matched append this to the download request and retry.""" - - extras: dict[str, Any] = field(default_factory=dict) - """Any extra data to store with the condition.""" - - enabled: bool = True - """Whether the condition is enabled.""" - - priority: int = 0 - """Priority of the condition.""" - - description: str = "" - """A description of what the condition does.""" - - def serialize(self) -> dict: - return self.__dict__ - - def json(self) -> str: - return Encoder().encode(self.serialize()) - - def get(self, key: str, default: Any = None) -> Any: - return self.serialize().get(key, default) - - -class Conditions(metaclass=Singleton): - """ - This class is used to manage the download conditions. - """ - - def __init__(self, file: Path | str | None = None, config: Config | None = None): - self._items: list[Condition] = [] - "The list of items." - - config = config or Config.get_instance() - - self._file: Path = Path(file) if file else Path(config.config_path) / "conditions.json" - "The path to the file where the items are stored." - - if self._file.exists() and "600" != self._file.stat().st_mode: - try: - self._file.chmod(0o600) - except Exception: - pass - - @staticmethod - def get_instance(file: Path | str | None = None, config: Config | None = None) -> "Conditions": - """ - Get the instance of the class. - - Returns: - Conditions: The instance of the class - - """ - return Conditions(file=file, config=config) - - async def on_shutdown(self, _: web.Application): - pass - - def attach(self, _: web.Application): - """ - Attach the work to the aiohttp application. - - Args: - _ (web.Application): The aiohttp application. - - Returns: - None - - """ - self.load() - - async def event_handler(_, __): - msg = "Not implemented" - raise Exception(msg) - - EventBus.get_instance().subscribe(Events.CONDITIONS_ADD, event_handler, f"{__class__.__name__}.save") - - def get_all(self) -> list[Condition]: - """Return the items.""" - return self._items - - def load(self) -> "Conditions": - """ - Load the items. - - Returns: - Conditions: The current instance. - - """ - self.clear() - - if not self._file.exists() or self._file.stat().st_size < 1: - return self - - try: - LOG.info(f"Loading '{self._file}'.") - items = json.loads(self._file.read_text()) - except Exception as e: - LOG.exception(e) - LOG.error(f"Error loading '{self._file}'. '{e}'.") - return self - - if not items or len(items) < 1: - LOG.debug(f"No items were found in '{self._file}'.") - return self - - need_save = False - - for i, item in enumerate(items): - try: - if "id" not in item: - item["id"] = str(uuid.uuid4()) - need_save = True - - if "extras" not in item: - item["extras"] = {} - need_save = True - - if "enabled" not in item: - item["enabled"] = True - need_save = True - - if "priority" not in item: - item["priority"] = 0 - need_save = True - - if "description" not in item: - item["description"] = "" - need_save = True - - item: Condition = init_class(Condition, item) - - self._items.append(item) - except Exception as e: - LOG.error(f"Failed to parse condition at list position '{i}'. '{e!s}'.") - continue - - if need_save: - LOG.warning("Saving conditions due to schema changes.") - self.save(self._items) - - return self - - def clear(self) -> "Conditions": - """ - Clear all items - - Returns: - conditions: The current instance. - - """ - if len(self._items) < 1: - return self - - self._items.clear() - - return self - - def validate(self, item: Condition | dict) -> bool: - """ - Validate item. - - Args: - item (Condition|dict): The item to validate. - - Returns: - bool: True if valid, False otherwise. - - """ - if not isinstance(item, dict): - if not isinstance(item, Condition): - msg = f"Unexpected '{type(item).__name__}' item type." - raise ValueError(msg) - - item = item.serialize() - - if not item.get("id"): - msg = "No id found." - raise ValueError(msg) - - if not item.get("name"): - msg = "No name found." - raise ValueError(msg) - - if not item.get("filter"): - msg = "No filter found." - raise ValueError(msg) - - try: - match_str(item.get("filter"), {}) - except Exception as e: - msg = f"Invalid filter. '{e!s}'." - raise ValueError(msg) from e - - if item.get("cli"): - try: - arg_converter(args=item.get("cli")) - except Exception as e: - msg = f"Invalid command options for yt-dlp. '{e!s}'." - raise ValueError(msg) from e - - if not isinstance(item.get("extras"), dict): - msg = "Extras must be a dictionary." - raise ValueError(msg) - - if item.get("enabled") is not None and not isinstance(item.get("enabled"), bool): - msg = "Enabled must be a boolean." - raise ValueError(msg) - - if item.get("priority") is not None: - priority = item.get("priority") - if not isinstance(priority, int): - msg = "Priority must be an integer." - raise ValueError(msg) - if priority < 0: - msg = "Priority must be >= 0." - raise ValueError(msg) - - if item.get("description") and not isinstance(item.get("description"), str): - msg = "Description must be a string." - raise ValueError(msg) - - return True - - def save(self, items: list[Condition | dict]) -> "Conditions": - """ - Save the items. - - Args: - items (list[Condition]): The items to save. - - Returns: - Conditions: The current instance. - - """ - for i, item in enumerate(items): - try: - if not isinstance(item, Condition): - item: Condition = init_class(Condition, item) - items[i] = item - except Exception as e: - LOG.error(f"Failed to save '{i}' due to parsing error. '{e!s}'.") - continue - - try: - self.validate(item) - except ValueError as e: - LOG.error(f"Failed to validate '{i}: {item.name}'. '{e!s}'.") - continue - - try: - self._file.write_text(json.dumps(obj=[item.serialize() for item in items], indent=4)) - LOG.info(f"Updated '{self._file}'.") - except Exception as e: - LOG.error(f"Error saving '{self._file}'. '{e!s}'.") - - return self - - def has(self, id_or_name: str) -> bool: - """ - Check if the item exists by id or name. - - Args: - id_or_name (str): The id or name of the preset. - - Returns: - bool: True if the item exists, False otherwise. - - """ - return self.get(id_or_name) is not None - - def get(self, id_or_name: str) -> Condition | None: - """ - Get the item by id or name. - - Args: - id_or_name (str): The id or name of the preset. - - Returns: - Condition|None: The item if found, None otherwise. - - """ - if not id_or_name: - return None - - for condition in self.get_all(): - if id_or_name not in (condition.id, condition.name): - continue - - return condition - - return None - - def match(self, info: dict) -> Condition | None: - """ - Check if any condition matches the info dict. - - Args: - info (dict): The info dict to check. - - Returns: - Condition|None: The condition if found, None otherwise. - - """ - if len(self._items) < 1 or not info or not isinstance(info, dict) or len(info) < 1: - return None - - for item in sorted(self.get_all(), key=lambda x: x.priority, reverse=True): - if not item.enabled: - continue - - if not item.filter: - LOG.error(f"Filter is empty for '{item.name}'.") - continue - - try: - if match_str(item.filter, info): - LOG.debug(f"Matched '{item.name}' with filter '{item.filter}'.") - - return item - except Exception as e: - LOG.error(f"Failed to evaluate '{item.name}'. '{e!s}'.") - continue - - return None - - def single_match(self, name: str, info: dict) -> Condition | None: - """ - Check if condition matches the info dict. - - Args: - name (str): The condition name to check. - info (dict): The info dict to check. - - Returns: - Condition|None: The condition if found, None otherwise. - - """ - if len(self._items) < 1 or not info or not isinstance(info, dict) or len(info) < 1: - return None - - item = self.get(name) - if not item or not item.enabled or not item.filter: - return None - - return item if match_str(item.filter, info) else None diff --git a/app/library/config.py b/app/library/config.py index 18832e53..c139ed42 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -238,6 +238,9 @@ class Config(metaclass=Singleton): new_version: str = "" "The new version available." + yt_new_version: str = "" + "The new yt-dlp version available." + _manual_vars: tuple = ( "temp_path", "config_path", @@ -255,6 +258,7 @@ class Config(metaclass=Singleton): "app_build_date", "app_branch", "new_version", + "yt_new_version", ) "The variables that are immutable." @@ -269,6 +273,7 @@ class Config(metaclass=Singleton): "download_info_expires", "auto_clear_history_days", "default_pagination", + "extract_info_concurrency", "flaresolverr_max_timeout", "flaresolverr_client_timeout", "flaresolverr_cache_ttl", @@ -327,6 +332,7 @@ class Config(metaclass=Singleton): "default_pagination", "check_for_updates", "new_version", + "yt_new_version", ) "The variables that are relevant to the frontend." diff --git a/app/library/dl_fields.py b/app/library/dl_fields.py deleted file mode 100644 index f901c4d3..00000000 --- a/app/library/dl_fields.py +++ /dev/null @@ -1,333 +0,0 @@ -import json -import logging -import re -import uuid -from dataclasses import dataclass, field -from enum import Enum -from pathlib import Path -from typing import Any - -from aiohttp import web - -from .config import Config -from .encoder import Encoder -from .Events import EventBus, Events -from .Singleton import Singleton -from .Utils import init_class - -LOG = logging.getLogger("DLFields") - - -class FieldType(str, Enum): - STRING = "string" - TEXT = "text" - BOOL = "bool" - - @classmethod - def all(cls) -> list[str]: - return [member.value for member in cls] - - @classmethod - def from_value(cls, value: str) -> "FieldType": - """ - Returns the FieldType enum member corresponding to the given value. - - Args: - value (str): The value to match against the enum members. - - Returns: - StoreType: The enum member that matches the value. - - Raises: - ValueError: If the value does not match any member. - - """ - for member in cls: - if member.value == value: - return member - - msg = f"Invalid StoreType value: {value}" - raise ValueError(msg) - - def __str__(self) -> str: - return self.value - - -@dataclass(kw_only=True) -class DLField: - id: str = field(default_factory=lambda: str(uuid.uuid4())) - """The id of the field.""" - - name: str - """The name of the preset.""" - - description: str - """The description of the preset.""" - - field: str - """The yt-dlp field to use in long format.""" - - kind: FieldType = FieldType.TEXT - """The kind of the field. i.e. string, bool""" - - icon: str = "" - """The icon of the field, it can be a font-awesome icon""" - - order: int = 0 - """The order of the field, used to sort the fields in the UI.""" - - value: str = "" - """The default value of the field, It's currently unused.""" - - extras: dict = field(default_factory=dict) - """Additional options for the field.""" - - def serialize(self) -> dict: - dct = self.__dict__ - dct["kind"] = str(self.kind) - dct["extras"] = {k: v for k, v in self.extras.items() if v is not None} - return dct - - def json(self) -> str: - return Encoder().encode(self.serialize()) - - def get(self, key: str, default: Any = None) -> Any: - return self.serialize().get(key, default) - - -class DLFields(metaclass=Singleton): - """ - This class is used to manage the DLFields. - """ - - def __init__(self, file: str | Path | None = None, config: Config | None = None): - self._items: list[DLField] = [] - "The list of items." - config: Config = config or Config.get_instance() - "The configuration instance." - self._file: Path = Path(file) if file else Path(config.config_path).joinpath("dl_fields.json") - "The path to the file where the items are stored." - - if self._file.exists() and "600" != self._file.stat().st_mode: - try: - self._file.chmod(0o600) - except Exception: - pass - - @staticmethod - def get_instance(file: str | Path | None = None, config: Config | None = None) -> "DLFields": - """ - Get the instance of the class. - - Returns: - DLFields: The instance of the class - - """ - return DLFields(file=file, config=config) - - async def on_shutdown(self, _: web.Application): - pass - - def attach(self, _: web.Application): - """ - Attach the class to the aiohttp application. - - Args: - _ (web.Application): The aiohttp application. - - Returns: - None - - """ - self.load() - - async def event_handler(_, __): - msg = "Not implemented" - raise Exception(msg) - - EventBus.get_instance().subscribe(Events.DLFIELDS_ADD, event_handler, f"{__class__.__name__}.add") - - def get_all(self) -> list[DLField]: - """Return the items.""" - return self._items - - def load(self) -> "DLFields": - """ - Load the items. - - Returns: - Presets: The current instance. - - """ - has: int = len(self._items) - self.clear() - - if not self._file.exists() or self._file.stat().st_size < 10: - return self - - try: - LOG.info(f"{'Reloading' if has else 'Loading'} '{self._file}'.") - items: dict = json.loads(self._file.read_text()) - except Exception as e: - LOG.error(f"Failed to parse '{self._file}'. '{e}'.") - return self - - if not items or len(items) < 1: - return self - - need_save = False - - for i, item in enumerate(items): - try: - if "id" not in item: - item["id"] = str(uuid.uuid4()) - need_save = True - - item: DLField = init_class(DLField, item) - - self._items.append(item) - except Exception as e: - LOG.error(f"Failed to parse '{self._file}:{i}'. '{e!s}'.") - continue - - if need_save: - LOG.info(f"Saving '{self._file}'.") - self.save(self._items) - - return self - - def clear(self) -> "DLFields": - """ - Clear all items. - - Returns: - DLFields: The current instance. - - """ - if len(self._items) < 1: - return self - - self._items.clear() - - return self - - def validate(self, item: DLField | dict) -> bool: - """ - Validate the item. - - Args: - item (DLField|dict): The item to validate. - - Returns: - bool: True if valid - - Raises: - ValueError: If the item is not valid. - - """ - if not isinstance(item, dict): - if not isinstance(item, DLField): - msg = f"Unexpected '{type(item).__name__}' type was given." - raise ValueError(msg) - - item = item.serialize() - - for key in ["id", "name", "description", "field", "kind"]: - if key not in item: - msg = f"Missing required key '{key}'." - raise ValueError(msg) - - if item.get("kind") not in FieldType.all(): - msg = f"Invalid field type '{item.get('kind')}'." - raise ValueError(msg) - - if item.get("extras") and not isinstance(item.get("extras"), dict): - msg = "Extras must be a dictionary." - raise ValueError(msg) - - if item.get("value") and not isinstance(item.get("value"), str): - msg = "Value must be a string." - raise ValueError(msg) - - if item.get("order") is not None and not isinstance(item.get("order"), int): - msg = "Order must be an integer." - raise ValueError(msg) - - if not isinstance(item.get("extras", {}), dict): - msg = "Extras must be a dictionary." - raise ValueError(msg) - - if re.match(r"^--[a-zA-Z0-9\-]+$", item.get("field", "").strip()) is None: - msg = "Invalid yt-dlp option field it must starts with '--' and contain only alphanumeric characters." - raise ValueError(msg) - - return True - - def save(self, items: list[DLField | dict]) -> "DLFields": - """ - Save the items. - - Args: - items (list[DLField]): The items to save. - - Returns: - Presets: The current instance. - - """ - for i, item in enumerate(items): - try: - if not isinstance(item, DLField): - item: DLField = init_class(DLField, item) - items[i] = item - except Exception as e: - LOG.error(f"Failed to save item '{i}' due to parsing error. '{e!s}'.") - continue - - try: - self.validate(item) - except ValueError as e: - LOG.error(f"Failed to validate item '{i}: {item.name}'. '{e}'.") - continue - - try: - self._file.write_text(json.dumps(obj=[item.serialize() for item in items], indent=4)) - LOG.info(f"Saved '{self._file}'.") - except Exception as e: - LOG.error(f"Failed to save '{self._file}'. '{e!s}'.") - - return self - - def has(self, id_or_name: str) -> bool: - """ - Check if the item exists by id or name. - - Args: - id_or_name (str): The id or name of the item. - - Returns: - bool: True if exists, False otherwise. - - """ - return self.get(id_or_name) is not None - - def get(self, id_or_name: str) -> DLField | None: - """ - Get the item by id or name. - - Args: - id_or_name (str): The id or name of the item. - - Returns: - Preset|None: The item if found, None otherwise. - - """ - if not id_or_name: - return None - - for item in self.get_all(): - if id_or_name not in (item.id, item.name): - continue - - return item - - return None diff --git a/app/library/downloads/item_adder.py b/app/library/downloads/item_adder.py index d0a23473..af7aeaba 100644 --- a/app/library/downloads/item_adder.py +++ b/app/library/downloads/item_adder.py @@ -1,17 +1,4 @@ -""" -Item addition and entry routing. - -This module handles the complete flow of adding items to the download queue: -- Preset and CLI validation -- Cookie file management -- Archive checking (early and post-extraction) -- Info extraction with yt-dlp -- Conditions matching and re-queuing -- Entry type routing (playlist, video, URL) -""" - import asyncio -import functools import logging import time import uuid @@ -20,16 +7,16 @@ from typing import TYPE_CHECKING import yt_dlp.utils -from app.library.conditions import Conditions +from app.features.conditions.service import Conditions +from app.features.presets.service import Presets from app.library.Events import Events from app.library.ItemDTO import ItemDTO -from app.library.Presets import Presets from app.library.Utils import ( archive_add, archive_read, arg_converter, create_cookies_file, - extract_info, + fetch_info, get_extras, merge_dict, ytdlp_reject, @@ -40,8 +27,8 @@ from .playlist_processor import process_playlist from .video_processor import add_video if TYPE_CHECKING: + from app.features.presets.schemas import Preset from app.library.ItemDTO import Item - from app.library.Presets import Preset from .queue_manager import DownloadQueue @@ -205,22 +192,14 @@ async def add( LOG.info(f"[P] Extracting '{item.url}'{' with cookies' if yt_conf.get('cookiefile') else ''}.") if not entry: - async with queue.extractors: - LOG.info(f"Extracting '{item.url}'{' with cookies' if yt_conf.get('cookiefile') else ''}.") - entry: dict | None = await asyncio.wait_for( - fut=asyncio.get_running_loop().run_in_executor( - None, - functools.partial( - extract_info, - config=yt_conf, - url=item.url, - debug=bool(queue.config.ytdlp_debug), - no_archive=False, - follow_redirect=True, - ), - ), - timeout=queue.config.extract_info_timeout, - ) + LOG.info(f"Extracting '{item.url}'{' with cookies' if yt_conf.get('cookiefile') else ''}.") + entry: dict | None = await fetch_info( + config=yt_conf, + url=item.url, + debug=bool(queue.config.ytdlp_debug), + no_archive=False, + follow_redirect=True, + ) if not entry: LOG.error(f"Unable to extract info for '{item.url}'. Logs: {logs}") @@ -288,7 +267,7 @@ async def add( ) return {"status": "error", "msg": message, "hidden": True} - if not item.requeued and (condition := Conditions.get_instance().match(info=entry)): + if not item.requeued and (condition := await Conditions.get_instance().match(info=entry)): already.pop() message = f"Condition '{condition.name}' matched for '{item!r}'." diff --git a/app/library/downloads/queue_manager.py b/app/library/downloads/queue_manager.py index b9d66ad3..88a35ce6 100644 --- a/app/library/downloads/queue_manager.py +++ b/app/library/downloads/queue_manager.py @@ -43,8 +43,6 @@ class DownloadQueue(metaclass=Singleton): "DataStore for the download queue." self.processors = asyncio.Semaphore(self.config.playlist_items_concurrency) "Semaphore to limit the number of concurrent processors." - self.extractors = asyncio.Semaphore(self.config.extract_info_concurrency) - "Semaphore to limit the number of concurrent extract_info calls." self.pool = PoolManager(queue=self, config=self.config) "Pool manager for coordinating download execution." diff --git a/app/library/encoder.py b/app/library/encoder.py index d16aaf12..72ba8e0b 100644 --- a/app/library/encoder.py +++ b/app/library/encoder.py @@ -16,16 +16,10 @@ class Encoder(json.JSONEncoder): def default(self, o): from .ItemDTO import ItemDTO - if isinstance(o, Path): - return str(o) - if isinstance(o, DateRange): return {"start": str(o.start).replace("-", ""), "end": str(o.end).replace("-", "")} - if isinstance(o, date): - return str(o) - - if isinstance(o, ImpersonateTarget): + if isinstance(o, (Path, date, ImpersonateTarget, ValueError)): return str(o) if isinstance(o, ItemDTO): @@ -35,6 +29,9 @@ class Encoder(json.JSONEncoder): if hasattr(o, "serialize"): return o.serialize() + if hasattr(o, "model_dump"): + return o.model_dump() + if hasattr(o, "__dict__"): return o.__dict__ diff --git a/app/library/migrate.py b/app/library/migrate.py index 29119344..17cda134 100644 --- a/app/library/migrate.py +++ b/app/library/migrate.py @@ -14,7 +14,8 @@ from importlib.machinery import ModuleSpec, SourceFileLoader from importlib.util import module_from_spec, spec_from_loader from typing import TYPE_CHECKING -import aiosqlite +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncConnection if TYPE_CHECKING: from types import ModuleType @@ -43,19 +44,19 @@ class InvalidNameError(Error): @contextlib.asynccontextmanager -async def execute( - conn: aiosqlite.Connection, sql: str, params: Sequence[object] | None = None -) -> AsyncIterator[aiosqlite.Cursor]: - params = [] if params is None else params - cursor = await conn.execute(sql, params) +async def execute(conn: AsyncConnection, sql: str, params: Sequence[object] | None = None) -> AsyncIterator: + params = {} if params is None else params + result = await conn.execute(text(sql), params) try: - yield cursor + yield result finally: - await cursor.close() + pass @contextlib.asynccontextmanager -async def transaction(conn: aiosqlite.Connection) -> AsyncIterator[None]: +async def transaction(conn: AsyncConnection) -> AsyncIterator[None]: + # SQLAlchemy AsyncConnection manages transactions automatically + # when used with context managers, so we just yield try: yield await conn.commit() @@ -104,10 +105,10 @@ class Migration: raise InvalidNameError(self.filename) return timestamp - async def upgrade(self, conn: aiosqlite.Connection) -> None: + async def upgrade(self, conn: AsyncConnection) -> None: await self.module.upgrade(conn) - async def downgrade(self, conn: aiosqlite.Connection) -> None: + async def downgrade(self, conn: AsyncConnection) -> None: await self.module.downgrade(conn) def has_method(self, name: str) -> bool: @@ -118,7 +119,7 @@ class Migration: class Database: - def __init__(self, db_url: aiosqlite.Connection | str, version_table: str = "migration_version"): + def __init__(self, db_url: AsyncConnection | str, version_table: str = "migration_version"): if not db_url: msg = "Database requires db_url." raise ValueError(msg) @@ -127,16 +128,18 @@ class Database: self._owns_connection = bool(isinstance(db_url, str)) if self._owns_connection: - self.conn: aiosqlite.Connection | None = None + self.conn: AsyncConnection | None = None self.db_url: str = db_url - self._ensure_connection() else: - self.conn: aiosqlite.Connection = db_url + self.conn: AsyncConnection = db_url async def __aenter__(self) -> "Database": - if self._owns_connection: - await self._ensure_connection() + if self._owns_connection and self.conn is None: + # Create connection from string URL + from sqlalchemy.ext.asyncio import create_async_engine + self._engine = create_async_engine(f"sqlite+aiosqlite:///{self.db_url}") + self.conn = await self._engine.connect() return self async def __aexit__(self, exc_type, exc, tb) -> None: @@ -147,23 +150,18 @@ class Database: async def close(self) -> None: if self._owns_connection and self.conn: await self.conn.close() + if hasattr(self, "_engine"): + await self._engine.dispose() self.conn = None - async def _ensure_connection(self) -> None: - if self.conn: - return - self.conn = await aiosqlite.connect(self.db_url) - self.conn.row_factory = aiosqlite.Row - async def is_version_controlled(self) -> bool: - await self._ensure_connection() assert self.conn - sql = """SELECT * FROM sqlite_master WHERE type = 'table' AND name = ?""" - async with execute(self.conn, sql, [self.version_table]) as cursor: - return bool(await cursor.fetchall()) + sql = """SELECT * FROM sqlite_master WHERE type = 'table' AND name = :version_table""" + async with execute(self.conn, sql, {"version_table": self.version_table}) as result: + rows = result.fetchall() + return bool(rows) async def upgrade(self, migrations: list[Migration], target_version: str | None = None) -> None: - await self._ensure_connection() assert self.conn if target_version: _assert_migration_exists(migrations, target_version) @@ -183,7 +181,6 @@ class Database: database_version: str = new_version async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None: - await self._ensure_connection() assert self.conn if target_version not in (0, "0"): _assert_migration_exists(migrations, target_version) @@ -210,29 +207,26 @@ class Database: Return the database's version, or None if it is not under version control. """ - await self._ensure_connection() assert self.conn if not await self.is_version_controlled(): return None sql: str = f"SELECT version FROM {self.version_table}" - async with execute(self.conn, sql) as cursor: - result = await cursor.fetchall() - return result[0][0] if result else "0" + async with execute(self.conn, sql) as result: + rows = result.fetchall() + return rows[0][0] if rows else "0" async def update_version(self, version: str) -> None: - await self._ensure_connection() assert self.conn - sql: str = f"UPDATE {self.version_table} SET version = ?" + sql: str = f"UPDATE {self.version_table} SET version = :version" async with transaction(self.conn): - await self.conn.execute(sql, [version]) + await self.conn.execute(text(sql), {"version": version}) async def initialize_version_control(self) -> None: - await self._ensure_connection() assert self.conn sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """ async with transaction(self.conn): - await self.conn.execute(sql) - await self.conn.execute(f"INSERT INTO {self.version_table} VALUES (0)") + await self.conn.execute(text(sql)) + await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"}) def __repr__(self) -> str: return f'Database("{self.db_url if self._owns_connection else "external_connection"}")' @@ -253,7 +247,7 @@ def load_migrations(directory: str) -> list[Migration]: return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))] -async def upgrade(db_url: aiosqlite.Connection | str, migration_dir: str, version: str | None = None) -> None: +async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: str | None = None) -> None: """ Upgrade the given database with the migrations contained in the migrations directory. If a version is not specified, upgrade @@ -266,7 +260,7 @@ async def upgrade(db_url: aiosqlite.Connection | str, migration_dir: str, versio await db.upgrade(load_migrations(migration_dir), version) -async def downgrade(db_url: str | aiosqlite.Connection, migration_dir: str, version: str) -> None: +async def downgrade(db_url: str | AsyncConnection, migration_dir: str, version: str) -> None: """ Downgrade the database to the given version with the migrations contained in the given migration directory. @@ -279,7 +273,7 @@ async def downgrade(db_url: str | aiosqlite.Connection, migration_dir: str, vers await db.downgrade(migrations, version) -async def get_version(db_url: aiosqlite.Connection | str) -> str | None: +async def get_version(db_url: AsyncConnection | str) -> str | None: """Return the migration version of the given database.""" async with Database(db_url) as db: return await db.get_version() @@ -317,11 +311,13 @@ Migration Name: %(name)s Migration Version: %(version)s \"\"\" +from sqlalchemy import text + async def upgrade(c): # add your upgrade step here - await c.execute("SELECT 1") + await c.execute(text("SELECT 1")) async def downgrade(c): # add your downgrade step here - await c.execute("SELECT 1") + await c.execute(text("SELECT 1")) """ diff --git a/app/library/router.py b/app/library/router.py index 2efc793e..22e356b9 100644 --- a/app/library/router.py +++ b/app/library/router.py @@ -1,8 +1,9 @@ import logging import re -from collections.abc import Awaitable +from collections.abc import Awaitable, Callable from enum import Enum from functools import wraps +from typing import Any LOG: logging.Logger = logging.getLogger(__name__) @@ -25,15 +26,15 @@ class Route: method (str): The HTTP method (GET, POST, etc.). path (str): The path for the route. name (str): The name of the route. - handler (Awaitable): The function that handles the route. + handler (Callable[..., Awaitable]): The function that handles the route. """ - def __init__(self, method: str, path: str, name: str, handler: Awaitable): + def __init__(self, method: str, path: str, name: str, handler: Callable[..., Awaitable[Any]]): self.method: str = method.upper() self.path: str = path self.name: str = name - self.handler: Awaitable = handler + self.handler: Callable[..., Awaitable[Any]] = handler ROUTES: dict[str, dict[str, Route]] = {} @@ -55,7 +56,12 @@ def make_route_name(method: str, path: str) -> str: return f"{method}:" + ".".join(segments or ["root"]) -def route(method: RouteType | str | list[str], path: str, name: str | None = None, **kwargs) -> Awaitable: +def route( + method: RouteType | str | list[str], + path: str, + name: str | None = None, + **kwargs, +) -> Callable[[Callable[..., Awaitable[Any]]], Callable[..., Awaitable[Any]]]: """ Decorator to mark a method as an HTTP route handler. @@ -71,7 +77,7 @@ def route(method: RouteType | str | list[str], path: str, name: str | None = Non """ methods = [method] if isinstance(method, (str, RouteType)) else method - def decorator(func): + def decorator(func: Callable[..., Awaitable[Any]]): @wraps(func) async def wrapper(*args, **kwargs): return await func(*args, **kwargs) @@ -97,7 +103,13 @@ def route(method: RouteType | str | list[str], path: str, name: str | None = Non return decorator -def add_route(method: RouteType | str | list[str], path: str, handler: Awaitable, name: str | None = None, **kwargs): +def add_route( + method: RouteType | str | list[str], + path: str, + handler: Callable[..., Awaitable[Any]], + name: str | None = None, + **kwargs, +): """ Decorator to mark a method as an HTTP route handler. @@ -129,7 +141,7 @@ def add_route(method: RouteType | str | list[str], path: str, handler: Awaitable ) -def get_route(route_type: RouteType, name: str) -> dict[str, Route] | None: +def get_route(route_type: RouteType, name: str) -> Route | None: """ Get the route information by name. diff --git a/app/library/sqlite_store.py b/app/library/sqlite_store.py index 7a1b7df3..f56f09c3 100644 --- a/app/library/sqlite_store.py +++ b/app/library/sqlite_store.py @@ -8,9 +8,12 @@ from dataclasses import fields from datetime import UTC, datetime from email.utils import formatdate -import aiosqlite from aiohttp import web +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.ext.asyncio.engine import AsyncConnection +from .Events import EventBus, Events from .ItemDTO import ItemDTO from .operations import Operation, matches_condition from .Services import Services @@ -48,6 +51,12 @@ class SqliteStore(metaclass=ThreadSafe): def attach(self, app: web.Application): Services.get_instance().add("sqlite_store", self) + + async def handle_event(_, __): + await self.get_connection() + + EventBus.get_instance().subscribe(Events.STARTED, handle_event, "SqliteStore.get_connection") + app.on_shutdown.append(self.on_shutdown) async def on_shutdown(self, _: web.Application): @@ -57,36 +66,60 @@ class SqliteStore(metaclass=ThreadSafe): def __init__(self, db_path: str, *, max_pending: int = 200, flush_interval: float = 0.05): self._db_path: str = db_path - self._conn: aiosqlite.Connection | None = None + self._engine: AsyncEngine | None = None + self._conn: AsyncConnection | None = None + self._sessionmaker: async_sessionmaker[AsyncSession] | None = None self._queue: asyncio.Queue[_Op] | None = None self._task: asyncio.Task | None = None + self._lock: asyncio.Lock = asyncio.Lock() self._flush_interval: float = flush_interval self._max_pending: int = max_pending - self._lock = asyncio.Lock() async def __aenter__(self) -> "SqliteStore": - await self._ensure_conn() + await self.get_connection() return self async def __aexit__(self, exc_type, exc, tb) -> None: await self.close() + def sessionmaker(self) -> async_sessionmaker[AsyncSession]: + """ + Return the SQLAlchemy async sessionmaker. + + This allows other parts of the system to create SQLAlchemy sessions + that share the same database connection/engine. + + Returns: + async_sessionmaker[AsyncSession]: The sessionmaker instance. + + Raises: + RuntimeError: If called before connection is initialized. + + """ + if not self._sessionmaker: + msg = "Database connection not initialized. Call get_connection() first or use within async context." + raise RuntimeError(msg) + return self._sessionmaker + async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]: - await self._ensure_conn() - cursor = await self._conn.execute( - 'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC', - (type_value,), + await self.get_connection() + result = await self._conn.execute( + text( + 'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value ORDER BY "created_at" ASC' + ), + {"type_value": type_value}, ) + rows = result.mappings().all() + items: list[tuple[str, ItemDTO]] = [] - async with cursor: - async for row in cursor: - row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 - data = json.loads(row["data"]) - data.pop("_id", None) - item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) - item._id = row["id"] - item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) - items.append((row["id"], item)) + for row in rows: + row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 + data = json.loads(row["data"]) + data.pop("_id", None) + item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) + item._id = row["id"] + item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) + items.append((row["id"], item)) return items async def exists(self, type_value: str, key: str | None = None, url: str | None = None) -> bool: @@ -97,23 +130,27 @@ class SqliteStore(metaclass=ThreadSafe): msg = "key or url must be provided." raise KeyError(msg) - await self._ensure_conn() + await self.get_connection() clauses: list[str] = [] - params: list[str] = [] + params: dict[str, str] = {"type_value": type_value} if key: - clauses.append('"id" = ?') - params.append(key) + clauses.append('"id" = :key') + params["key"] = key if url: - clauses.append("json_extract(data, '$.url') = ?") - params.append(url) + clauses.append("json_extract(data, '$.url') = :url") + params["url"] = url where_clause = " OR ".join(clauses) - query = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? AND ({where_clause}) LIMIT 1' # noqa: S608 - cursor = await self._conn.execute(query, (type_value, *params)) - row = await cursor.fetchone() + query = ( + f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) LIMIT 1' # noqa: S608 + ) + + result = await self._conn.execute(text(query), params) + row = result.mappings().first() + if not row: return None @@ -126,12 +163,13 @@ class SqliteStore(metaclass=ThreadSafe): return item async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None: - await self._ensure_conn() - cursor = await self._conn.execute( - 'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? AND "id" = ?', - (type_value, id), + await self.get_connection() + result = await self._conn.execute( + text('SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" = :id'), + {"type_value": type_value, "id": id}, ) - row = await cursor.fetchone() + row = result.mappings().first() + if not row: return None @@ -154,10 +192,11 @@ class SqliteStore(metaclass=ThreadSafe): if not kwargs: return None - await self._ensure_conn() + await self.get_connection() clauses: list[str] = [] - params: list[str | float | int] = [] + params: dict[str, str | float | int] = {"type_value": type_value} + param_counter = 0 def _safe_key(key: str) -> str | None: return key if key.replace("_", "").isalnum() else None @@ -178,47 +217,51 @@ class SqliteStore(metaclass=ThreadSafe): except ValueError: operation = Operation.EQUAL - path = f"$.{safe_key}" - json_extract = f"json_extract(data, '{path}')" + path: str = f"$.{safe_key}" + json_extract: str = f"json_extract(data, '{path}')" + param_name: str = f"param_{param_counter}" + param_counter += 1 if Operation.EQUAL == operation: - clauses.append(f"{json_extract} = ?") - params.append(value) + clauses.append(f"{json_extract} = :{param_name}") + params[param_name] = value elif Operation.NOT_EQUAL == operation: - clauses.append(f"{json_extract} != ?") - params.append(value) + clauses.append(f"{json_extract} != :{param_name}") + params[param_name] = value elif Operation.CONTAIN == operation: - clauses.append(f"{json_extract} LIKE ? ESCAPE '\\'") - params.append(f"%{value}%") + clauses.append(f"{json_extract} LIKE :{param_name} ESCAPE '\\'") + params[param_name] = f"%{value}%" elif Operation.NOT_CONTAIN == operation: - clauses.append(f"({json_extract} IS NULL OR {json_extract} NOT LIKE ? ESCAPE '\\')") - params.append(f"%{value}%") + clauses.append(f"({json_extract} IS NULL OR {json_extract} NOT LIKE :{param_name} ESCAPE '\\')") + params[param_name] = f"%{value}%" elif Operation.STARTS_WITH == operation: - clauses.append(f"{json_extract} LIKE ? ESCAPE '\\'") - params.append(f"{value}%") + clauses.append(f"{json_extract} LIKE :{param_name} ESCAPE '\\'") + params[param_name] = f"{value}%" elif Operation.ENDS_WITH == operation: - clauses.append(f"{json_extract} LIKE ? ESCAPE '\\'") - params.append(f"%{value}") + clauses.append(f"{json_extract} LIKE :{param_name} ESCAPE '\\'") + params[param_name] = f"%{value}" elif Operation.GREATER_THAN == operation: - clauses.append(f"{json_extract} > ?") - params.append(value) + clauses.append(f"{json_extract} > :{param_name}") + params[param_name] = value elif Operation.LESS_THAN == operation: - clauses.append(f"{json_extract} < ?") - params.append(value) + clauses.append(f"{json_extract} < :{param_name}") + params[param_name] = value elif Operation.GREATER_EQUAL == operation: - clauses.append(f"{json_extract} >= ?") - params.append(value) + clauses.append(f"{json_extract} >= :{param_name}") + params[param_name] = value elif Operation.LESS_EQUAL == operation: - clauses.append(f"{json_extract} <= ?") - params.append(value) + clauses.append(f"{json_extract} <= :{param_name}") + params[param_name] = value if not clauses: return None - where_clause = " OR ".join(f"({clause})" for clause in clauses) - query = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? AND ({where_clause}) ORDER BY "created_at" ASC LIMIT 1' # noqa: S608 - cursor = await self._conn.execute(query, (type_value, *params)) - row = await cursor.fetchone() + where_clause: str = " OR ".join(f"({clause})" for clause in clauses) + query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) ORDER BY "created_at" ASC LIMIT 1' # noqa: S608 + + result = await self._conn.execute(text(query), params) + row = result.mappings().first() + if not row: return None @@ -232,23 +275,25 @@ class SqliteStore(metaclass=ThreadSafe): return item if any(matches_condition(k, v, item.__dict__) for k, v in kwargs.items()) else None async def count(self, type_value: str, status_filter: str | None = None) -> int: - await self._ensure_conn() - where_clauses = ['"type" = ?'] - query_params: list[str] = [type_value] + await self.get_connection() + where_clauses: list[str] = ['"type" = :type_value'] + params: dict[str, str] = {"type_value": type_value} if status_filter: if status_filter.startswith("!"): status_value = status_filter[1:] - where_clauses.append("json_extract(data, '$.status') != ?") - query_params.append(status_value) + where_clauses.append("json_extract(data, '$.status') != :status") + params["status"] = status_value else: - where_clauses.append("json_extract(data, '$.status') = ?") - query_params.append(status_filter) + where_clauses.append("json_extract(data, '$.status') = :status") + params["status"] = status_filter + + where_clause: str = " AND ".join(where_clauses) + count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 + + result = await self._conn.execute(text(count_query), params) + row = result.mappings().first() - where_clause = " AND ".join(where_clauses) - count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 - cursor = await self._conn.execute(count_query, tuple(query_params)) - row = await cursor.fetchone() return row["count"] if row else 0 async def paginate( @@ -259,74 +304,79 @@ class SqliteStore(metaclass=ThreadSafe): order: str, status_filter: str | None = None, ) -> tuple[list[tuple[str, ItemDTO]], int, int, int]: - await self._ensure_conn() - where_clauses = ['"type" = ?'] - query_params: list[str | int] = [type_value] + await self.get_connection() + where_clauses: list[str] = ['"type" = :type_value'] + params: dict[str, str | int] = {"type_value": type_value} if status_filter: if status_filter.startswith("!"): status_value = status_filter[1:] - where_clauses.append("json_extract(data, '$.status') != ?") - query_params.append(status_value) + where_clauses.append("json_extract(data, '$.status') != :status") + params["status"] = status_value else: - where_clauses.append("json_extract(data, '$.status') = ?") - query_params.append(status_filter) + where_clauses.append("json_extract(data, '$.status') = :status") + params["status"] = status_filter + + where_clause: str = " AND ".join(where_clauses) + count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 + + result = await self._conn.execute(text(count_query), params) + count_row = result.mappings().first() - where_clause = " AND ".join(where_clauses) - count_query = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608 - count_cursor = await self._conn.execute(count_query, tuple(query_params)) - count_row = await count_cursor.fetchone() total_items = count_row["count"] if count_row else 0 total_pages = (total_items + per_page - 1) // per_page if total_items > 0 else 1 if page > total_pages and total_items > 0: page = total_pages - offset = (page - 1) * per_page - query_params.extend([per_page, offset]) + offset: int = (page - 1) * per_page + params["limit"] = per_page + params["offset"] = offset - cursor = await self._conn.execute( - f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT ? OFFSET ?', # noqa: S608 - tuple(query_params), - ) + query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT :limit OFFSET :offset' # noqa: S608 + + result = await self._conn.execute(text(query), params) + rows = result.mappings().all() items: list[tuple[str, ItemDTO]] = [] - async with cursor: - async for row in cursor: - row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 - data = json.loads(row["data"]) - data.pop("_id", None) - item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) - item._id = row["id"] - item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) - items.append((row["id"], item)) + for row in rows: + row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007 + data = json.loads(row["data"]) + data.pop("_id", None) + item = init_class(ItemDTO, data, ITEM_DTO_FIELDS) + item._id = row["id"] + item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp()) + items.append((row["id"], item)) return items, total_items, page, total_pages async def upsert(self, type_value: str, item: ItemDTO) -> None: - await self._ensure_conn() + await self.get_connection() await self._upsert_now(type_value, item) async def delete(self, type_value: str, key: str) -> None: - await self._ensure_conn() - await self._conn.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (type_value, key)) + await self.get_connection() + await self._conn.execute( + text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'), + {"type_value": type_value, "key": key}, + ) + await self._conn.commit() async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int: - await self._ensure_conn() - keys_list = list(keys) + await self.get_connection() + keys_list: list[str] = list(keys) if not keys_list: return 0 - placeholders = ",".join("?" for _ in keys_list) - params = [type_value, *keys_list] - cur = await self._conn.execute( - f'DELETE FROM "history" WHERE "type" = ? AND "id" IN ({placeholders})', # noqa: S608 - tuple(params), - ) - return cur.rowcount if cur else 0 + placeholders = ",".join(f":key_{i}" for i in range(len(keys_list))) + params: dict[str, str] = {"type_value": type_value} + params.update({f"key_{i}": key for i, key in enumerate(keys_list)}) - async def commit(self) -> None: - if self._conn: - await self._conn.commit() + result = await self._conn.execute( + text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608 + params, + ) + await self._conn.commit() + return result.rowcount if result else 0 async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None: await self._enqueue(_Op("upsert", type_value, item, None, None)) @@ -340,11 +390,9 @@ class SqliteStore(metaclass=ThreadSafe): async def flush(self) -> None: if self._queue: await self._queue.join() - if self._conn: - await self._conn.commit() async def shutdown(self) -> None: - """Flush pending writes, stop writer task, and commit.""" + """Flush pending writes and stop writer task.""" if self._queue: try: await self._queue.put(_Op(Terminator(), "", None, None, None)) @@ -360,20 +408,22 @@ class SqliteStore(metaclass=ThreadSafe): with contextlib.suppress(asyncio.CancelledError): await asyncio.wait_for(self._task, timeout=2) - if self._conn: - LOG.debug("Committing final changes to SqliteStore...") - await self._conn.commit() - self._task = None self._queue = None async def close(self) -> None: await self.shutdown() + if self._conn: - LOG.debug("Closing SqliteStore connection...") await self._conn.close() self._conn = None + if self._engine: + await self._engine.dispose() + self._engine = None + + self._sessionmaker = None + async def _enqueue(self, op: _Op) -> None: self._ensure_worker() await self._queue.put(op) @@ -399,45 +449,101 @@ class SqliteStore(metaclass=ThreadSafe): await asyncio.sleep(self._flush_interval) async def _apply(self, op: _Op) -> None: - await self._ensure_conn() + await self.get_connection() if op.op == "upsert" and op.item: - await self._upsert_now(op.type_value, op.item) + await self._upsert_now_conn(self._conn, op.type_value, op.item) + await self._conn.commit() elif op.op == "delete" and op.key: - await self._conn.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (op.type_value, op.key)) - elif op.op == "bulk_delete" and op.keys: - placeholders = ",".join("?" for _ in op.keys) - params = [op.type_value, *op.keys] await self._conn.execute( - f'DELETE FROM "history" WHERE "type" = ? AND "id" IN ({placeholders})', # noqa: S608 - tuple(params), + text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'), + {"type_value": op.type_value, "key": op.key}, ) - await self._conn.commit() + await self._conn.commit() + elif op.op == "bulk_delete" and op.keys: + placeholders = ",".join(f":key_{i}" for i in range(len(op.keys))) + params: dict[str, str] = {"type_value": op.type_value} + params.update({f"key_{i}": key for i, key in enumerate(op.keys)}) + await self._conn.execute( + text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608 + params, + ) + await self._conn.commit() async def _upsert_now(self, type_value: str, item: ItemDTO) -> None: - await self._ensure_conn() + await self.get_connection() + await self._upsert_now_conn(self._conn, type_value, item) + await self._conn.commit() + + async def _upsert_now_conn(self, conn, type_value: str, item: ItemDTO) -> None: + """Helper to upsert using an existing connection.""" sql = """ INSERT INTO "history" ("id", "type", "url", "data") - VALUES (?, ?, ?, ?) - ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?, created_at = ? + VALUES (:id, :type, :url, :data) + ON CONFLICT DO UPDATE SET "type" = :type2, "url" = :url2, "data" = :data2, created_at = :created_at """ encoded = item.json() - await self._conn.execute( - sql.strip(), - ( - item._id, - type_value, - item.url, - encoded, - type_value, - item.url, - encoded, - datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), - ), + await conn.execute( + text(sql.strip()), + { + "id": item._id, + "type": type_value, + "url": item.url, + "data": encoded, + "type2": type_value, + "url2": item.url, + "data2": encoded, + "created_at": datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), + }, ) - async def _ensure_conn(self) -> None: + async def execute_raw(self, query: str, params: dict | tuple | None = None) -> None: + """Execute raw SQL query (for testing purposes).""" + await self.get_connection() + if isinstance(params, tuple): + # Convert positional params to dict for SQLAlchemy + # Assuming queries use ? placeholders, we need to count them + placeholders = query.count("?") + if placeholders != len(params): + msg = ( + f"Parameter count mismatch: query has {placeholders} placeholders but {len(params)} params provided" + ) + raise ValueError(msg) + # Create numbered parameters + param_dict = {f"p{i}": params[i] for i in range(len(params))} + # Replace ? with :p0, :p1, etc. + for i in range(len(params)): + query = query.replace("?", f":p{i}", 1) + await self._conn.execute(text(query), param_dict) + elif isinstance(params, dict): + await self._conn.execute(text(query), params) + else: + await self._conn.execute(text(query)) + await self._conn.commit() + + async def fetch_raw(self, query: str, params: dict | tuple | None = None): + """Fetch results from raw SQL query (for testing purposes).""" + await self.get_connection() + if isinstance(params, tuple): + # Convert positional params to dict for SQLAlchemy + placeholders = query.count("?") + if placeholders != len(params): + msg = ( + f"Parameter count mismatch: query has {placeholders} placeholders but {len(params)} params provided" + ) + raise ValueError(msg) + param_dict = {f"p{i}": params[i] for i in range(len(params))} + for i in range(len(params)): + query = query.replace("?", f":p{i}", 1) + result = await self._conn.execute(text(query), param_dict) + elif isinstance(params, dict): + result = await self._conn.execute(text(query), params) + else: + result = await self._conn.execute(text(query)) + return result.mappings().all() + + async def get_connection(self) -> AsyncConnection: if self._conn: - return + return self._conn if not self._db_path: msg = "No database path specified for SqliteStore." @@ -446,21 +552,31 @@ class SqliteStore(metaclass=ThreadSafe): from app.library import migrate from app.main import ROOT_PATH - if not self._db_path.startswith(":memory"): + if self._db_path.startswith(":memory"): + db_url: str = "sqlite+aiosqlite:///file::memory:?cache=shared&uri=true" + else: os.makedirs(os.path.dirname(self._db_path) or ".", exist_ok=True) + db_url: str = f"sqlite+aiosqlite:///{self._db_path}" - self._conn = await aiosqlite.connect(database=self._db_path, isolation_level=None) - self._conn.row_factory = aiosqlite.Row - version = await migrate.get_version(self._conn) - if version: + self._engine = create_async_engine( + db_url, + echo=False, + connect_args={"check_same_thread": False, "uri": self._db_path.startswith(":memory")}, + ) + self._conn = await self._engine.connect() + + if version := await migrate.get_version(self._conn): LOG.debug(f"DB Version: '{version}'.") await migrate.upgrade(self._conn, ROOT_PATH / "migrations") if not version: - version = await migrate.get_version(self._conn) - LOG.debug(f"DB Version after initial migration: '{version}'.") + LOG.debug(f"DB Version after initial migration: '{await migrate.get_version(self._conn)}'.") - await self._conn.execute("PRAGMA journal_mode=wal") - await self._conn.execute("PRAGMA busy_timeout=5000") - await self._conn.execute("PRAGMA foreign_keys=ON") + await self._conn.execute(text("PRAGMA journal_mode=wal")) + await self._conn.execute(text("PRAGMA busy_timeout=5000")) + await self._conn.execute(text("PRAGMA foreign_keys=ON")) await self._conn.commit() + + self._sessionmaker = async_sessionmaker(bind=self._engine, class_=AsyncSession, expire_on_commit=False) + + return self._conn diff --git a/app/main.py b/app/main.py index c9860b63..6dfe8dde 100644 --- a/app/main.py +++ b/app/main.py @@ -14,21 +14,22 @@ from pathlib import Path import magic from aiohttp import web +from app.features.conditions.service import Conditions +from app.features.dl_fields.service import DLFields +from app.features.notifications.service import Notifications +from app.features.presets.deps import get_presets_repo +from app.features.tasks.definitions.deps import get_task_definitions_repo +from app.features.tasks.service import Tasks from app.library.BackgroundWorker import BackgroundWorker -from app.library.conditions import Conditions +from app.library.cache import Cache from app.library.config import Config -from app.library.dl_fields import DLFields from app.library.downloads import DownloadQueue from app.library.Events import EventBus, Events from app.library.HttpAPI import HttpAPI from app.library.HttpSocket import HttpSocket -from app.library.Notifications import Notification -from app.library.Presets import Presets from app.library.Scheduler import Scheduler from app.library.Services import Services from app.library.sqlite_store import SqliteStore -from app.library.TaskDefinitions import TaskDefinitions -from app.library.Tasks import Tasks from app.library.UpdateChecker import UpdateChecker LOG = logging.getLogger("app") @@ -110,16 +111,17 @@ class Main: SqliteStore.get_instance(db_path=self._config.db_file).attach(self._app) BackgroundWorker.get_instance().attach(self._app) Scheduler.get_instance().attach(self._app) + Cache.get_instance().attach(self._app) self._socket.attach(self._app) self._http.attach(self._app) - Presets.get_instance().attach(self._app) + get_presets_repo().attach(self._app) Tasks.get_instance().attach(self._app) - Notification.get_instance().attach(self._app) + Notifications.get_instance().attach(self._app) Conditions.get_instance().attach(self._app) DLFields.get_instance().attach(self._app) - TaskDefinitions.get_instance().attach(self._app) + get_task_definitions_repo().attach(self._app) DownloadQueue.get_instance().attach(self._app) UpdateChecker.get_instance().attach(self._app) diff --git a/app/migrations/20231115152938_initial.py b/app/migrations/20231115152938_initial.py index 043327db..c71f2759 100644 --- a/app/migrations/20231115152938_initial.py +++ b/app/migrations/20231115152938_initial.py @@ -5,28 +5,25 @@ Migration Name: initial Migration Version: 20231115152938 """ +from sqlalchemy import text + async def upgrade(c): - sql = """ - CREATE TABLE "history" ( - "id" TEXT PRIMARY KEY UNIQUE NOT NULL, - "type" TEXT NOT NULL, - "url" TEXT NOT NULL, - "data" JSON NOT NULL, - "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP - ); - """ - await c.execute(sql) - - sql = """ - CREATE INDEX "history_type" ON "history" ("type"); - """ - await c.execute(sql) - - sql = """ - CREATE UNIQUE INDEX "history_url" ON "history" ("url"); - """ - await c.execute(sql) + sql: list[str] = [ + """ + CREATE TABLE "history" ( + "id" TEXT PRIMARY KEY UNIQUE NOT NULL, + "type" TEXT NOT NULL, + "url" TEXT NOT NULL, + "data" JSON NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + """, + 'CREATE INDEX "history_type" ON "history" ("type");', + 'CREATE UNIQUE INDEX "history_url" ON "history" ("url");', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) await c.commit() @@ -36,4 +33,4 @@ async def downgrade(c): DROP TABLE IF EXISTS "history"; """ - await c.execute(sql) + await c.execute(text(sql)) diff --git a/app/migrations/20251116173731_add_status_index.py b/app/migrations/20251116173731_add_status_index.py index 0778b414..588c3a12 100644 --- a/app/migrations/20251116173731_add_status_index.py +++ b/app/migrations/20251116173731_add_status_index.py @@ -5,8 +5,10 @@ Migration Name: add_status_index Migration Version: 20251116173731 """ +from sqlalchemy import text -async def upgrade(connection): + +async def upgrade(c): """ Add index on json_extract(data, '$.status') for better query performance. @@ -16,14 +18,14 @@ async def upgrade(connection): sql = """ CREATE INDEX IF NOT EXISTS "history_status" ON "history" (json_extract("data", '$.status')); """ - await connection.execute(sql) + await c.execute(text(sql)) -async def downgrade(connection): +async def downgrade(c): """ Remove the status index. """ sql = """ DROP INDEX IF EXISTS "history_status"; """ - await connection.execute(sql) + await c.execute(text(sql)) diff --git a/app/migrations/20260117184822_add_conditions_table.py b/app/migrations/20260117184822_add_conditions_table.py new file mode 100644 index 00000000..7b30f7d0 --- /dev/null +++ b/app/migrations/20260117184822_add_conditions_table.py @@ -0,0 +1,39 @@ +""" +This module contains a db migration. + +Migration Name: add_conditions_table +Migration Version: 20260117184822 +""" + +from sqlalchemy import text + + +async def upgrade(c): + sql: list[str] = [ + """ + CREATE TABLE IF NOT EXISTS "conditions" ( + "id" INTEGER PRIMARY KEY AUTOINCREMENT, + "name" TEXT NOT NULL UNIQUE, + "filter" TEXT NOT NULL, + "cli" TEXT NOT NULL DEFAULT '', + "extras" JSON NOT NULL DEFAULT '{}', + "enabled" INTEGER NOT NULL DEFAULT 1, + "priority" INTEGER NOT NULL DEFAULT 0, + "description" TEXT NOT NULL DEFAULT '', + "created_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + """, + 'CREATE INDEX IF NOT EXISTS "ix_conditions_name" ON "conditions" ("name");', + 'CREATE INDEX IF NOT EXISTS "ix_conditions_enabled" ON "conditions" ("enabled");', + 'CREATE INDEX IF NOT EXISTS "ix_conditions_priority" ON "conditions" ("priority");', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) + + +async def downgrade(c): + sql = """ + DROP TABLE IF EXISTS "conditions"; + """ + await c.execute(text(sql)) diff --git a/app/migrations/20260119162444_add_dl_fields_table.py b/app/migrations/20260119162444_add_dl_fields_table.py new file mode 100644 index 00000000..d318976b --- /dev/null +++ b/app/migrations/20260119162444_add_dl_fields_table.py @@ -0,0 +1,40 @@ +""" +This module contains a db migration. + +Migration Name: add_dl_fields_table +Migration Version: 20260119162444 +""" + +from sqlalchemy import text + + +async def upgrade(c): + sql: list[str] = [ + """ + CREATE TABLE IF NOT EXISTS "dl_fields" ( + "id" INTEGER PRIMARY KEY AUTOINCREMENT, + "name" TEXT NOT NULL UNIQUE, + "description" TEXT NOT NULL DEFAULT '', + "field" TEXT NOT NULL, + "kind" TEXT NOT NULL DEFAULT 'text', + "icon" TEXT NOT NULL DEFAULT '', + "order" INTEGER NOT NULL DEFAULT 0, + "value" TEXT NOT NULL DEFAULT '', + "extras" JSON NOT NULL DEFAULT '{}', + "created_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + """, + 'CREATE INDEX IF NOT EXISTS "ix_dl_fields_name" ON "dl_fields" ("name");', + 'CREATE INDEX IF NOT EXISTS "ix_dl_fields_order" ON "dl_fields" ("order");', + 'CREATE INDEX IF NOT EXISTS "ix_dl_fields_kind" ON "dl_fields" ("kind");', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) + + +async def downgrade(c): + sql = """ + DROP TABLE IF EXISTS "dl_fields"; + """ + await c.execute(text(sql)) diff --git a/app/migrations/20260121134838_add_notifications_table.py b/app/migrations/20260121134838_add_notifications_table.py new file mode 100644 index 00000000..0ef80bb5 --- /dev/null +++ b/app/migrations/20260121134838_add_notifications_table.py @@ -0,0 +1,40 @@ +""" +This module contains a db migration. + +Migration Name: add_notifications_table +Migration Version: 20260121134838 +""" + +from sqlalchemy import text + + +async def upgrade(c): + sql: list[str] = [ + """ + CREATE TABLE IF NOT EXISTS "notifications" ( + "id" INTEGER PRIMARY KEY AUTOINCREMENT, + "name" TEXT NOT NULL UNIQUE, + "on" JSON NOT NULL DEFAULT '[]', + "presets" JSON NOT NULL DEFAULT '[]', + "enabled" BOOLEAN NOT NULL DEFAULT 1, + "request_url" TEXT NOT NULL, + "request_method" TEXT NOT NULL DEFAULT 'POST', + "request_type" TEXT NOT NULL DEFAULT 'json', + "request_data_key" TEXT NOT NULL DEFAULT 'data', + "request_headers" JSON NOT NULL DEFAULT '[]', + "created_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + """, + 'CREATE INDEX IF NOT EXISTS "ix_notifications_name" ON "notifications" ("name");', + 'CREATE INDEX IF NOT EXISTS "ix_notifications_enabled" ON "notifications" ("enabled");', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) + + +async def downgrade(c): + sql = """ + DROP TABLE IF EXISTS "notifications"; + """ + await c.execute(text(sql)) diff --git a/app/migrations/20260122201858_add_task_definitions.py b/app/migrations/20260122201858_add_task_definitions.py new file mode 100644 index 00000000..927373db --- /dev/null +++ b/app/migrations/20260122201858_add_task_definitions.py @@ -0,0 +1,38 @@ +""" +This module contains a db migration. + +Migration Name: add_task_definitions +Migration Version: 20260122201858 +""" + +from sqlalchemy import text + + +async def upgrade(c): + sql: list[str] = [ + """ + CREATE TABLE IF NOT EXISTS "task_definitions" ( + "id" INTEGER PRIMARY KEY AUTOINCREMENT, + "name" TEXT NOT NULL UNIQUE, + "priority" INTEGER NOT NULL DEFAULT 0, + "enabled" BOOLEAN NOT NULL DEFAULT 1, + "match_url" JSON NOT NULL, + "definition" JSON NOT NULL DEFAULT '{}', + "created_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + """, + 'CREATE INDEX IF NOT EXISTS "ix_task_definitions_name" ON "task_definitions" ("name");', + 'CREATE INDEX IF NOT EXISTS "ix_task_definitions_priority" ON "task_definitions" ("priority");', + 'CREATE INDEX IF NOT EXISTS "ix_task_definitions_match_url" ON "task_definitions" ("match_url");', + 'CREATE INDEX IF NOT EXISTS "ix_task_definitions_enabled" ON "task_definitions" ("enabled");', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) + + +async def downgrade(c): + sql = """ + DROP TABLE IF EXISTS "task_definitions"; + """ + await c.execute(text(sql)) diff --git a/app/migrations/20260124171740_add_tasks_table.py b/app/migrations/20260124171740_add_tasks_table.py new file mode 100644 index 00000000..7ff56c81 --- /dev/null +++ b/app/migrations/20260124171740_add_tasks_table.py @@ -0,0 +1,46 @@ +""" +This module contains a db migration. + +Migration Name: add_tasks_table +Migration Version: 20260124171740 +""" + +from sqlalchemy import text + + +async def upgrade(c): + sql: list[str] = [ + """ + CREATE TABLE IF NOT EXISTS tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + url TEXT NOT NULL, + folder TEXT NOT NULL DEFAULT '', + preset TEXT NOT NULL DEFAULT '', + timer TEXT NOT NULL DEFAULT '', + template TEXT NOT NULL DEFAULT '', + cli TEXT NOT NULL DEFAULT '', + auto_start INTEGER NOT NULL DEFAULT 1, + handler_enabled INTEGER NOT NULL DEFAULT 1, + enabled INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + """, + 'CREATE INDEX IF NOT EXISTS "ix_tasks_name" ON "tasks" ("name")', + 'CREATE INDEX IF NOT EXISTS "ix_tasks_enabled" ON "tasks" ("enabled")', + 'CREATE INDEX IF NOT EXISTS "ix_tasks_timer" ON "tasks" ("timer")', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) + + +async def downgrade(c): + sql: list[str] = [ + 'DROP INDEX IF EXISTS "ix_tasks_name"', + 'DROP INDEX IF EXISTS "ix_tasks_enabled"', + 'DROP INDEX IF EXISTS "ix_tasks_timer"', + 'DROP TABLE IF EXISTS "tasks"', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) diff --git a/app/migrations/20260125125900_add_presets_table.py b/app/migrations/20260125125900_add_presets_table.py new file mode 100644 index 00000000..e2b496d6 --- /dev/null +++ b/app/migrations/20260125125900_add_presets_table.py @@ -0,0 +1,44 @@ +""" +This module contains a db migration. + +Migration Name: add_presets_table +Migration Version: 20260125125900 +""" + +from sqlalchemy import text + + +async def upgrade(c): + sql: list[str] = [ + """ + CREATE TABLE IF NOT EXISTS presets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + description TEXT NOT NULL DEFAULT '', + folder TEXT NOT NULL DEFAULT '', + template TEXT NOT NULL DEFAULT '', + cookies TEXT NOT NULL DEFAULT '', + cli TEXT NOT NULL DEFAULT '', + is_default INTEGER NOT NULL DEFAULT 0, + priority INTEGER NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """, + 'CREATE INDEX IF NOT EXISTS "ix_presets_name" ON "presets" ("name")', + 'CREATE INDEX IF NOT EXISTS "ix_presets_is_default" ON "presets" ("is_default")', + 'CREATE INDEX IF NOT EXISTS "ix_presets_priority" ON "presets" ("priority")', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) + + +async def downgrade(c): + sql: list[str] = [ + 'DROP INDEX IF EXISTS "ix_presets_name"', + 'DROP INDEX IF EXISTS "ix_presets_is_default"', + 'DROP INDEX IF EXISTS "ix_presets_priority"', + 'DROP TABLE IF EXISTS "presets"', + ] + for sql_stmt in sql: + await c.execute(text(sql_stmt)) diff --git a/app/routes/api/archiver.py b/app/routes/api/archiver.py index c732f5a9..4183deda 100644 --- a/app/routes/api/archiver.py +++ b/app/routes/api/archiver.py @@ -4,8 +4,8 @@ from collections.abc import Iterable from aiohttp import web from aiohttp.web import Request, Response +from app.features.presets.service import Presets from app.library.Archiver import Archiver -from app.library.Presets import Presets from app.library.router import route from app.library.YTDLPOpts import YTDLPOpts diff --git a/app/routes/api/conditions.py b/app/routes/api/conditions.py index eecc001b..33c43457 100644 --- a/app/routes/api/conditions.py +++ b/app/routes/api/conditions.py @@ -1,215 +1,3 @@ -import asyncio -import functools -import logging -import uuid -from collections import OrderedDict +"""Migrated API routes for feature submodule.""" -from aiohttp import web -from aiohttp.web import Request, Response - -from app.library.cache import Cache -from app.library.conditions import Condition, Conditions -from app.library.config import Config -from app.library.encoder import Encoder -from app.library.Events import EventBus, Events -from app.library.mini_filter import match_str -from app.library.router import route -from app.library.Utils import extract_info, init_class, validate_uuid -from app.library.YTDLPOpts import YTDLPOpts - -LOG: logging.Logger = logging.getLogger(__name__) - - -@route("GET", "api/conditions/", "conditions_list") -async def conditions_list(encoder: Encoder) -> Response: - """ - Get the conditions - - Args: - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object. - - """ - return web.json_response( - data=Conditions.get_instance().get_all(), - status=web.HTTPOk.status_code, - dumps=encoder.encode, - ) - - -@route("PUT", "api/conditions/", "conditions_add") -async def conditions_add(request: Request, encoder: Encoder, notify: EventBus) -> Response: - """ - Save Conditions. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - notify (EventBus): The event bus instance. - - Returns: - Response: The response object - - """ - data = await request.json() - - if not isinstance(data, list): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - items: list = [] - - cls = Conditions.get_instance() - - for item in data: - if not isinstance(item, dict): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("name"): - return web.json_response( - {"error": "Name is required.", "data": item}, status=web.HTTPBadRequest.status_code - ) - - if not item.get("filter"): - return web.json_response( - {"error": "Filter is required.", "data": item}, status=web.HTTPBadRequest.status_code - ) - - if not item.get("cli") and len(item.get("extras", {}).keys()) < 1: - return web.json_response( - {"error": "A Condition Must have cli options or extras", "data": item}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("cli"): - item["cli"] = "" - - if not item.get("extras"): - item["extras"] = {} - - if "enabled" not in item: - item["enabled"] = True - - if "priority" not in item: - item["priority"] = 0 - - if "description" not in item: - item["description"] = "" - - if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): - item["id"] = str(uuid.uuid4()) - - try: - cls.validate(item) - except ValueError as e: - return web.json_response( - {"error": f"Failed to validate condition '{item.get('name')}'. '{e!s}'"}, - status=web.HTTPBadRequest.status_code, - ) - - items.append(init_class(Condition, item)) - - try: - items = cls.save(items=items).load().get_all() - except Exception as e: - LOG.exception(e) - return web.json_response( - {"error": "Failed to save conditions.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) - - notify.emit(Events.CONDITIONS_UPDATE, data=items) - return web.json_response(data=items, status=web.HTTPOk.status_code, dumps=encoder.encode) - - -@route("POST", "api/conditions/test/", "conditions_test") -async def conditions_test(request: Request, encoder: Encoder, cache: Cache, config: Config) -> Response: - """ - Test condition against URL. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - cache (Cache): The cache instance. - config (Config): The configuration instance. - - Returns: - Response: The response object - - """ - params = await request.json() - - if not isinstance(params, dict): - return web.json_response( - {"error": "Invalid request body expecting dict."}, - status=web.HTTPBadRequest.status_code, - ) - - url: str | None = params.get("url") - if not url: - return web.json_response({"error": "url is required."}, status=web.HTTPBadRequest.status_code) - - cond: str | None = params.get("condition") - if not cond: - return web.json_response({"error": "condition is required."}, status=web.HTTPBadRequest.status_code) - - try: - preset: str = params.get("preset", config.default_preset) - key: str = cache.hash(url + str(preset)) - if not cache.has(key): - data: dict | None = await asyncio.wait_for( - fut=asyncio.get_running_loop().run_in_executor( - None, - functools.partial( - extract_info, - config=YTDLPOpts.get_instance().preset(name=preset).get_all(), - url=url, - debug=False, - no_archive=True, - follow_redirect=True, - sanitize_info=True, - ), - ), - timeout=120, - ) - - if not data: - return web.json_response( - data={"error": f"Failed to extract info from '{url!s}'."}, - status=web.HTTPBadRequest.status_code, - ) - cache.set(key=key, value=data, ttl=600) - else: - data = cache.get(key) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": f"Failed to extract video info. '{e!s}'"}, - status=web.HTTPInternalServerError.status_code, - ) - - try: - status: bool = match_str(cond, data) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": str(e)}, - status=web.HTTPBadRequest.status_code, - ) - - return web.json_response( - data={ - "status": status, - "condition": cond, - "data": OrderedDict(sorted(data.items(), key=lambda item: len(str(item[1])))), - }, - status=web.HTTPOk.status_code, - dumps=encoder.encode, - ) +import app.features.conditions.router # noqa: F401 diff --git a/app/routes/api/dl_fields.py b/app/routes/api/dl_fields.py index c8b0fac2..5bd99bb4 100644 --- a/app/routes/api/dl_fields.py +++ b/app/routes/api/dl_fields.py @@ -1,100 +1,3 @@ -import logging -import uuid +"""Migrated API routes for dl_fields feature.""" -from aiohttp import web -from aiohttp.web import Request, Response - -from app.library.dl_fields import DLField, DLFields -from app.library.encoder import Encoder -from app.library.Events import EventBus, Events -from app.library.router import route -from app.library.Utils import init_class, validate_uuid - -LOG: logging.Logger = logging.getLogger(__name__) - - -@route("GET", "api/dl_fields/", "dl_fields") -async def dl_fields(request: Request, encoder: Encoder) -> Response: - """ - Get the dl_fields. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object. - - """ - data: list[DLField] = DLFields.get_instance().get_all() - filter_fields: str | None = request.query.get("filter", None) - - if filter_fields: - fields: list[str] = [field.strip() for field in filter_fields.split(",")] - data = [{key: value for key, value in preset.serialize().items() if key in fields} for preset in data] - - return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) - - -@route("PUT", "api/dl_fields/", "dl_fields_add") -async def dl_fields_add(request: Request, encoder: Encoder, notify: EventBus) -> Response: - """ - Add presets. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - notify (EventBus): The event bus instance. - - Returns: - Response: The response object - - """ - data = await request.json() - - if not isinstance(data, list): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - items: list[DLField] = [] - - cls = DLFields.get_instance() - - for item in data: - if not isinstance(item, dict): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("name"): - return web.json_response( - {"error": "name is required.", "data": item}, status=web.HTTPBadRequest.status_code - ) - - if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): - item["id"] = str(uuid.uuid4()) - - try: - cls.validate(item) - except ValueError as e: - return web.json_response( - {"error": f"Failed to validate preset '{item.get('name')}'. '{e!s}'"}, - status=web.HTTPBadRequest.status_code, - ) - - items.append(init_class(DLField, item)) - - try: - items = cls.save(items=items).load().get_all() - except Exception as e: - LOG.exception(e) - return web.json_response( - {"error": "Failed to save download fields.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) - - notify.emit(Events.DLFIELDS_UPDATE, data=items) - return web.json_response(data=items, status=web.HTTPOk.status_code, dumps=encoder.encode) +import app.features.dl_fields.router # noqa: F401 diff --git a/app/routes/api/history.py b/app/routes/api/history.py index f0e6e1be..d31f7d43 100644 --- a/app/routes/api/history.py +++ b/app/routes/api/history.py @@ -5,13 +5,14 @@ from typing import TYPE_CHECKING, Any from aiohttp import web from aiohttp.web import Request, Response +from app.features.presets.schemas import Preset +from app.features.presets.service import Presets from app.library.config import Config from app.library.DataStore import StoreType from app.library.downloads import Download, DownloadQueue from app.library.encoder import Encoder from app.library.Events import EventBus, Events from app.library.ItemDTO import Item -from app.library.Presets import Preset, Presets from app.library.router import route if TYPE_CHECKING: @@ -116,6 +117,29 @@ async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, c ) +@route("GET", "api/history/live", "items_live") +async def items_live(queue: DownloadQueue, encoder: Encoder) -> Response: + """ + Get live queue data + + Args: + queue (DownloadQueue): The download queue instance. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object with live queue items. + + """ + return web.json_response( + data={ + "queue": (await queue.get("queue"))["queue"], + "history_count": await queue.done.get_total_count(), + }, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + @route("DELETE", "api/history/", "items_delete") async def items_delete(request: Request, queue: DownloadQueue, encoder: Encoder) -> Response: """ @@ -361,33 +385,144 @@ async def items_add(request: Request, queue: DownloadQueue, encoder: Encoder) -> Returns: Response: The response object. + Query Parameters: + sync (bool): If true, wait for all items to be processed synchronously. Default: false + """ data = await request.json() if isinstance(data, dict): data = [data] - items = [] + items: list[Item] = [] for item in data: try: items.append(Item.format(item)) except ValueError as e: return web.json_response(data={"error": str(e), "data": item}, status=web.HTTPBadRequest.status_code) - status: list[dict] = await asyncio.wait_for( - fut=asyncio.gather(*[queue.add(item=item) for item in items]), - timeout=None, + if "true" == request.query.get("sync", "false").lower(): + status: list[dict] = await asyncio.wait_for( + fut=asyncio.gather(*[queue.add(item=item) for item in items]), + timeout=None, + ) + + response: list[dict[str, Any]] = [] + + for i, item in enumerate(items): + it = {"item": item, "status": "ok" == status[i].get("status"), "msg": status[i].get("msg")} + if status[i].get("hidden"): + it["hidden"] = True + response.append(it) + + return web.json_response(data=response, status=web.HTTPOk.status_code, dumps=encoder.encode) + + from app.library.downloads.utils import handle_task_exception + + batch_id: str = f"batch_{asyncio.get_running_loop().time():.0f}" + + for idx, item in enumerate(items): + if not item.extras: + item.extras = {} + + item.extras["batch_id"] = batch_id + item.extras["batch_index"] = idx + item.extras["batch_total"] = len(items) + + task = asyncio.create_task( + queue.add(item=item), + name=f"bulk_add_{batch_id}_{idx}", + ) + task.add_done_callback(lambda t: handle_task_exception(t, LOG)) + + return web.json_response( + data={ + "status": "accepted", + "message": f"Accepted {len(items)} item(s) for processing", + "batch_id": batch_id, + "count": len(items), + }, + status=web.HTTPAccepted.status_code, + dumps=encoder.encode, ) - response: list[dict[str, Any]] = [] - for i, item in enumerate(items): - it = {"item": item, "status": "ok" == status[i].get("status"), "msg": status[i].get("msg")} - if status[i].get("hidden"): - it["hidden"] = True - response.append(it) +@route("POST", "api/history/start", "items_start") +async def items_start(request: Request, queue: DownloadQueue, encoder: Encoder) -> Response: + """ + Start one or more queued downloads. - return web.json_response(data=response, status=web.HTTPOk.status_code, dumps=encoder.encode) + Args: + request (Request): The request object. + queue (DownloadQueue): The download queue instance. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + data = await request.json() + if not (ids := data.get("ids", [])): + return web.json_response(data={"error": "ids array is required."}, status=web.HTTPBadRequest.status_code) + + if not isinstance(ids, list): + return web.json_response(data={"error": "ids must be an array."}, status=web.HTTPBadRequest.status_code) + + status: dict[str, str] = await queue.start_items(ids) + + return web.json_response(data=status, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("POST", "api/history/pause", "items_pause") +async def items_pause(request: Request, queue: DownloadQueue, encoder: Encoder) -> Response: + """ + Pause one or more queued downloads. + + Args: + request (Request): The request object. + queue (DownloadQueue): The download queue instance. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + data = await request.json() + if not (ids := data.get("ids", [])): + return web.json_response(data={"error": "ids array is required."}, status=web.HTTPBadRequest.status_code) + + if not isinstance(ids, list): + return web.json_response(data={"error": "ids must be an array."}, status=web.HTTPBadRequest.status_code) + + status: dict[str, str] = await queue.pause_items(ids) + + return web.json_response(data=status, status=web.HTTPOk.status_code, dumps=encoder.encode) + + +@route("POST", "api/history/cancel", "items_cancel") +async def items_cancel(request: Request, queue: DownloadQueue, encoder: Encoder) -> Response: + """ + Cancel one or more queued downloads. + + Args: + request (Request): The request object. + queue (DownloadQueue): The download queue instance. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + data = await request.json() + if not (ids := data.get("ids", [])): + return web.json_response(data={"error": "ids array is required."}, status=web.HTTPBadRequest.status_code) + + if not isinstance(ids, list): + return web.json_response(data={"error": "ids must be an array."}, status=web.HTTPBadRequest.status_code) + + status: dict[str, str] = await queue.cancel(ids) + + return web.json_response(data=status, status=web.HTTPOk.status_code, dumps=encoder.encode) @route("POST", r"api/history/{id}/archive", "history.item.archive.add") diff --git a/app/routes/api/logs.py b/app/routes/api/logs.py index 6e00a23b..073d4fc4 100644 --- a/app/routes/api/logs.py +++ b/app/routes/api/logs.py @@ -1,3 +1,5 @@ +import asyncio +import logging from pathlib import Path from aiohttp import web @@ -6,7 +8,9 @@ from aiohttp.web import Request, Response from app.library.config import Config from app.library.encoder import Encoder from app.library.router import route -from app.library.Utils import read_logfile +from app.library.Utils import read_logfile, tail_log + +LOG: logging.Logger = logging.getLogger(__name__) @route("GET", "api/logs/", "logs") @@ -48,3 +52,56 @@ async def logs(request: Request, config: Config, encoder: Encoder) -> Response: status=web.HTTPOk.status_code, dumps=encoder.encode, ) + + +@route("GET", "api/logs/stream", "logs.stream") +async def stream_logs(request: Request, config: Config, encoder: Encoder) -> Response | web.StreamResponse: + if not config.file_logging: + return web.json_response( + data={"error": "File logging is not enabled."}, + status=web.HTTPNotFound.status_code, + ) + + log_file = Path(config.config_path) / "logs" / "app.log" + if not log_file.exists(): + return web.json_response( + data={"error": "Log file is not available."}, + status=web.HTTPNotFound.status_code, + ) + + response = web.StreamResponse( + status=web.HTTPOk.status_code, + headers={ + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + "Connection": "keep-alive", + }, + ) + await response.prepare(request) + + async def emit_log(data: dict) -> None: + if request.transport is None or request.transport.is_closing(): + raise asyncio.CancelledError + payload = f"event: log_lines\ndata: {encoder.encode(data)}\n\n" + await response.write(payload.encode("utf-8")) + + log_task: asyncio.Task[None] = asyncio.create_task(tail_log(file=log_file, emitter=emit_log), name="log_stream") + + try: + LOG.debug("Log streaming connected.") + while not log_task.done(): + await asyncio.sleep(0.5) + if request.transport is None or request.transport.is_closing(): + log_task.cancel() + break + await log_task + except asyncio.CancelledError: + pass + finally: + LOG.debug("Log streaming disconnected.") + try: + await response.write_eof() + except ConnectionResetError: + pass + + return response diff --git a/app/routes/api/notifications.py b/app/routes/api/notifications.py index 6174b62b..efb10124 100644 --- a/app/routes/api/notifications.py +++ b/app/routes/api/notifications.py @@ -1,108 +1,3 @@ -import logging -import uuid +"""Migrated API routes for feature submodule.""" -from aiohttp import web -from aiohttp.web import Request, Response - -from app.library.encoder import Encoder -from app.library.Events import EventBus, Events -from app.library.Notifications import Notification, NotificationEvents -from app.library.router import route -from app.library.Utils import validate_uuid - -LOG: logging.Logger = logging.getLogger(__name__) - - -@route("GET", "api/notifications/", "notifications_list") -async def notifications_list(encoder: Encoder) -> Response: - """ - Get the notification targets. - - Args: - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object. - - """ - return web.json_response( - data={ - "notifications": Notification.get_instance().get_targets(), - "allowedTypes": list(NotificationEvents.get_events().values()), - }, - status=web.HTTPOk.status_code, - dumps=encoder.encode, - ) - - -@route("PUT", "api/notifications/", "notification_add") -async def notification_add(request: Request, encoder: Encoder) -> Response: - """ - Add notification targets. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object. - - """ - post = await request.json() - if not isinstance(post, list): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - targets: list = [] - - ins: Notification = Notification.get_instance() - for item in post: - if not isinstance(item, dict): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("id", None) or validate_uuid(item.get("id"), version=4): - item["id"] = str(uuid.uuid4()) - - try: - Notification.validate(item) - except ValueError as e: - return web.json_response( - {"error": f"Invalid notification target settings. {e!s}", "data": item}, - status=web.HTTPBadRequest.status_code, - ) - - targets.append(ins.make_target(item)) - - try: - ins.save(targets=targets) - ins.load() - except Exception as e: - LOG.exception(e) - return web.json_response({"error": "Failed to save tasks."}, status=web.HTTPInternalServerError.status_code) - - data = {"notifications": targets, "allowedTypes": list(NotificationEvents.get_events().values())} - - return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) - - -@route("POST", "api/notifications/test/", "notification_test") -async def notification_test(encoder: Encoder, notify: EventBus) -> Response: - """ - Test the notification. - - Args: - encoder (Encoder): The encoder instance. - notify (EventBus): The event bus instance. - - Returns: - Response: The response object. - - """ - notify.emit(Events.TEST, title="Test Notification", message="This is a test notification.") - - return web.json_response(data={}, status=web.HTTPOk.status_code, dumps=encoder.encode) +import app.features.notifications.router # noqa: F401 diff --git a/app/routes/api/presets.py b/app/routes/api/presets.py index 69eecac7..79d59507 100644 --- a/app/routes/api/presets.py +++ b/app/routes/api/presets.py @@ -1,100 +1 @@ -import logging -import uuid - -from aiohttp import web -from aiohttp.web import Request, Response - -from app.library.encoder import Encoder -from app.library.Events import EventBus, Events -from app.library.Presets import Preset, Presets -from app.library.router import route -from app.library.Utils import init_class, validate_uuid - -LOG: logging.Logger = logging.getLogger(__name__) - - -@route("GET", "api/presets/", "presets") -async def presets(request: Request, encoder: Encoder) -> Response: - """ - Get the presets. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object. - - """ - data: list[Preset] = Presets.get_instance().get_all() - filter_fields: str | None = request.query.get("filter", None) - - if filter_fields: - fields: list[str] = [field.strip() for field in filter_fields.split(",")] - data = [{key: value for key, value in preset.serialize().items() if key in fields} for preset in data] - - return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode) - - -@route("PUT", "api/presets/", "presets_add") -async def presets_add(request: Request, encoder: Encoder, notify: EventBus) -> Response: - """ - Add presets. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - notify (EventBus): The event bus instance. - - Returns: - Response: The response object - - """ - data = await request.json() - - if not isinstance(data, list): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - presets: list = [] - - cls = Presets.get_instance() - - for item in data: - if not isinstance(item, dict): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("name"): - return web.json_response( - {"error": "name is required.", "data": item}, status=web.HTTPBadRequest.status_code - ) - - if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): - item["id"] = str(uuid.uuid4()) - - try: - cls.validate(item) - except ValueError as e: - return web.json_response( - {"error": f"Failed to validate preset '{item.get('name')}'. '{e!s}'"}, - status=web.HTTPBadRequest.status_code, - ) - - presets.append(init_class(Preset, item)) - - try: - presets = cls.save(items=presets).load().get_all() - except Exception as e: - LOG.exception(e) - return web.json_response( - {"error": "Failed to save presets.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) - - notify.emit(Events.PRESETS_UPDATE, data=presets) - return web.json_response(data=presets, status=web.HTTPOk.status_code, dumps=encoder.encode) +import app.features.presets.router # noqa: F401 diff --git a/app/routes/api/system.py b/app/routes/api/system.py index 490b98b8..2c4345f4 100644 --- a/app/routes/api/system.py +++ b/app/routes/api/system.py @@ -1,21 +1,67 @@ import asyncio +import errno import logging +import os +import shlex import time +from pathlib import Path +from typing import TYPE_CHECKING from aiohttp import web from aiohttp.web import Request, Response from aiohttp.web_runner import GracefulExit +from app.features.dl_fields.service import DLFields +from app.features.presets.service import Presets from app.library.config import Config from app.library.downloads import DownloadQueue from app.library.encoder import Encoder from app.library.Events import EventBus, Events from app.library.router import route from app.library.UpdateChecker import UpdateChecker +from app.library.Utils import list_folders + +if TYPE_CHECKING: + from asyncio import Task + from asyncio.events import AbstractEventLoop + from asyncio.subprocess import Process LOG: logging.Logger = logging.getLogger(__name__) +@route("GET", "api/system/configuration", "system.configuration") +async def system_config(queue: DownloadQueue, config: Config, encoder: Encoder) -> Response: + """ + Pause non-active downloads. + + Args: + queue (DownloadQueue): The download queue instance. + config (Config): The config instance. + encoder (Encoder): The encoder instance. + + Returns: + Response: The response object. + + """ + return web.json_response( + data={ + "app": config.frontend(), + "presets": Presets.get_instance().get_all(), + "dl_fields": await DLFields.get_instance().get_all_serialized(), + "paused": queue.is_paused(), + "folders": list_folders( + path=Path(config.download_path), + base=Path(config.download_path), + depth_limit=config.download_path_depth - 1, + ), + "history_count": await queue.done.get_total_count(), + "queue": (await queue.get("queue"))["queue"], + }, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + @route("POST", "api/system/pause", "system.pause") async def downloads_pause(queue: DownloadQueue, encoder: Encoder, notify: EventBus) -> Response: """ @@ -147,34 +193,202 @@ async def check_updates(config: Config, encoder: Encoder, update_checker: Update Response: The response object. """ - if not config.check_for_updates: - return web.json_response( - {"error": "Update checking is disabled in configuration."}, - status=web.HTTPBadRequest.status_code, - dumps=encoder.encode, - ) - - # If update already found, return cached result - if config.new_version: + if config.new_version or config.yt_new_version: return web.json_response( data={ - "status": "update_available", - "current_version": config.app_version, - "new_version": config.new_version, + "app": { + "status": "update_available" if config.new_version else "up_to_date", + "current_version": config.app_version, + "new_version": config.new_version if config.new_version else None, + }, + "ytdlp": { + "status": "update_available" if config.yt_new_version else "up_to_date", + "current_version": config._ytdlp_version(), + "new_version": config.yt_new_version if config.yt_new_version else None, + }, }, status=web.HTTPOk.status_code, dumps=encoder.encode, ) - # Run check and await result - status, new_version = await update_checker.check_for_updates() + (app_status, app_new_version), (ytdlp_status, ytdlp_new_version) = await update_checker.check_for_updates() return web.json_response( data={ - "status": status, - "current_version": config.app_version, - "new_version": new_version, + "app": { + "status": app_status, + "current_version": config.app_version, + "new_version": app_new_version, + }, + "ytdlp": { + "status": ytdlp_status, + "current_version": config._ytdlp_version(), + "new_version": ytdlp_new_version, + }, }, status=web.HTTPOk.status_code, dumps=encoder.encode, ) + + +@route("POST", "api/system/terminal", "system.terminal") +async def stream_terminal(request: Request, config: Config, encoder: Encoder) -> Response | web.StreamResponse: + if not config.console_enabled: + return web.json_response( + {"error": "Console feature is disabled."}, + status=web.HTTPForbidden.status_code, + ) + + if not request.can_read_body: + return web.json_response( + {"error": "Request body is required."}, + status=web.HTTPBadRequest.status_code, + ) + + payload = await request.json() + if not isinstance(payload, dict): + return web.json_response( + {"error": "Invalid request payload."}, + status=web.HTTPBadRequest.status_code, + ) + + raw_command = payload.get("command") + if not isinstance(raw_command, str) or not raw_command.strip(): + return web.json_response( + {"error": "Command is required."}, + status=web.HTTPBadRequest.status_code, + ) + + response = web.StreamResponse( + status=web.HTTPOk.status_code, + headers={ + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + "Connection": "keep-alive", + }, + ) + await response.prepare(request) + + async def emit_event(event: str, data: dict) -> None: + if request.transport is None or request.transport.is_closing(): + return + payload: str = f"event: {event}\ndata: {encoder.encode(data)}\n\n" + await response.write(payload.encode("utf-8")) + + returncode: int = -1 + try: + LOG.info("Cli command from client. '%s'", raw_command) + + args: list[str] = ["yt-dlp", *shlex.split(raw_command, posix=os.name != "nt")] + env_vars: dict[str, str] = os.environ.copy() + env_vars.update( + { + "PWD": config.download_path, + "FORCE_COLOR": "1", + "PYTHONUNBUFFERED": "1", + } + ) + + if "nt" != os.name: + env_vars.update( + { + "TERM": "xterm-256color", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US.UTF-8", + "SHELL": "/bin/bash", + } + ) + + try: + import pty + + master_fd, slave_fd = pty.openpty() + stdin_arg = asyncio.subprocess.DEVNULL + stdout_arg = stderr_arg = slave_fd + use_pty = True + except ImportError: + use_pty = False + master_fd = slave_fd = None + stdin_arg = asyncio.subprocess.DEVNULL + stdout_arg = asyncio.subprocess.PIPE + stderr_arg = asyncio.subprocess.STDOUT + + creationflags = 0 + if os.name == "nt": + import subprocess + + creationflags = subprocess.CREATE_NO_WINDOW + + proc: Process = await asyncio.create_subprocess_exec( + *args, + cwd=config.download_path, + stdin=stdin_arg, + stdout=stdout_arg, + stderr=stderr_arg, + env=env_vars, + creationflags=creationflags, + ) + + if use_pty: + assert slave_fd is not None + try: + os.close(slave_fd) + except Exception as e: + LOG.error("Error closing PTY. '%s'.", str(e)) + + async def reader() -> None: + if use_pty is False: + assert proc.stdout is not None + async for raw_line in proc.stdout: + line = raw_line.rstrip(b"\n") + await emit_event("output", {"type": "stdout", "line": line.decode("utf-8", errors="replace")}) + return + + assert master_fd is not None + read_fd = master_fd + loop: AbstractEventLoop = asyncio.get_running_loop() + buffer: bytes = b"" + while True: + try: + chunk: bytes = await loop.run_in_executor(None, lambda: os.read(read_fd, 1024)) + except OSError as e: + if e.errno == errno.EIO: + break + raise + + if not chunk: + if buffer: + await emit_event( + "output", + {"type": "stdout", "line": buffer.decode("utf-8", errors="replace")}, + ) + break + + buffer += chunk + *lines, buffer = buffer.split(b"\n") + + for line in lines: + await emit_event( + "output", + {"type": "stdout", "line": line.decode("utf-8", errors="replace")}, + ) + if master_fd is None: + return + try: + os.close(master_fd) + except Exception as e: + LOG.error("Error closing PTY. '%s'.", str(e)) + + read_task: Task = asyncio.create_task(reader(), name="cli_reader") + + returncode = await proc.wait() + await read_task + except Exception as e: + LOG.error("CLI execute exception was thrown.") + LOG.exception(e) + await emit_event("output", {"type": "stderr", "line": str(e)}) + finally: + await emit_event("close", {"exitcode": returncode}) + await response.write_eof() + + return response diff --git a/app/routes/api/task_definitions.py b/app/routes/api/task_definitions.py index bda8f8fb..83dd62c0 100644 --- a/app/routes/api/task_definitions.py +++ b/app/routes/api/task_definitions.py @@ -1,145 +1,3 @@ -import logging -from typing import Any +"""Migrated API routes for feature submodule.""" -from aiohttp import web -from aiohttp.web import Request, Response - -from app.library.encoder import Encoder -from app.library.router import route -from app.library.TaskDefinitions import TaskDefinitionRecord, TaskDefinitions - -LOG: logging.Logger = logging.getLogger(__name__) - - -@route("GET", "api/task_definitions/", "task_definitions") -async def task_definitions_list(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: - include: str | None = request.query.get("include") - include_definition: bool = "definition" == include - - return web.json_response( - data=[item.serialize(include_definition=include_definition) for item in task_definitions.list()], - status=web.HTTPOk.status_code, - dumps=encoder.encode, - ) - - -@route("GET", "api/task_definitions/{identifier}", "task_definitions_get") -async def task_definitions_get(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: - identifier: str = request.match_info.get("identifier", "").strip() - if not identifier: - return web.json_response( - data={"error": "Missing task definition identifier."}, - status=web.HTTPBadRequest.status_code, - ) - - record: TaskDefinitionRecord | None = task_definitions.get(identifier) - if not record: - return web.json_response( - data={"error": f"Task definition '{identifier}' not found."}, - status=web.HTTPNotFound.status_code, - ) - - return web.json_response( - data=record.serialize(include_definition=True), - status=web.HTTPOk.status_code, - dumps=encoder.encode, - ) - - -@route("POST", "api/task_definitions/", "task_definitions_create") -async def task_definitions_create(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: - try: - payload: Any = await request.json() - if not isinstance(payload, dict): - return web.json_response( - data={"error": "Invalid request body; expected JSON object."}, - status=web.HTTPBadRequest.status_code, - ) - - if "definition" in payload: - if not isinstance(payload["definition"], dict): - return web.json_response( - data={"error": "definition must be a JSON object when provided."}, - status=web.HTTPBadRequest.status_code, - ) - payload = payload["definition"] - - record: TaskDefinitionRecord = task_definitions.create(payload) - except (ValueError, TypeError) as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": "Failed to create task definition."}, - status=web.HTTPInternalServerError.status_code, - ) - - return web.json_response( - data=record.serialize(include_definition=True), - status=web.HTTPCreated.status_code, - dumps=encoder.encode, - ) - - -@route("PUT", "api/task_definitions/{identifier}", "task_definitions_update") -async def task_definitions_update(request: Request, encoder: Encoder, task_definitions: TaskDefinitions) -> Response: - identifier: str = request.match_info.get("identifier", "").strip() - if not identifier: - return web.json_response( - data={"error": "Missing task definition identifier."}, - status=web.HTTPBadRequest.status_code, - ) - - try: - payload: Any = await request.json() - if not isinstance(payload, dict): - return web.json_response( - data={"error": "Invalid request body; expected JSON object."}, - status=web.HTTPBadRequest.status_code, - ) - - if "definition" in payload: - if not isinstance(payload["definition"], dict): - return web.json_response( - data={"error": "definition must be a JSON object when provided."}, - status=web.HTTPBadRequest.status_code, - ) - payload = payload["definition"] - - record: TaskDefinitionRecord = task_definitions.update(identifier, payload) - except (ValueError, TypeError) as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": "Failed to update task definition."}, - status=web.HTTPInternalServerError.status_code, - ) - - return web.json_response( - data=record.serialize(include_definition=True), - status=web.HTTPOk.status_code, - dumps=encoder.encode, - ) - - -@route("DELETE", "api/task_definitions/{identifier}", "task_definitions_delete") -async def task_definitions_delete(request: Request, task_definitions: TaskDefinitions) -> Response: - identifier: str = request.match_info.get("identifier", "").strip() - if not identifier: - return web.json_response( - data={"error": "Missing task definition identifier."}, - status=web.HTTPBadRequest.status_code, - ) - - try: - task_definitions.delete(identifier) - except ValueError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": "Failed to delete task definition."}, status=web.HTTPInternalServerError.status_code - ) - - return web.json_response(data={"status": "deleted"}, status=web.HTTPOk.status_code) +import app.features.tasks.definitions.router # noqa: F401 diff --git a/app/routes/api/tasks.py b/app/routes/api/tasks.py index 4ed97dca..30a548c2 100644 --- a/app/routes/api/tasks.py +++ b/app/routes/api/tasks.py @@ -1,467 +1,3 @@ -import asyncio -import functools -import logging -import uuid -from pathlib import Path -from typing import TYPE_CHECKING, Any +"""Migrated API routes for feature submodule.""" -from aiohttp import web -from aiohttp.web import Request, Response - -from app.library.ag_utils import ag -from app.library.config import Config -from app.library.encoder import Encoder -from app.library.router import route -from app.library.Tasks import Task, TaskFailure, TaskResult, Tasks -from app.library.Utils import get_channel_images, get_file, init_class, parse_outtmpl, validate_url, validate_uuid - -if TYPE_CHECKING: - from pathlib import Path - -LOG: logging.Logger = logging.getLogger(__name__) - - -@route("POST", "api/tasks/inspect", "task_handler_inspect") -async def task_handler_inspect(request: Request, tasks: Tasks, encoder: Encoder, config: Config) -> Response: - """ - Check if handler can process the given URL. - - Args: - request (Request): The request object. - tasks (Tasks): The tasks instance. - encoder (Encoder): The encoder instance. - config (Config): The config instance. - - Returns: - Response: The response object. - - """ - data = await request.json() - - url: str | None = data.get("url") if isinstance(data, dict) else None - if not url: - return web.json_response({"error": "url is required."}, status=web.HTTPBadRequest.status_code) - - static_only: bool = data.get("static_only", False) if isinstance(data, dict) else False - if not static_only: - try: - validate_url(url, allow_internal=config.allow_internal_urls) - except ValueError as e: - return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code) - - preset: str = data.get("preset", "") if isinstance(data, dict) else "" - handler_name: str | None = data.get("handler") if isinstance(data, dict) else None - - try: - result: TaskResult | TaskFailure = await tasks.get_handler().inspect( - url=url, preset=preset, handler_name=handler_name, static_only=static_only - ) - except Exception as e: - LOG.exception(e) - return web.json_response( - {"error": "Failed to inspect handler.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) - - return web.json_response( - data=result, - status=web.HTTPBadRequest.status_code if isinstance(result, TaskFailure) else web.HTTPOk.status_code, - dumps=encoder.encode, - ) - - -@route("GET", "api/tasks/", "tasks") -async def tasks(encoder: Encoder) -> Response: - """ - Get the tasks. - - Args: - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object. - - """ - return web.json_response(data=Tasks.get_instance().get_all(), status=web.HTTPOk.status_code, dumps=encoder.encode) - - -@route("PUT", "api/tasks/", "tasks_add") -async def tasks_add(request: Request, encoder: Encoder) -> Response: - """ - Add tasks to the queue. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object - - """ - data = await request.json() - - if not isinstance(data, list): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - tasks: list = [] - - ins = Tasks.get_instance() - - for item in data: - if not isinstance(item, dict): - return web.json_response( - {"error": "Invalid request body expecting list with dicts."}, - status=web.HTTPBadRequest.status_code, - ) - - if not item.get("url"): - return web.json_response({"error": "url is required.", "data": item}, status=web.HTTPBadRequest.status_code) - - if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): - item["id"] = str(uuid.uuid4()) - - if not item.get("template", None): - item["template"] = "" - - if not item.get("cli", None): - item["cli"] = "" - - try: - Tasks.validate(item) - except ValueError as e: - return web.json_response( - {"error": f"Failed to validate task '{item.get('name', '??')}'. '{e!s}'"}, - status=web.HTTPBadRequest.status_code, - ) - - tasks.append(init_class(Task, item)) - - try: - tasks = ins.save(tasks=tasks).load().get_all() - except Exception as e: - LOG.exception(e) - return web.json_response( - {"error": "Failed to save tasks.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) - - return web.json_response(data=tasks, status=web.HTTPOk.status_code, dumps=encoder.encode) - - -@route("POST", "api/tasks/{id}/mark", "tasks_mark") -async def task_mark(request: Request, encoder: Encoder) -> Response: - """ - Mark all items from task as downloaded. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object - - """ - task_id: str = request.match_info.get("id", None) - - if not task_id: - return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) - - tasks: Tasks = Tasks.get_instance() - try: - task: Task | None = tasks.get(task_id) - if not task: - return web.json_response( - data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code - ) - - _status, _message = task.mark() - if not _status: - return web.json_response(data={"error": _message}, status=web.HTTPBadRequest.status_code) - - return web.json_response(data={"message": _message}, status=web.HTTPOk.status_code, dumps=encoder.encode) - except ValueError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - - -@route("DELETE", "api/tasks/{id}/mark", "tasks_unmark") -async def task_unmark(request: Request, encoder: Encoder) -> Response: - """ - Remove All tasks items from download archive. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object - - """ - task_id: str = request.match_info.get("id", None) - - if not task_id: - return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) - - tasks: Tasks = Tasks.get_instance() - try: - task: Task | None = tasks.get(task_id) - if not task: - return web.json_response( - data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code - ) - - _status, _message = task.unmark() - if not _status: - return web.json_response(data={"error": _message}, status=web.HTTPBadRequest.status_code) - - return web.json_response(data={"message": _message}, status=web.HTTPOk.status_code, dumps=encoder.encode) - except ValueError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - - -@route("POST", "api/tasks/{id}/metadata", "tasks_metadata") -async def task_metadata(request: Request, config: Config, encoder: Encoder) -> Response: - """ - Generate metadata for the task. - - Args: - request (Request): The request object. - config (Config): The config instance. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object - - """ - task_id: str = request.match_info.get("id", None) - - if not task_id: - return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) - - tasks: Tasks = Tasks.get_instance() - try: - task: Task | None = tasks.get(task_id) - if not task: - return web.json_response( - data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code - ) - - (save_path, _) = get_file(config.download_path, task.folder) - if not str(save_path or "").startswith(str(config.download_path)): - return web.json_response(data={"error": "Invalid task folder."}, status=web.HTTPBadRequest.status_code) - - if not save_path.exists(): - save_path.mkdir(parents=True, exist_ok=True) - - metadata, status, message = await asyncio.wait_for( - fut=asyncio.get_running_loop().run_in_executor( - None, - functools.partial(task.fetch_metadata, full=False), - ), - timeout=120, - ) - if not status: - return web.json_response(data={"error": message}, status=web.HTTPBadRequest.status_code) - - if not task.folder: - try: - ytdlp_opts: dict = task.get_ytdlp_opts().get_all() - outtmpl: str = parse_outtmpl( - output_template=ytdlp_opts.get("outtmpl", {}).get("default", "{title} [{id}]"), - info_dict=metadata, - params=ytdlp_opts, - ) - if outtmpl: - _path: Path = save_path / outtmpl - if not _path.is_dir(): - _path: Path = _path.parent - - (save_path, _) = get_file(config.download_path, _path.relative_to(config.download_path)) - if not str(save_path or "").startswith(str(config.download_path)): - return web.json_response( - data={"error": "Invalid final path folder."}, status=web.HTTPBadRequest.status_code - ) - - if not save_path.exists(): - save_path.mkdir(parents=True, exist_ok=True) - except Exception as e: - LOG.warning(f"Failed to resolve final path from outtmpl. '{e!s}'") - - info = { - "id": ag(metadata, ["id", "channel_id"]), - "id_type": metadata.get("extractor", "").split(":")[0].lower() if metadata.get("extractor") else None, - "title": ag(metadata, ["title", "fulltitle"]) or None, - "description": metadata.get("description", ""), - "uploader": metadata.get("uploader", ""), - "tags": metadata.get("tags", []), - "year": metadata.get("release_year"), - "thumbnails": get_channel_images(metadata.get("thumbnails", {})), - } - - if not info.get("title"): - return web.json_response( - data={"error": "Failed to get title from metadata."}, status=web.HTTPBadRequest.status_code - ) - - LOG.info(f"Generating metadata for task '{task.name}' in '{save_path!s}'") - - from yt_dlp.utils import sanitize_filename - - from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP - - title: str = sanitize_filename(info.get("title")) - info_file: Path = save_path / f"{title} [{info.get('id')}].info.json" - info_file.write_text(encoder.encode(metadata), encoding="utf-8") - info["json_file"] = str(info_file.relative_to(config.download_path)) - - xml_file: Path = save_path / "tvshow.nfo" - info["nfo_file"] = str(xml_file.relative_to(config.download_path)) - - xml_content = "\n" - xml_content += f" {NFOMakerPP._escape_text(info.get('title'))}\n" - if info.get("description"): - xml_content += ( - f" {NFOMakerPP._escape_text(NFOMakerPP._clean_description(info.get('description')))}\n" - ) - if info.get("id"): - xml_content += f" {NFOMakerPP._escape_text(info.get('id'))}\n" - if info.get("id_type") and info.get("id"): - xml_content += f' {NFOMakerPP._escape_text(info.get("id"))}\n' - if info.get("uploader"): - xml_content += f" {NFOMakerPP._escape_text(info.get('uploader'))}\n" - if info.get("tags", []): - for tag in info.get("tags", []): - xml_content += f" {NFOMakerPP._escape_text(tag)}\n" - if info.get("year"): - xml_content += f" {info.get('year')}\n" - xml_content += " Continuing\n" - xml_content += "\n" - xml_file.write_text(xml_content, encoding="utf-8") - - try: - from yt_dlp.utils.networking import random_user_agent - - from app.library.httpx_client import async_client - - ytdlp_args: dict = task.get_ytdlp_opts().get_all() - opts: dict[str, Any] = { - "headers": { - "User-Agent": request.headers.get("User-Agent", ytdlp_args.get("user_agent", random_user_agent())), - }, - } - if proxy := ytdlp_args.get("proxy"): - opts["proxy"] = proxy - - try: - from httpx_curl_cffi import AsyncCurlTransport, CurlOpt - - opts["transport"] = AsyncCurlTransport( - impersonate="chrome", - default_headers=True, - curl_options={CurlOpt.FRESH_CONNECT: True}, - ) - opts.pop("headers", None) - except Exception: - pass - - async with async_client(**opts) as client: - for key in info.get("thumbnails", {}): - try: - url = info["thumbnails"][key] - LOG.info(f"Fetching thumbnail '{key}' from '{url}'") - if not url: - continue - - try: - validate_url(url, allow_internal=config.allow_internal_urls) - except ValueError: - LOG.warning(f"Invalid thumbnail url '{url}'") - continue - - resp = await client.request(method="GET", url=url, follow_redirects=True) - - img_file = save_path / f"{key}.jpg" - img_file.write_bytes(resp.content) - info["thumbnails"][key] = str(img_file.relative_to(config.download_path)) - except Exception as e: - LOG.warning(f"Failed to fetch thumbnail '{key}' from '{url}'. '{e!s}'") - continue - except Exception as e: - LOG.warning(f"Failed to fetch thumbnails. '{e!s}'") - - return web.json_response(data=info, status=web.HTTPOk.status_code, dumps=encoder.encode) - except ValueError as e: - return web.json_response(data={"error": str(e)}, status=web.HTTPBadRequest.status_code) - - -@route("PATCH", "api/tasks/{id}", "tasks_update") -async def task_update(request: Request, encoder: Encoder) -> Response: - """ - Update specific fields of a task. - - Args: - request (Request): The request object. - encoder (Encoder): The encoder instance. - - Returns: - Response: The response object - - """ - if not (task_id := request.match_info.get("id", None)): - return web.json_response(data={"error": "No task id."}, status=web.HTTPBadRequest.status_code) - - try: - data = await request.json() - except Exception as e: - return web.json_response( - data={"error": "Invalid JSON in request body.", "message": str(e)}, - status=web.HTTPBadRequest.status_code, - ) - - if not isinstance(data, dict): - return web.json_response( - {"error": "Request body must be a JSON object."}, - status=web.HTTPBadRequest.status_code, - ) - - tasks: Tasks = Tasks.get_instance() - try: - task: Task | None = tasks.get(task_id) - if not task: - return web.json_response( - data={"error": f"Task '{task_id}' does not exist."}, status=web.HTTPNotFound.status_code - ) - - task_dict: dict = task.serialize() - protected_fields: set[str] = {"id"} - updated: bool = False - - for key, value in data.items(): - if key in protected_fields or key not in task_dict: - continue - - setattr(task, key, value) - updated = True - - if not updated: - return web.json_response( - data={"error": "No valid fields to update."}, - status=web.HTTPBadRequest.status_code, - ) - - try: - Tasks.validate(task) - except ValueError as e: - return web.json_response({"error": f"Validation failed: {e!s}"}, status=web.HTTPBadRequest.status_code) - - tasks.save(tasks=tasks.get_all()).load() - - return web.json_response(data=task, status=web.HTTPOk.status_code, dumps=encoder.encode) - except Exception as e: - LOG.exception(e) - return web.json_response( - data={"error": "Failed to update task.", "message": str(e)}, - status=web.HTTPInternalServerError.status_code, - ) +import app.features.tasks.router # noqa: F401 diff --git a/app/routes/api/yt_dlp.py b/app/routes/api/yt_dlp.py index eeb6d9f6..996e0248 100644 --- a/app/routes/api/yt_dlp.py +++ b/app/routes/api/yt_dlp.py @@ -1,5 +1,3 @@ -import asyncio -import functools import json import logging import time @@ -9,17 +7,17 @@ from typing import Any from aiohttp import web from aiohttp.web import Request, Response +from app.features.presets.service import Presets from app.library.cache import Cache from app.library.config import Config from app.library.encoder import Encoder from app.library.ItemDTO import Item -from app.library.Presets import Presets from app.library.router import route from app.library.Utils import ( REMOVE_KEYS, archive_read, arg_converter, - extract_info, + fetch_info, get_archive_id, validate_url, ) @@ -165,20 +163,13 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response: }, } - data: dict | None = await asyncio.wait_for( - fut=asyncio.get_running_loop().run_in_executor( - None, - functools.partial( - extract_info, - config=ytdlp_opts, - url=url, - debug=False, - no_archive=True, - follow_redirect=True, - sanitize_info=True, - ), - ), - timeout=120, + data: dict | None = await fetch_info( + config=ytdlp_opts, + url=url, + debug=False, + no_archive=True, + follow_redirect=True, + sanitize_info=True, ) if not data or not isinstance(data, dict): diff --git a/app/routes/socket/connection.py b/app/routes/socket/connection.py index dd761fb9..c05690c1 100644 --- a/app/routes/socket/connection.py +++ b/app/routes/socket/connection.py @@ -1,160 +1,15 @@ -import asyncio import logging -from pathlib import Path -from typing import Any -import socketio - -from app.library.config import Config -from app.library.dl_fields import DLFields -from app.library.downloads import DownloadQueue -from app.library.Events import EventBus, Events -from app.library.Presets import Presets from app.library.router import RouteType, route -from app.library.Utils import list_folders, tail_log LOG: logging.Logger = logging.getLogger(__name__) -class _Data: - subscribers: dict[str, list[str]] = {} - log_task: asyncio.Task | None = None - - @route(RouteType.SOCKET, "connect", "socket_connect") -async def connect(config: Config, queue: DownloadQueue, notify: EventBus, sid: str): - notify.emit( - Events.CONFIGURATION, - data={ - "config": config.frontend(), - "presets": Presets.get_instance().get_all(), - "dl_fields": DLFields.get_instance().get_all(), - "paused": queue.is_paused(), - }, - title="Client connected", - message=f"Client '{sid}' connected.", - to=sid, - ) - - notify.emit( - Events.CONNECTED, - data={ - "folders": list_folders( - path=Path(config.download_path), - base=Path(config.download_path), - depth_limit=config.download_path_depth - 1, - ), - "history_count": await queue.done.get_total_count(), - "queue": (await queue.get("queue"))["queue"], - }, - title="Sending initial download data", - message=f"Sending initial download data to client '{sid}'.", - to=sid, - ) - - notify.emit( - Events.ACTIVE_QUEUE, - data={"queue": (await queue.get("queue"))["queue"]}, - title="Sending initial active queue data", - message=f"Sending active queue data to client '{sid}'.", - to=sid, - ) +async def connect(sid: str): + pass @route(RouteType.SOCKET, "disconnect", "socket_disconnect") -async def disconnect(sio: socketio.AsyncServer, sid: str, data: str = None): - """ - Handle client disconnection. - - Args: - sio (socketio.AsyncServer): The Socket.IO server instance. - sid (str): The session ID of the client. - data (str): The reason for disconnection. - - """ - LOG.debug(f"Client '{sid}' disconnected. {data}") - - for event in _Data.subscribers: - if sid in _Data.subscribers[event]: - await unsubscribe(sio=sio, sid=sid, data=event) - - -@route(RouteType.SOCKET, "subscribe", "socket_subscribe") -async def subscribe(config: Config, notify: EventBus, sio: socketio.AsyncServer, sid: str, data: str | Any): - """ - Subscribe to a specific event. - - Args: - config (Config): The configuration instance. - notify (EventBus): The event bus to use for notifications. - sio (socketio.AsyncServer): The Socket.IO server instance. - sid (str): The session ID of the client. - data (str): The event to subscribe to. - - """ - if not isinstance(data, str) or not data: - notify.emit( - Events.LOG_ERROR, - title="Subscription Error", - message="Invalid event type was expecting a string.", - to=sid, - ) - return - - if data not in _Data.subscribers: - _Data.subscribers[data] = [] - - if sid not in _Data.subscribers[data]: - _Data.subscribers[data].append(sid) - LOG.debug(f"Client '{sid}' subscribed to event '{data}'.") - await sio.emit(Events.SUBSCRIBED, data={"event": data}, to=sid) - - async def emit_logs(data: dict): - await subscribe_emit(sio=sio, event="log_lines", data=data) - - if "log_lines" == data and _Data.log_task is None: - log_file = Path(config.config_path) / "logs" / "app.log" - LOG.debug(f"Starting tailing '{log_file!s}'.") - _Data.log_task = asyncio.create_task(tail_log(file=log_file, emitter=emit_logs), name="tail_log") - - -@route(RouteType.SOCKET, "unsubscribe", "socket_unsubscribe") -async def unsubscribe(sio: socketio.AsyncServer, sid: str, data: str): - """ - Unsubscribe from a specific event. - - Args: - sio (socketio.AsyncServer): The Socket.IO server instance. - sid (str): The session ID of the client. - data (str): The event to unsubscribe from. - - """ - if data not in _Data.subscribers: - return - - if sid not in _Data.subscribers[data]: - return - - _Data.subscribers[data].remove(sid) - await sio.emit(Events.UNSUBSCRIBED, data={"event": data}, to=sid) - - LOG.debug(f"Client '{sid}' unsubscribed from event '{data}'.") - - if "log_lines" != data or not _Data.log_task or "log_lines" not in _Data.subscribers: - return - - if len(_Data.subscribers["log_lines"]) < 1: - try: - LOG.debug("Stopping log tailing task.") - _Data.log_task.cancel() - _Data.log_task = None - except asyncio.CancelledError: - pass - - -async def subscribe_emit(sio: socketio.AsyncServer, event: str, data: dict): - if event not in _Data.subscribers or len(_Data.subscribers[event]) < 1: - return - - for sid in _Data.subscribers[event]: - await sio.emit(event=event, data=data, to=sid) +async def disconnect(sid: str): + pass diff --git a/app/routes/socket/terminal.py b/app/routes/socket/terminal.py deleted file mode 100644 index 146c39ff..00000000 --- a/app/routes/socket/terminal.py +++ /dev/null @@ -1,146 +0,0 @@ -import logging -import os -from typing import TYPE_CHECKING - -from app.library.config import Config -from app.library.Events import EventBus, Events -from app.library.router import RouteType, route - -if TYPE_CHECKING: - from asyncio import Task - from asyncio.events import AbstractEventLoop - from asyncio.subprocess import Process - -LOG: logging.Logger = logging.getLogger(__name__) - - -@route(RouteType.SOCKET, "cli_post", "socket_cli_post") -async def cli_post(config: Config, notify: EventBus, sid: str, data: str): - if not config.console_enabled: - notify.emit( - Events.LOG_ERROR, - title="Feature disabled", - message="Console feature is disabled.", - to=sid, - ) - return - - if not data: - notify.emit(Events.CLI_CLOSE, data={"exitcode": 0}, to=sid) - return - - import asyncio - import errno - import shlex - import subprocess # ignore - - returncode: int = -1 - try: - LOG.info(f"Cli command from client '{sid}'. '{data}'") - - args: list[str] = ["yt-dlp", *shlex.split(data, posix=os.name != "nt")] - _env: dict[str, str] = os.environ.copy() - _env.update( - { - "PWD": config.download_path, - "FORCE_COLOR": "1", - "PYTHONUNBUFFERED": "1", - } - ) - - if "nt" != os.name: - _env.update( - { - "TERM": "xterm-256color", - "LANG": "en_US.UTF-8", - "LC_ALL": "en_US.UTF-8", - "SHELL": "/bin/bash", - } - ) - - try: - import pty - - master_fd, slave_fd = pty.openpty() - stdin_arg = asyncio.subprocess.DEVNULL - stdout_arg = stderr_arg = slave_fd - use_pty = True - except ImportError: - use_pty = False - master_fd = slave_fd = None - stdin_arg = asyncio.subprocess.DEVNULL - stdout_arg = asyncio.subprocess.PIPE - stderr_arg = asyncio.subprocess.STDOUT - - proc: Process = await asyncio.create_subprocess_exec( - *args, - cwd=config.download_path, - stdin=stdin_arg, - stdout=stdout_arg, - stderr=stderr_arg, - env=_env, - creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0, - ) - - if use_pty: - try: - os.close(slave_fd) - except Exception as e: - LOG.error(f"Error closing PTY. '{e!s}'.") - - async def reader(sid: str): - if use_pty is False: - assert proc.stdout is not None - async for raw_line in proc.stdout: - line = raw_line.rstrip(b"\n") - notify.emit( - Events.CLI_OUTPUT, - data={"type": "stdout", "line": line.decode("utf-8", errors="replace")}, - to=sid, - ) - return - - loop: AbstractEventLoop = asyncio.get_running_loop() - buffer: bytes = b"" - while True: - try: - chunk: bytes = await loop.run_in_executor(None, lambda: os.read(master_fd, 1024)) - except OSError as e: - if e.errno == errno.EIO: - break - raise - - if not chunk: - if buffer: - notify.emit( - Events.CLI_OUTPUT, - data={"type": "stdout", "line": buffer.decode("utf-8", errors="replace")}, - to=sid, - ) - break - - buffer += chunk - *lines, buffer = buffer.split(b"\n") - - for line in lines: - notify.emit( - Events.CLI_OUTPUT, - data={"type": "stdout", "line": line.decode("utf-8", errors="replace")}, - to=sid, - ) - try: - os.close(master_fd) - except Exception as e: - LOG.error(f"Error closing PTY. '{e!s}'.") - - read_task: Task = asyncio.create_task(reader(sid=sid), name=f"cli_reader_{sid}") - - returncode: int = await proc.wait() - - await read_task - except Exception as e: - LOG.error(f"CLI execute exception was thrown for client '{sid}'.") - LOG.exception(e) - notify.emit(Events.CLI_OUTPUT, data={"type": "stderr", "line": str(e)}, to=sid) - finally: - notify.emit(Events.CLI_CLOSE, data={"exitcode": returncode}, to=sid) diff --git a/app/schema/task_definition.json b/app/schema/task_definition.json index 1e3c8917..76a1bd6e 100644 --- a/app/schema/task_definition.json +++ b/app/schema/task_definition.json @@ -5,11 +5,15 @@ "type": "object", "required": [ "name", - "match", - "parse" + "match_url", + "definition" ], "additionalProperties": false, "properties": { + "id": { + "type": "integer", + "description": "Auto-generated unique identifier (read-only)." + }, "name": { "type": "string", "minLength": 1, @@ -21,49 +25,39 @@ "description": "Optional ordering priority. Lower numbers are listed first.", "default": 0 }, - "match": { + "enabled": { + "type": "boolean", + "description": "Whether this definition is active. Disabled definitions are not matched.", + "default": true + }, + "match_url": { "type": "array", "minItems": 1, "items": { - "oneOf": [ - { - "type": "string", - "minLength": 1, - "description": "Glob pattern matched against task URLs." - }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "regex": { - "type": "string", - "minLength": 1, - "description": "Regular expression applied to task URLs." - }, - "glob": { - "type": "string", - "minLength": 1, - "description": "Glob pattern applied to task URLs." - } - }, - "anyOf": [ - { - "required": [ - "regex" - ] - }, - { - "required": [ - "glob" - ] - } - ] - } - ] + "type": "string", + "minLength": 1, + "description": "URL pattern. Use glob patterns (e.g. 'https://example.com/*') or regex (e.g. '/https://example\\.com/post/[0-9]+/')." }, - "description": "Patterns that determine which tasks use this definition." + "description": "Patterns that determine which tasks use this definition. Regex patterns are detected by /pattern/ format." }, - "engine": { + "created_at": { + "type": "string", + "format": "date-time", + "description": "Timestamp when definition was created (read-only)." + }, + "updated_at": { + "type": "string", + "format": "date-time", + "description": "Timestamp when definition was last updated (read-only)." + }, + "definition": { + "type": "object", + "required": [ + "parse" + ], + "additionalProperties": false, + "properties": { + "engine": { "type": "object", "additionalProperties": false, "properties": { @@ -123,9 +117,9 @@ } } }, - "description": "Optional engine configuration (defaults to HTTPX)." - }, - "request": { + "description": "Optional engine configuration (defaults to HTTPX)." + }, + "request": { "type": "object", "additionalProperties": false, "properties": { @@ -165,7 +159,7 @@ } ] }, - "json": { + "json_data": { "description": "JSON payload for POST requests.", "type": [ "object", @@ -182,81 +176,93 @@ "description": "Timeout in seconds for the request." } }, - "description": "Optional HTTP request overrides." - }, - "response": { + "description": "Optional HTTP request overrides." + }, + "response": { "type": "object", "additionalProperties": false, "properties": { - "type": { - "type": "string", - "enum": [ - "html", - "json" - ], - "default": "html", - "description": "Body format returned by the target URL." - } - } - }, - "parse": { - "type": "object", - "minProperties": 1, - "description": "Field extraction rules and optional container definition.", - "additionalProperties": false, - "properties": { - "items": { - "$ref": "#/definitions/container" - } - }, - "patternProperties": { - "^(?!items$).+$": { - "$ref": "#/definitions/extractionRule" - } - }, - "allOf": [ - { - "if": { - "not": { - "required": [ - "items" - ] + "type": { + "type": "string", + "enum": [ + "html", + "json" + ], + "default": "html", + "description": "Body format returned by the target URL." + } + } + }, + "parse": { + "type": "object", + "minProperties": 1, + "description": "Field extraction rules and optional container definition.", + "additionalProperties": false, + "properties": { + "items": { + "$ref": "#/definitions/container" } }, - "then": { - "required": [ - "link" - ] - } + "patternProperties": { + "^(?!items$).+$": { + "$ref": "#/definitions/extractionRule" + } + }, + "allOf": [ + { + "if": { + "not": { + "required": [ + "items" + ] + } + }, + "then": { + "required": [ + "link" + ] + } + } + ] } - ] + } } }, "allOf": [ { "if": { "properties": { - "engine": { + "definition": { "properties": { - "type": { "const": "selenium" } + "engine": { + "properties": { + "type": { "const": "selenium" } + }, + "required": ["type"] + } }, - "required": ["type"] + "required": ["engine"] } }, - "required": ["engine"] + "required": ["definition"] }, "then": { "properties": { - "engine": { + "definition": { "properties": { - "options": { - "required": ["url"] + "engine": { + "properties": { + "options": { + "required": ["url"] + } + }, + "required": ["options"] } }, - "required": ["options"] + "required": ["engine"] } }, - "required": ["engine"] + "required": ["definition"] } } ], @@ -421,66 +427,68 @@ "examples": [ { "name": "example", - "match": [ + "match_url": [ "https://example.com/articles/*", - { - "regex": "https://example.com/post/[0-9]+" - } + "/https://example\\.com/post/[0-9]+/" ], - "engine": { - "type": "httpx" - }, - "request": { - "method": "GET", - "headers": { - "User-Agent": "MyCustomAgent/1.0" - } - }, - "parse": { - "link": { - "type": "css", - "expression": ".article a.link", - "attribute": "href" + "definition": { + "engine": { + "type": "httpx" }, - "title": { - "type": "css", - "expression": ".article .title", - "attribute": "text" + "request": { + "method": "GET", + "headers": { + "User-Agent": "MyCustomAgent/1.0" + } }, - "id": { - "type": "regex", - "expression": "id=(?P[0-9]+)", - "post_filter": { - "filter": "(?P[0-9]+)", - "value": "id" + "parse": { + "link": { + "type": "css", + "expression": ".article a.link", + "attribute": "href" + }, + "title": { + "type": "css", + "expression": ".article .title", + "attribute": "text" + }, + "id": { + "type": "regex", + "expression": "id=(?P[0-9]+)", + "post_filter": { + "filter": "(?P[0-9]+)", + "value": "id" + } } } } }, { "name": "container-example", - "match": [ + "match_url": [ "https://example.com/list" ], - "parse": { - "items": { - "type": "css", - "selector": ".cards .card", - "fields": { - "link": { - "type": "css", - "expression": ".card-header a", - "attribute": "href" - }, - "title": { - "type": "css", - "expression": ".card-header a", - "attribute": "text" - }, - "poet": { - "type": "css", - "expression": "footer .card-footer-item:first-child a", - "attribute": "text" + "definition": { + "parse": { + "items": { + "type": "css", + "selector": ".cards .card", + "fields": { + "link": { + "type": "css", + "expression": ".card-header a", + "attribute": "href" + }, + "title": { + "type": "css", + "expression": ".card-header a", + "attribute": "text" + }, + "poet": { + "type": "css", + "expression": "footer .card-footer-item:first-child a", + "attribute": "text" + } } } } diff --git a/app/scripts/create_migration.py b/app/scripts/create_migration.py index c3ac4447..f9e8fe68 100644 --- a/app/scripts/create_migration.py +++ b/app/scripts/create_migration.py @@ -8,6 +8,7 @@ Provides commands to create, list, and apply database migrations. from __future__ import annotations import argparse +import asyncio import sys import traceback from pathlib import Path @@ -40,32 +41,32 @@ def cmd_create(args: argparse.Namespace) -> None: o_print(f"created migration {path}") -def cmd_version(args: argparse.Namespace) -> None: - version = migrate.get_version(args.database_path) +async def cmd_version(args: argparse.Namespace) -> None: + version = await migrate.get_version(args.database_path) if version: o_print(f"the db [{args.database_path}] is at version {version}") else: o_print(f"the db [{args.database_path}] is not under version control") -def cmd_upgrade(args: argparse.Namespace) -> None: +async def cmd_upgrade(args: argparse.Namespace) -> None: if args.version: o_print(f"upgrading db [{args.database_path}] to version [{args.version}]") else: o_print(f"upgrading db [{args.database_path}] to most recent version") - migrate.upgrade(args.database_path, args.migration_dir, args.version) + await migrate.upgrade(args.database_path, args.migration_dir, args.version) - new_version = migrate.get_version(args.database_path) + new_version = await migrate.get_version(args.database_path) if args.version is not None: assert new_version == args.version o_print(f"upgraded [{args.database_path}] successfully to version [{new_version}]") -def cmd_downgrade(args: argparse.Namespace) -> None: +async def cmd_downgrade(args: argparse.Namespace) -> None: o_print(f"downgrading db [{args.database_path}] to version [{args.version}]") - migrate.downgrade(args.database_path, args.migration_dir, args.version) + await migrate.downgrade(args.database_path, args.migration_dir, args.version) o_print(f"downgraded [{args.database_path}] successfully to version [{args.version}]") @@ -122,7 +123,11 @@ def run(argv: list[str]) -> int: try: func: Callable[[argparse.Namespace], None] = args.func - func(args) + # Check if function is async + if asyncio.iscoroutinefunction(func): + asyncio.run(func(args)) + else: + func(args) return 0 except migrate.InvalidMigrationError: diff --git a/app/scripts/seed_db.py b/app/scripts/seed_db.py index d3539104..29cc9bce 100644 --- a/app/scripts/seed_db.py +++ b/app/scripts/seed_db.py @@ -2,9 +2,9 @@ from __future__ import annotations import argparse +import asyncio import logging import re -import sqlite3 import sys import time import uuid @@ -14,6 +14,9 @@ from itertools import cycle, islice from pathlib import Path from typing import TYPE_CHECKING +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine + APP_ROOT = str((Path(__file__).parent / ".." / "..").resolve()) if APP_ROOT not in sys.path: sys.path.insert(0, APP_ROOT) @@ -65,11 +68,13 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def connect_db(db_file: str) -> sqlite3.Connection: - conn = sqlite3.connect(database=db_file, isolation_level=None) - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA journal_mode=wal") - return conn +async def connect_db(db_file: str) -> AsyncConnection: + """Create async SQLAlchemy connection.""" + engine = create_async_engine(f"sqlite+aiosqlite:///{db_file}") + conn = await engine.connect() + await conn.execute(text("PRAGMA journal_mode=wal")) + await conn.commit() + return conn, engine def find_files(root: Path, extension: str) -> list[Path]: @@ -187,14 +192,15 @@ def generate_rows( yield _build_row(path, root, idx, status, preset, store) -def insert_batches( - conn: sqlite3.Connection, +async def insert_batches( + conn: AsyncConnection, rows: Iterable[tuple[str, str, str, str, str]], batch_size: int, ) -> int: + """Insert rows in batches using SQLAlchemy.""" sql = """ INSERT INTO "history" ("id", "type", "url", "data", "created_at") - VALUES (?, ?, ?, ?, ?) + VALUES (:p0, :p1, :p2, :p3, :p4) ON CONFLICT("id") DO UPDATE SET "type" = excluded.type, "url" = excluded.url, @@ -207,19 +213,24 @@ def insert_batches( for row in rows: batch.append(row) if len(batch) >= batch_size: - conn.executemany(sql, batch) + # Convert tuples to dicts for SQLAlchemy + params = [{"p0": r[0], "p1": r[1], "p2": r[2], "p3": r[3], "p4": r[4]} for r in batch] + await conn.execute(text(sql), params) + await conn.commit() total_written += len(batch) LOG.info("Inserted %d rows...", total_written) batch.clear() if batch: - conn.executemany(sql, batch) + params = [{"p0": r[0], "p1": r[1], "p2": r[2], "p3": r[3], "p4": r[4]} for r in batch] + await conn.execute(text(sql), params) + await conn.commit() total_written += len(batch) return total_written -def main() -> None: +async def main() -> None: args = parse_args() logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") @@ -234,14 +245,15 @@ def main() -> None: return rows = generate_rows(files, args.root, args.count, args.status, args.preset, store) - conn = connect_db(args.db_file) + conn, engine = await connect_db(args.db_file) try: - written = insert_batches(conn, rows, args.batch) + written = await insert_batches(conn, rows, args.batch) finally: - conn.close() + await conn.close() + await engine.dispose() LOG.info("Done. Inserted %d rows into '%s'.", written, store.value) if __name__ == "__main__": - main() + asyncio.run(main()) diff --git a/app/tests/test_cache.py b/app/tests/test_cache.py index 8b03cda2..b76017b2 100644 --- a/app/tests/test_cache.py +++ b/app/tests/test_cache.py @@ -18,6 +18,7 @@ All cache operations and edge cases are covered. import asyncio import threading import time +from unittest.mock import MagicMock import pytest @@ -255,6 +256,74 @@ class TestCache: retrieved = self.cache.get("object_key") assert retrieved == custom_obj + @pytest.mark.asyncio + async def test_cleanup_removes_expired_entries(self): + """Test that _cleanup removes only expired entries.""" + # Set some keys with different TTLs + self.cache.set("permanent", "value") + self.cache.set("short", "value1", ttl=0.1) + self.cache.set("medium", "value2", ttl=1.0) + + # Wait for short to expire + time.sleep(0.15) + + # Run cleanup + await self.cache.cleanup() + + # Verify only expired key was removed + assert self.cache.get("permanent") == "value", "Should keep permanent key" + assert self.cache.get("medium") == "value2", "Should keep non-expired key" + assert self.cache.get("short") is None, "Should remove expired key" + + @pytest.mark.asyncio + async def test_cleanup_with_no_expired_entries(self): + """Test that _cleanup handles cache with no expired entries.""" + self.cache.set("key1", "value1") + self.cache.set("key2", "value2", ttl=1.0) + + # Run cleanup when nothing is expired + await self.cache.cleanup() + + # Verify all keys still exist + assert self.cache.get("key1") == "value1", "Should keep non-expired key" + assert self.cache.get("key2") == "value2", "Should keep non-expired key" + + @pytest.mark.asyncio + async def test_cleanup_with_empty_cache(self): + """Test that _cleanup handles empty cache without errors.""" + # Clear cache first + self.cache.clear() + + # Should not raise any errors + await self.cache.cleanup() + + @pytest.mark.asyncio + async def test_attach_registers_with_services(self): + """Test that attach method registers cache with Services and schedules cleanup.""" + from app.library.Scheduler import Scheduler + from app.library.Services import Services + + # Reset singletons + Cache._reset_singleton() + Scheduler._reset_singleton() + Services._reset_singleton() + + # Get event loop for scheduler + loop = asyncio.get_event_loop() + + # Create cache and attach + cache = Cache.get_instance() + mock_app = MagicMock() + cache.attach(mock_app) + + # Verify cache is registered with Services + services = Services.get_instance() + assert services.get("cache") is cache, "Should register cache with Services" + + # Verify cleanup job is scheduled + scheduler = Scheduler.get_instance(loop=loop) + assert scheduler.has(f"{Cache.__name__}.{Cache.cleanup.__name__}"), "Should schedule cleanup job" + if __name__ == "__main__": pytest.main([__file__]) diff --git a/app/tests/test_conditions.py b/app/tests/test_conditions.py deleted file mode 100644 index 2021c5e0..00000000 --- a/app/tests/test_conditions.py +++ /dev/null @@ -1,893 +0,0 @@ -""" -Tests for conditions.py - Download conditions management. - -This test suite provides comprehensive coverage for the conditions module: -- Tests Condition dataclass functionality -- Tests Conditions singleton class behavior -- Tests condition loading, saving, and validation -- Tests condition matching against info dicts -- Tests error handling and edge cases - -Total test functions: 15+ -All condition management functionality and edge cases are covered. -""" - -import json -import tempfile -import uuid -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -from app.library.conditions import Condition, Conditions - - -class TestCondition: - """Test the Condition dataclass.""" - - def test_condition_creation_with_defaults(self): - """Test creating a condition with default values.""" - condition = Condition(name="test", filter="duration > 60") - - assert condition.id, "Check that ID is generated" - assert isinstance(condition.id, str) - - # Check required fields - assert condition.name == "test" - assert condition.filter == "duration > 60" - - assert condition.cli == "", "Check defaults" - assert condition.extras == {} - assert condition.enabled is True - - def test_condition_creation_with_all_fields(self): - """Test creating a condition with all fields specified.""" - test_id = str(uuid.uuid4()) - extras = {"key": "value", "number": 42} - - condition = Condition( - id=test_id, name="full_test", filter="uploader = 'test'", cli="--format best", extras=extras, enabled=False - ) - - assert condition.id == test_id - assert condition.name == "full_test" - assert condition.filter == "uploader = 'test'" - assert condition.cli == "--format best" - assert condition.extras == extras - assert condition.enabled is False - - def test_condition_serialize(self): - """Test condition serialization to dict.""" - condition = Condition( - name="serialize_test", filter="title ~= 'test'", cli="--audio-quality 0", extras={"tag": "music"} - ) - - serialized = condition.serialize() - - assert isinstance(serialized, dict) - assert serialized["name"] == "serialize_test" - assert serialized["filter"] == "title ~= 'test'" - assert serialized["cli"] == "--audio-quality 0" - assert serialized["extras"] == {"tag": "music"} - assert "id" in serialized - - def test_condition_json(self): - """Test condition JSON serialization.""" - condition = Condition(name="json_test", filter="duration < 300") - - json_str = condition.json() - - assert isinstance(json_str, str) - # Should be valid JSON - data = json.loads(json_str) - assert data["name"] == "json_test" - assert data["filter"] == "duration < 300" - - def test_condition_get_method(self): - """Test condition get method for accessing fields.""" - condition = Condition(name="get_test", filter="view_count > 1000", extras={"category": "popular"}) - - assert condition.get("name") == "get_test" - assert condition.get("filter") == "view_count > 1000" - assert condition.get("extras") == {"category": "popular"} - assert condition.get("nonexistent") is None - assert condition.get("nonexistent", "default") == "default" - - -class TestConditions: - """Test the Conditions singleton class.""" - - def setup_method(self): - """Set up test fixtures by clearing singleton instances.""" - Conditions._reset_singleton() - - def test_conditions_singleton(self): - """Test that Conditions follows singleton pattern.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "test_conditions.json" - - instance1 = Conditions(file=file_path) - instance2 = Conditions.get_instance() - - assert instance1 is instance2 - - def test_conditions_initialization(self): - """Test Conditions initialization with file path.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "test_conditions.json" - - conditions = Conditions(file=file_path) - - assert conditions._file == file_path - assert isinstance(conditions._items, list) - - @patch("app.library.conditions.Config.get_instance") - def test_conditions_default_file_path(self, mock_config): - """Test Conditions uses default config path when no file specified.""" - mock_config_instance = MagicMock() - mock_config_instance.config_path = "/test/config" - mock_config.return_value = mock_config_instance - - conditions = Conditions() - - expected_path = Path("/test/config") / "conditions.json" - assert conditions._file == expected_path - - def test_get_all_empty(self): - """Test get_all returns empty list when no conditions loaded.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "empty_conditions.json" - conditions = Conditions(file=file_path) - - result = conditions.get_all() - - assert isinstance(result, list) - assert len(result) == 0 - - def test_clear(self): - """Test clearing all conditions.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "clear_test.json" - conditions = Conditions(file=file_path) - - # Add some test conditions - conditions._items = [ - Condition(name="test1", filter="duration > 60"), - Condition(name="test2", filter="uploader = 'test'"), - ] - - result = conditions.clear() - - assert result is conditions # Should return self - assert len(conditions._items) == 0 - - def test_clear_when_already_empty(self): - """Test clearing when conditions list is already empty.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "already_empty.json" - conditions = Conditions(file=file_path) - - result = conditions.clear() - - assert result is conditions - assert len(conditions._items) == 0 - - def test_load_nonexistent_file(self): - """Test loading from non-existent file.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "nonexistent.json" - conditions = Conditions(file=file_path) - - result = conditions.load() - - assert result is conditions - assert len(conditions._items) == 0 - - def test_load_empty_file(self): - """Test loading from empty file.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "empty.json" - file_path.touch() # Create empty file - - conditions = Conditions(file=file_path) - result = conditions.load() - - assert result is conditions - assert len(conditions._items) == 0 - - def test_load_valid_conditions(self): - """Test loading valid conditions from file.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "valid_conditions.json" - - # Create test data - test_data = [ - { - "id": str(uuid.uuid4()), - "name": "short_videos", - "filter": "duration < 300", - "cli": "--format worst", - "extras": {"category": "short"}, - "enabled": True, - "priority": 0, - "description": "Download short videos", - }, - { - "id": str(uuid.uuid4()), - "name": "music_videos", - "filter": "title ~= 'music'", - "cli": "--audio-quality 0", - "extras": {"type": "audio"}, - "enabled": False, - "priority": 5, - "description": "High priority music videos", - }, - ] - - file_path.write_text(json.dumps(test_data, indent=4)) - - conditions = Conditions(file=file_path) - result = conditions.load() - - assert result is conditions - assert len(conditions._items) == 2 - - assert conditions._items[0].name == "short_videos", "Check first condition" - assert conditions._items[0].filter == "duration < 300" - assert conditions._items[0].cli == "--format worst" - assert conditions._items[0].extras == {"category": "short"} - assert conditions._items[0].enabled is True - assert conditions._items[0].priority == 0 - assert conditions._items[0].description == "Download short videos" - - assert conditions._items[1].name == "music_videos", "Check second condition" - assert conditions._items[1].filter == "title ~= 'music'" - assert conditions._items[1].enabled is False - assert conditions._items[1].priority == 5 - assert conditions._items[1].description == "High priority music videos" - - def test_load_conditions_without_id(self): - """Test loading conditions that don't have ID (should generate ID).""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "no_id_conditions.json" - - # Create test data without ID - test_data = [{"name": "no_id_test", "filter": "duration > 120"}] - - file_path.write_text(json.dumps(test_data)) - - with patch.object(Conditions, "save") as mock_save: - conditions = Conditions(file=file_path) - conditions.load() - - assert len(conditions._items) == 1, "Should have generated ID" - assert conditions._items[0].id - assert conditions._items[0].name == "no_id_test" - - # Should call save due to changes - mock_save.assert_called_once() - - def test_load_conditions_without_extras(self): - """Test loading conditions that don't have extras field.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "no_extras_conditions.json" - - test_data = [{"id": str(uuid.uuid4()), "name": "no_extras_test", "filter": "uploader = 'test'"}] - - file_path.write_text(json.dumps(test_data)) - - with patch.object(Conditions, "save") as mock_save: - conditions = Conditions(file=file_path) - conditions.load() - - assert len(conditions._items) == 1, "Should have generated empty extras" - assert conditions._items[0].extras == {} - - # Should call save due to changes - mock_save.assert_called_once() - - def test_load_conditions_without_enabled(self): - """Test loading conditions that don't have enabled field (should default to True).""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "no_enabled_conditions.json" - - test_data = [{"id": str(uuid.uuid4()), "name": "no_enabled_test", "filter": "duration > 60", "extras": {}}] - - file_path.write_text(json.dumps(test_data)) - - with patch.object(Conditions, "save") as mock_save: - conditions = Conditions(file=file_path) - conditions.load() - - assert len(conditions._items) == 1, "Should have generated enabled=True" - assert conditions._items[0].enabled is True - - # Should call save due to changes - mock_save.assert_called_once() - - def test_load_conditions_without_priority(self): - """Test loading conditions that don't have priority field (should default to 0).""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "no_priority_conditions.json" - - test_data = [{"id": str(uuid.uuid4()), "name": "no_priority_test", "filter": "duration > 60", "extras": {}}] - - file_path.write_text(json.dumps(test_data)) - - with patch.object(Conditions, "save") as mock_save: - conditions = Conditions(file=file_path) - conditions.load() - - assert len(conditions._items) == 1, "Should have generated priority=0" - assert conditions._items[0].priority == 0 - - # Should call save due to changes - mock_save.assert_called_once() - - def test_load_conditions_without_description(self): - """Test loading conditions that don't have description field (should default to empty string).""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "no_description_conditions.json" - - test_data = [ - {"id": str(uuid.uuid4()), "name": "no_description_test", "filter": "duration > 60", "extras": {}} - ] - - file_path.write_text(json.dumps(test_data)) - - with patch.object(Conditions, "save") as mock_save: - conditions = Conditions(file=file_path) - conditions.load() - - assert len(conditions._items) == 1, "Should have generated description=''" - assert conditions._items[0].description == "" - - # Should call save due to changes - mock_save.assert_called_once() - - def test_load_invalid_json(self): - """Test loading file with invalid JSON.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid.json" - file_path.write_text("invalid json content") - - conditions = Conditions(file=file_path) - result = conditions.load() - - assert result is conditions - assert len(conditions._items) == 0 - - def test_load_invalid_condition_data(self): - """Test loading file with invalid condition data.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid_data.json" - - # Missing required fields - test_data = [ - {"id": "valid", "name": "valid", "filter": "duration > 60"}, - {"invalid": "data"}, # Missing required fields - ] - - file_path.write_text(json.dumps(test_data)) - - conditions = Conditions(file=file_path) - result = conditions.load() - - assert result is conditions, "Should load only valid conditions" - assert len(conditions._items) == 1 - assert conditions._items[0].name == "valid" - - def test_validate_valid_condition_dict(self): - """Test validating a valid condition dictionary.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "validate_test.json" - conditions = Conditions(file=file_path) - - valid_condition = { - "id": str(uuid.uuid4()), - "name": "valid_test", - "filter": "duration > 60", - "cli": "--format best", - "extras": {"key": "value"}, - } - - result = conditions.validate(valid_condition) - assert result is True - - def test_validate_valid_condition_object(self): - """Test validating a valid Condition object.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "validate_obj_test.json" - conditions = Conditions(file=file_path) - - valid_condition = Condition(name="valid_obj_test", filter="uploader = 'test'") - - result = conditions.validate(valid_condition) - assert result is True - - def test_validate_missing_id(self): - """Test validating condition without ID.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "no_id_validate.json" - conditions = Conditions(file=file_path) - - invalid_condition = {"name": "no_id_test", "filter": "duration > 60"} - - with pytest.raises(ValueError, match="No id found"): - conditions.validate(invalid_condition) - - def test_validate_missing_name(self): - """Test validating condition without name.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "no_name_validate.json" - conditions = Conditions(file=file_path) - - invalid_condition = {"id": str(uuid.uuid4()), "filter": "duration > 60"} - - with pytest.raises(ValueError, match="No name found"): - conditions.validate(invalid_condition) - - def test_validate_missing_filter(self): - """Test validating condition without filter.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "no_filter_validate.json" - conditions = Conditions(file=file_path) - - invalid_condition = {"id": str(uuid.uuid4()), "name": "no_filter_test"} - - with pytest.raises(ValueError, match="No filter found"): - conditions.validate(invalid_condition) - - def test_validate_invalid_filter(self): - """Test validating condition with invalid filter syntax.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid_filter.json" - conditions = Conditions(file=file_path) - - # Use a filter that will cause a syntax error in the parser - invalid_condition = { - "id": str(uuid.uuid4()), - "name": "invalid_filter_test", - "filter": "duration > & < 60", # Invalid syntax with consecutive operators - "cli": "", - "extras": {}, - } - - with pytest.raises(ValueError, match="Invalid filter"): - conditions.validate(invalid_condition) - - def test_validate_invalid_cli(self): - """Test validating condition with invalid CLI options.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid_cli.json" - conditions = Conditions(file=file_path) - - invalid_condition = { - "id": str(uuid.uuid4()), - "name": "invalid_cli_test", - "filter": "duration > 60", - "cli": "--invalid-option-that-does-not-exist", - } - - with pytest.raises(ValueError, match="Invalid command options"): - conditions.validate(invalid_condition) - - def test_validate_invalid_extras_type(self): - """Test validating condition with non-dict extras.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid_extras.json" - conditions = Conditions(file=file_path) - - invalid_condition = { - "id": str(uuid.uuid4()), - "name": "invalid_extras_test", - "filter": "duration > 60", - "extras": "not a dict", - } - - with pytest.raises(ValueError, match="Extras must be a dictionary"): - conditions.validate(invalid_condition) - - def test_validate_invalid_enabled_type(self): - """Test validating condition with non-boolean enabled field.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid_enabled.json" - conditions = Conditions(file=file_path) - - invalid_condition = { - "id": str(uuid.uuid4()), - "name": "invalid_enabled_test", - "filter": "duration > 60", - "extras": {}, - "enabled": "yes", # Should be boolean, not string - } - - with pytest.raises(ValueError, match="Enabled must be a boolean"): - conditions.validate(invalid_condition) - - def test_validate_invalid_priority_type(self): - """Test validating condition with non-integer priority field.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid_priority_type.json" - conditions = Conditions(file=file_path) - - invalid_condition = { - "id": str(uuid.uuid4()), - "name": "invalid_priority_type_test", - "filter": "duration > 60", - "extras": {}, - "priority": "10", # Should be integer, not string - } - - with pytest.raises(ValueError, match="Priority must be an integer"): - conditions.validate(invalid_condition) - - def test_validate_invalid_priority_negative(self): - """Test validating condition with negative priority.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid_priority_negative.json" - conditions = Conditions(file=file_path) - - invalid_condition = { - "id": str(uuid.uuid4()), - "name": "invalid_priority_negative_test", - "filter": "duration > 60", - "extras": {}, - "priority": -5, # Should be >= 0 - } - - with pytest.raises(ValueError, match="Priority must be >= 0"): - conditions.validate(invalid_condition) - - def test_validate_invalid_description_type(self): - """Test validating condition with non-string description field.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid_description.json" - conditions = Conditions(file=file_path) - - invalid_condition = { - "id": str(uuid.uuid4()), - "name": "invalid_description_test", - "filter": "duration > 60", - "extras": {}, - "description": 123, # Should be string, not integer - } - - with pytest.raises(ValueError, match="Description must be a string"): - conditions.validate(invalid_condition) - - def test_validate_invalid_item_type(self): - """Test validating invalid item type.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid_type.json" - conditions = Conditions(file=file_path) - - with pytest.raises(ValueError, match=r"Unexpected.*item type"): - conditions.validate("invalid type") - - def test_save_conditions(self): - """Test saving conditions to file.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "save_test.json" - conditions = Conditions(file=file_path) - - test_conditions = [ - Condition(name="save_test1", filter="duration > 60"), - Condition(name="save_test2", filter="uploader = 'test'"), - ] - - result = conditions.save(test_conditions) - - assert result is conditions - assert file_path.exists() - - # Verify file content - saved_data = json.loads(file_path.read_text()) - assert len(saved_data) == 2 - assert saved_data[0]["name"] == "save_test1" - assert saved_data[1]["name"] == "save_test2" - - def test_save_conditions_dict_format(self): - """Test saving conditions in dictionary format.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "save_dict_test.json" - conditions = Conditions(file=file_path) - - test_conditions = [ - {"id": str(uuid.uuid4()), "name": "dict_test", "filter": "duration < 300", "cli": "", "extras": {}} - ] - - conditions.save(test_conditions) - - assert file_path.exists() - saved_data = json.loads(file_path.read_text()) - assert len(saved_data) == 1 - assert saved_data[0]["name"] == "dict_test" - - def test_has_condition_by_id(self): - """Test checking if condition exists by ID.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "has_test.json" - conditions = Conditions(file=file_path) - - test_id = str(uuid.uuid4()) - test_condition = Condition(id=test_id, name="has_test", filter="duration > 60") - conditions._items = [test_condition] - - assert conditions.has(test_id) is True - assert conditions.has("nonexistent") is False - - def test_has_condition_by_name(self): - """Test checking if condition exists by name.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "has_name_test.json" - conditions = Conditions(file=file_path) - - test_condition = Condition(name="has_name_test", filter="uploader = 'test'") - conditions._items = [test_condition] - - assert conditions.has("has_name_test") is True - assert conditions.has("nonexistent_name") is False - - def test_get_condition_by_id(self): - """Test getting condition by ID.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "get_id_test.json" - conditions = Conditions(file=file_path) - - test_id = str(uuid.uuid4()) - test_condition = Condition(id=test_id, name="get_id_test", filter="duration > 120") - conditions._items = [test_condition] - - result = conditions.get(test_id) - assert result is test_condition - - assert conditions.get("nonexistent") is None - - def test_get_condition_by_name(self): - """Test getting condition by name.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "get_name_test.json" - conditions = Conditions(file=file_path) - - test_condition = Condition(name="get_name_test", filter="title ~= 'music'") - conditions._items = [test_condition] - - result = conditions.get("get_name_test") - assert result is test_condition - - def test_get_condition_empty_id_or_name(self): - """Test getting condition with empty ID or name.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "empty_get_test.json" - conditions = Conditions(file=file_path) - - assert conditions.get("") is None - assert conditions.get(None) is None - - @patch("app.library.conditions.match_str") - def test_match_condition_found(self, mock_match_str): - """Test matching condition against info dict.""" - mock_match_str.return_value = True - - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "match_test.json" - conditions = Conditions(file=file_path) - - test_condition = Condition(name="match_test", filter="duration > 60") - conditions._items = [test_condition] - - info_dict = {"duration": 120, "title": "Test Video"} - result = conditions.match(info_dict) - - assert result is test_condition - mock_match_str.assert_called_once_with("duration > 60", info_dict) - - @patch("app.library.conditions.match_str") - def test_match_condition_not_found(self, mock_match_str): - """Test matching when no condition matches.""" - mock_match_str.return_value = False - - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "no_match_test.json" - conditions = Conditions(file=file_path) - - test_condition = Condition(name="no_match_test", filter="duration > 300") - conditions._items = [test_condition] - - info_dict = {"duration": 60, "title": "Short Video"} - result = conditions.match(info_dict) - - assert result is None - - def test_match_empty_conditions(self): - """Test matching with empty conditions list.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "empty_match_test.json" - conditions = Conditions(file=file_path) - - info_dict = {"duration": 120} - result = conditions.match(info_dict) - - assert result is None - - def test_match_invalid_info_dict(self): - """Test matching with invalid info dict.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid_info_test.json" - conditions = Conditions(file=file_path) - - test_condition = Condition(name="test", filter="duration > 60") - conditions._items = [test_condition] - - assert conditions.match(None) is None, "Test with None" - - assert conditions.match({}) is None, "Test with empty dict" - - assert conditions.match("not a dict") is None, "Test with non-dict" - - @patch("app.library.conditions.match_str") - def test_match_filter_evaluation_error(self, mock_match_str): - """Test matching when filter evaluation raises exception.""" - mock_match_str.side_effect = Exception("Filter error") - - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "filter_error_test.json" - conditions = Conditions(file=file_path) - - test_condition = Condition(name="error_test", filter="invalid filter") - conditions._items = [test_condition] - - info_dict = {"duration": 120} - result = conditions.match(info_dict) - - assert result is None - - @patch("app.library.conditions.match_str") - def test_match_disabled_condition(self, mock_match_str): - """Test that disabled conditions are skipped during matching.""" - mock_match_str.return_value = True - - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "disabled_match_test.json" - conditions = Conditions(file=file_path) - - # Add disabled condition - disabled_condition = Condition(name="disabled_test", filter="duration > 60", enabled=False) - enabled_condition = Condition(name="enabled_test", filter="duration > 120", enabled=True) - conditions._items = [disabled_condition, enabled_condition] - - info_dict = {"duration": 150} - result = conditions.match(info_dict) - - assert result is enabled_condition, "Should skip disabled condition and match enabled one" - # Should only call match_str once for enabled condition - mock_match_str.assert_called_once_with("duration > 120", info_dict) - - @patch("app.library.conditions.match_str") - def test_match_priority_sorting(self, mock_match_str): - """Test that conditions are evaluated by priority (highest first).""" - # All filters will match, so we test that highest priority wins - mock_match_str.return_value = True - - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "priority_test.json" - conditions = Conditions(file=file_path) - - # Add conditions with different priorities (not in priority order) - low_priority = Condition(name="low", filter="duration > 60", priority=1) - high_priority = Condition(name="high", filter="duration > 60", priority=10) - default_priority = Condition(name="default", filter="duration > 60", priority=0) - - # Add in wrong order to verify sorting - conditions._items = [low_priority, default_priority, high_priority] - - info_dict = {"duration": 120} - result = conditions.match(info_dict) - - assert result is high_priority, "Should match high_priority first (priority=10)" - # Should only call match_str once for highest priority condition - mock_match_str.assert_called_once_with("duration > 60", info_dict) - - def test_match_empty_filter(self): - """Test matching with condition that has empty filter.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "empty_filter_test.json" - conditions = Conditions(file=file_path) - - test_condition = Condition(name="empty_filter", filter="") - conditions._items = [test_condition] - - info_dict = {"duration": 120} - result = conditions.match(info_dict) - - assert result is None - - @patch("app.library.conditions.match_str") - def test_single_match_found(self, mock_match_str): - """Test single condition matching.""" - mock_match_str.return_value = True - - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "single_match_test.json" - conditions = Conditions(file=file_path) - - test_condition = Condition(name="single_test", filter="uploader = 'test'") - conditions._items = [test_condition] - - info_dict = {"uploader": "test", "title": "Test Video"} - result = conditions.single_match("single_test", info_dict) - - assert result is test_condition - mock_match_str.assert_called_once_with("uploader = 'test'", info_dict) - - @patch("app.library.conditions.match_str") - def test_single_match_not_found(self, mock_match_str): - """Test single condition matching when condition doesn't match.""" - mock_match_str.return_value = False - - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "single_no_match_test.json" - conditions = Conditions(file=file_path) - - test_condition = Condition(name="single_no_match", filter="uploader = 'other'") - conditions._items = [test_condition] - - info_dict = {"uploader": "test", "title": "Test Video"} - result = conditions.single_match("single_no_match", info_dict) - - assert result is None - - def test_single_match_nonexistent_condition(self): - """Test single matching with non-existent condition name.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "nonexistent_single_test.json" - conditions = Conditions(file=file_path) - - info_dict = {"duration": 120} - result = conditions.single_match("nonexistent", info_dict) - - assert result is None - - def test_single_match_condition_no_filter(self): - """Test single matching with condition that has no filter.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "no_filter_single_test.json" - conditions = Conditions(file=file_path) - - test_condition = Condition(name="no_filter_single", filter="") - conditions._items = [test_condition] - - info_dict = {"duration": 120} - result = conditions.single_match("no_filter_single", info_dict) - - assert result is None - - def test_single_match_disabled_condition(self): - """Test single matching with disabled condition.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "disabled_single_test.json" - conditions = Conditions(file=file_path) - - test_condition = Condition(name="disabled_single", filter="duration > 60", enabled=False) - conditions._items = [test_condition] - - info_dict = {"duration": 120} - result = conditions.single_match("disabled_single", info_dict) - - assert result is None, "Should return None because condition is disabled" - - def test_single_match_invalid_inputs(self): - """Test single matching with invalid inputs.""" - with tempfile.TemporaryDirectory() as temp_dir: - file_path = Path(temp_dir) / "invalid_single_test.json" - conditions = Conditions(file=file_path) - - assert conditions.single_match("test", {"duration": 120}) is None, "Test with empty conditions" - - assert conditions.single_match("test", None) is None, "Test with None info" - - assert conditions.single_match("test", {}) is None, "Test with empty info dict" - - assert conditions.single_match("test", "not a dict") is None, "Test with non-dict info" diff --git a/app/tests/test_datastore.py b/app/tests/test_datastore.py index 8b34c3e0..336397d7 100644 --- a/app/tests/test_datastore.py +++ b/app/tests/test_datastore.py @@ -13,11 +13,25 @@ from app.library.operations import Operation from app.library.sqlite_store import SqliteStore +async def reset_sqlite_store() -> None: + """Close and reset SqliteStore singleton for testing.""" + if SqliteStore in SqliteStore._instances: + instance = SqliteStore._instances[SqliteStore] + # Only close if there's an active connection to avoid event loop issues + if instance._conn is not None: + try: + await instance.close() + except RuntimeError: + # Event loop issues - just reset without closing + pass + SqliteStore._reset_singleton() + + async def make_db(data: int = 0) -> SqliteStore: """Create a temporary database with test data.""" - SqliteStore._reset_singleton() + await reset_sqlite_store() ins = SqliteStore.get_instance(db_path=":memory:") - await ins._ensure_conn() + await ins.get_connection() base_time = datetime.now(UTC) for i in range(data): @@ -33,7 +47,7 @@ async def make_db(data: int = 0) -> SqliteStore: "status": "finished", } - await ins._conn.execute( + await ins.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( f"test-id-{i}", @@ -43,8 +57,6 @@ async def make_db(data: int = 0) -> SqliteStore: created_at, ), ) - if data > 0: - await ins._conn.commit() return ins @@ -91,11 +103,10 @@ class TestDataStore: data = asdict(dto) data.pop("_id", None) created = datetime(2024, 1, 2, 3, 4, 5, tzinfo=UTC) - await db._conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ("abc", "queue", "http://x", json.dumps(data), created.strftime("%Y-%m-%d %H:%M:%S")), ) - await db._conn.commit() items = await store.saved_items() assert len(items) == 1 @@ -119,21 +130,20 @@ class TestDataStore: assert ret is store._dict[item._id] # Verify row written; JSON should not contain datetime field - cur = await db._conn.execute("SELECT * FROM history WHERE id=?", (item._id,)) - row = await cur.fetchone() - assert row is not None - assert row["type"] == "queue" - assert row["url"] == item.url - assert '"datetime"' not in row["data"] - assert row["id"] == item._id + rows = await db.fetch_raw("SELECT * FROM history WHERE id=?", (item._id,)) + assert len(rows) == 1, "Should find one row" + row = rows[0] + assert row["type"] == "queue", "Type should be queue" + assert row["url"] == item.url, "URL should match" + assert '"datetime"' not in row["data"], "JSON should not contain datetime field" + assert row["id"] == item._id, "ID should match" # Delete and ensure removal await store.delete(item._id) await asyncio.sleep(0) await db.flush() - cur2 = await db._conn.execute("SELECT * FROM history WHERE id=?", (item._id,)) - row2 = await cur2.fetchone() - assert row2 is None + rows2 = await db.fetch_raw("SELECT * FROM history WHERE id=?", (item._id,)) + assert len(rows2) == 0, "Row should be deleted" await db.close() @pytest.mark.asyncio @@ -368,7 +378,6 @@ class TestDataStore: async def test_load(self) -> None: """Test loading items from database into memory.""" db = await make_db() - conn = db._conn # Insert items directly into database item1_data = asdict(make_item(id="vid1", url="http://example.com/1", title="Video 1")) @@ -377,7 +386,7 @@ class TestDataStore: item2_data.pop("_id", None) created = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id1", @@ -387,7 +396,7 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id2", @@ -397,7 +406,6 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) - await conn.commit() store = DataStore(StoreType.QUEUE, db) assert len(store._dict) == 0 @@ -414,7 +422,7 @@ class TestDataStore: async def test_load_with_different_store_types(self) -> None: """Test that load only loads items matching the store type.""" db = await make_db() - conn = db._conn + # conn = db._conn # No longer needed # Insert items with different types item1_data = asdict(make_item(id="vid1", url="http://example.com/1")) @@ -423,7 +431,7 @@ class TestDataStore: item2_data.pop("_id", None) created = datetime(2024, 1, 1, 12, 0, 0, tzinfo=UTC) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id1", @@ -433,7 +441,7 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", ( "id2", @@ -443,7 +451,7 @@ class TestDataStore: created.strftime("%Y-%m-%d %H:%M:%S"), ), ) - await conn.commit() + # await conn.commit() # Auto-committed # Load QUEUE store - should only get queue items queue_store = DataStore(StoreType.QUEUE, db) @@ -541,18 +549,18 @@ class TestDataStore: """Test get_total_count with items in database.""" store = await make_store_async(StoreType.QUEUE) db = store._connection - conn = db._conn + # conn = db._conn # No longer needed # Add items directly to database created = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S") for i in range(5): item_data = asdict(make_item(id=f"vid{i}")) item_data.pop("_id", None) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", (f"id{i}", str(StoreType.QUEUE), f"http://example.com/{i}", json.dumps(item_data), created), ) - await conn.commit() + # await conn.commit() # Auto-committed count = await store.get_total_count() assert count == 5 @@ -562,7 +570,7 @@ class TestDataStore: async def test_get_total_count_respects_store_type(self) -> None: """Test that get_total_count only counts items of the correct type.""" db = await make_db() - conn = db._conn + # conn = db._conn # No longer needed created = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S") @@ -570,7 +578,7 @@ class TestDataStore: for i in range(3): item_data = asdict(make_item(id=f"vid{i}")) item_data.pop("_id", None) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", (f"q{i}", str(StoreType.QUEUE), f"http://example.com/queue/{i}", json.dumps(item_data), created), ) @@ -579,11 +587,11 @@ class TestDataStore: for i in range(2): item_data = asdict(make_item(id=f"vid{i}")) item_data.pop("_id", None) - await conn.execute( + await db.execute_raw( "INSERT INTO history (id, type, url, data, created_at) VALUES (?, ?, ?, ?, ?)", (f"h{i}", str(StoreType.HISTORY), f"http://example.com/history/{i}", json.dumps(item_data), created), ) - await conn.commit() + # await conn.commit() # Auto-committed queue_store = DataStore(StoreType.QUEUE, db) assert await queue_store.get_total_count() == 3 @@ -641,9 +649,8 @@ class TestDataStore: # Verify nothing was deleted from database await store._connection.flush() - cursor = await db._conn.execute("SELECT * FROM history WHERE id=?", ("nonexistent_id",)) - row = await cursor.fetchone() - assert row is None + rows = await db.fetch_raw("SELECT * FROM history WHERE id=?", ("nonexistent_id",)) + assert len(rows) == 0, "Should find no rows" await db.close() @pytest.mark.asyncio @@ -720,10 +727,9 @@ class TestDataStore: await store._connection.flush() # Verify datetime field is not in stored JSON - cursor = await db._conn.execute("SELECT data FROM history WHERE id=?", (item._id,)) - row = await cursor.fetchone() - assert row is not None - data = json.loads(row["data"]) + rows = await db.fetch_raw("SELECT data FROM history WHERE id=?", (item._id,)) + assert len(rows) == 1, "Should find one row" + data = json.loads(rows[0]["data"]) assert "datetime" not in data await db.close() @@ -742,10 +748,9 @@ class TestDataStore: await store._connection.flush() # Verify live_in field is not in stored JSON when status is finished - cursor = await conn._conn.execute("SELECT data FROM history WHERE id=?", (item._id,)) - row = await cursor.fetchone() - assert row is not None - data = json.loads(row["data"]) + rows = await conn.fetch_raw("SELECT data FROM history WHERE id=?", (item._id,)) + assert len(rows) == 1, "Should find one row" + data = json.loads(rows[0]["data"]) assert "live_in" not in data await store._connection.close() @@ -764,10 +769,9 @@ class TestDataStore: await store._connection.flush() # Verify live_in field IS in stored JSON when status is not finished - cursor = await conn._conn.execute("SELECT data FROM history WHERE id=?", (item._id,)) - row = await cursor.fetchone() - assert row is not None - data = json.loads(row["data"]) + rows = await conn.fetch_raw("SELECT data FROM history WHERE id=?", (item._id,)) + assert len(rows) == 1, "Should find one row" + data = json.loads(rows[0]["data"]) assert "live_in" in data assert data["live_in"] == "PT5M" await store._connection.close() diff --git a/app/tests/test_datastore_pagination.py b/app/tests/test_datastore_pagination.py index 7c54b286..031fc7fb 100644 --- a/app/tests/test_datastore_pagination.py +++ b/app/tests/test_datastore_pagination.py @@ -10,11 +10,25 @@ from app.library.ItemDTO import ItemDTO from app.library.sqlite_store import SqliteStore +async def reset_sqlite_store() -> None: + """Close and reset SqliteStore singleton for testing.""" + if SqliteStore in SqliteStore._instances: + instance = SqliteStore._instances[SqliteStore] + # Only close if there's an active connection to avoid event loop issues + if instance._conn is not None: + try: + await instance.close() + except RuntimeError: + # Event loop issues - just reset without closing + pass + SqliteStore._reset_singleton() + + async def make_db(data: int = 100) -> SqliteStore: """Create a temporary database with test data.""" - SqliteStore._reset_singleton() + await reset_sqlite_store() ins = SqliteStore.get_instance(db_path=":memory:") - await ins._ensure_conn() + await ins.get_connection() base_time = datetime.now(UTC) for i in range(data): @@ -30,7 +44,7 @@ async def make_db(data: int = 100) -> SqliteStore: "status": "finished", } - await ins._conn.execute( + await ins.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( f"test-id-{i}", @@ -40,8 +54,6 @@ async def make_db(data: int = 100) -> SqliteStore: created_at, ), ) - if data > 0: - await ins._conn.commit() return ins @@ -226,7 +238,7 @@ class TestDataStorePagination: db = await make_db(data=100) try: - await db._conn.execute( + await db.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( "pending-id", @@ -236,7 +248,7 @@ class TestDataStorePagination: datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), ), ) - await db._conn.execute( + await db.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( "downloading-id", @@ -246,7 +258,6 @@ class TestDataStorePagination: datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), ), ) - await db._conn.commit() datastore = DataStore(type=StoreType.HISTORY, connection=db) # Filter for finished items only @@ -287,7 +298,7 @@ class TestDataStorePagination: } db = await make_db(data=0) try: - await db._conn.execute( + await db.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( "pending-id-2", @@ -297,7 +308,7 @@ class TestDataStorePagination: datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), ), ) - await db._conn.execute( + await db.execute_raw( 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', ( "error-id", @@ -307,7 +318,6 @@ class TestDataStorePagination: datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), ), ) - await db._conn.commit() datastore = DataStore(type=StoreType.HISTORY, connection=db) diff --git a/app/tests/test_dl_fields.py b/app/tests/test_dl_fields.py deleted file mode 100644 index ba7a9e6c..00000000 --- a/app/tests/test_dl_fields.py +++ /dev/null @@ -1,621 +0,0 @@ -""" -Tests for dl_fields.py - Download fields management. - -This test suite provides comprehensive coverage for the dl_fields module: -- Tests FieldType enum functionality -- Tests DLField dataclass functionality -- Tests DLFields singleton class behavior -- Tests field loading, saving, and validation -- Tests field CRUD operations -- Tests error handling and edge cases - -Total test functions: 25+ -All dl_fields management functionality and edge cases are covered. -""" - -import json -import tempfile -import uuid -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -from app.library.dl_fields import DLField, DLFields, FieldType - - -class TestFieldType: - """Test the FieldType enum.""" - - def test_field_type_values(self): - """Test FieldType enum values.""" - assert FieldType.STRING == "string" - assert FieldType.TEXT == "text" - assert FieldType.BOOL == "bool" - - def test_field_type_all(self): - """Test FieldType.all() method.""" - all_types = FieldType.all() - expected = ["string", "text", "bool"] - assert all_types == expected - assert len(all_types) == 3 - - def test_field_type_from_value_valid(self): - """Test FieldType.from_value() with valid values.""" - assert FieldType.from_value("string") == FieldType.STRING - assert FieldType.from_value("text") == FieldType.TEXT - assert FieldType.from_value("bool") == FieldType.BOOL - - def test_field_type_from_value_invalid(self): - """Test FieldType.from_value() with invalid values.""" - with pytest.raises(ValueError, match="Invalid StoreType value"): - FieldType.from_value("invalid") - - with pytest.raises(ValueError, match="Invalid StoreType value"): - FieldType.from_value("number") - - with pytest.raises(ValueError, match="Invalid StoreType value"): - FieldType.from_value("") - - def test_field_type_str(self): - """Test FieldType string conversion.""" - assert str(FieldType.STRING) == "string" - assert str(FieldType.TEXT) == "text" - assert str(FieldType.BOOL) == "bool" - - -class TestDLField: - """Test the DLField dataclass.""" - - def test_dl_field_creation_with_defaults(self): - """Test creating a DLField with default values.""" - field = DLField(name="test_field", description="Test description", field="--test-option") - - assert field.id, "Check that ID is generated" - assert isinstance(field.id, str) - - # Check required fields - assert field.name == "test_field" - assert field.description == "Test description" - assert field.field == "--test-option" - - assert field.kind == FieldType.TEXT, "Check defaults" - assert field.icon == "" - assert field.order == 0 - assert field.value == "" - assert field.extras == {} - - def test_dl_field_creation_with_all_fields(self): - """Test creating a DLField with all fields specified.""" - test_id = str(uuid.uuid4()) - extras = {"key": "value", "number": 42} - - field = DLField( - id=test_id, - name="custom_field", - description="Custom description", - field="--custom-option", - kind=FieldType.BOOL, - icon="fa-check", - order=5, - value="default_value", - extras=extras, - ) - - assert field.id == test_id - assert field.name == "custom_field" - assert field.description == "Custom description" - assert field.field == "--custom-option" - assert field.kind == FieldType.BOOL - assert field.icon == "fa-check" - assert field.order == 5 - assert field.value == "default_value" - assert field.extras == extras - - def test_dl_field_serialize(self): - """Test DLField serialization.""" - extras = {"key": "value", "none_value": None} - field = DLField(name="test", description="Test field", field="--test", kind=FieldType.STRING, extras=extras) - - serialized = field.serialize() - - assert serialized["name"] == "test" - assert serialized["description"] == "Test field" - assert serialized["field"] == "--test" - assert serialized["kind"] == "string" - assert serialized["extras"] == {"key": "value"} # None values filtered out - - def test_dl_field_json(self): - """Test DLField JSON encoding.""" - field = DLField(name="json_test", description="JSON test field", field="--json-test") - - json_str = field.json() - assert isinstance(json_str, str) - - # Should be valid JSON - parsed = json.loads(json_str) - assert parsed["name"] == "json_test" - assert parsed["description"] == "JSON test field" - assert parsed["field"] == "--json-test" - - def test_dl_field_get_method(self): - """Test DLField get method.""" - field = DLField(name="get_test", description="Get test field", field="--get-test", order=3) - - assert field.get("name") == "get_test" - assert field.get("order") == 3 - assert field.get("nonexistent") is None - assert field.get("nonexistent", "default") == "default" - - -class TestDLFields: - """Test the DLFields class.""" - - def setup_method(self): - """Set up test fixtures.""" - DLFields._reset_singleton() - - @pytest.fixture - def temp_file(self): - """Create a temporary file for testing.""" - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - temp_path = Path(f.name) - yield temp_path - if temp_path.exists(): - temp_path.unlink() - - @pytest.fixture - def sample_fields_data(self): - """Sample field data for testing.""" - return [ - { - "id": str(uuid.uuid4()), - "name": "quality", - "description": "Video quality setting", - "field": "--format", - "kind": "string", - "icon": "fa-video", - "order": 1, - "value": "best", - "extras": {"options": ["best", "worst", "720p"]}, - }, - { - "id": str(uuid.uuid4()), - "name": "audio_only", - "description": "Extract audio only", - "field": "--extract-audio", - "kind": "bool", - "icon": "fa-music", - "order": 2, - "value": "", - "extras": {}, - }, - ] - - @patch("app.library.dl_fields.Config") - def test_dl_fields_singleton_behavior(self, mock_config): - """Test that DLFields follows singleton pattern.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields1 = DLFields() - fields2 = DLFields() - assert fields1 is fields2 - - @patch("app.library.dl_fields.Config") - def test_dl_fields_get_instance(self, mock_config): - """Test DLFields.get_instance() method.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields1 = DLFields.get_instance() - fields2 = DLFields.get_instance() - assert fields1 is fields2 - - @patch("app.library.dl_fields.Config") - def test_dl_fields_initialization(self, mock_config): - """Test DLFields initialization.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - with tempfile.TemporaryDirectory() as temp_dir: - temp_file = Path(temp_dir) / "test_fields.json" - - fields = DLFields(file=str(temp_file)) - - assert fields._file == temp_file - - @patch("app.library.dl_fields.Config") - def test_dl_fields_load_empty_file(self, mock_config, temp_file): - """Test loading from empty/non-existent file.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields(file=str(temp_file)) - result = fields.load() - - assert result is fields - assert fields.get_all() == [] - - @patch("app.library.dl_fields.Config") - def test_dl_fields_load_valid_file(self, mock_config, temp_file, sample_fields_data): - """Test loading from valid JSON file.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - # Write sample data to temp file - temp_file.write_text(json.dumps(sample_fields_data, indent=2)) - - fields = DLFields(file=str(temp_file)) - fields.load() - - loaded_fields = fields.get_all() - assert len(loaded_fields) == 2 - - # Check first field - field1 = loaded_fields[0] - assert field1.name == "quality" - assert field1.description == "Video quality setting" - assert field1.field == "--format" - assert field1.kind == FieldType.STRING - - # Check second field - field2 = loaded_fields[1] - assert field2.name == "audio_only" - assert field2.kind == FieldType.BOOL - - @patch("app.library.dl_fields.Config") - def test_dl_fields_load_invalid_json(self, mock_config, temp_file): - """Test loading from invalid JSON file.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - # Write invalid JSON - temp_file.write_text("invalid json content") - - fields = DLFields(file=str(temp_file)) - fields.load() - - assert fields.get_all() == [], "Should handle error gracefully and return empty list" - - @patch("app.library.dl_fields.Config") - def test_dl_fields_load_missing_id_auto_generation(self, mock_config, temp_file): - """Test that missing IDs are auto-generated during load.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - # Sample data without IDs - data_without_ids = [ - {"name": "test_field", "description": "Test description", "field": "--test", "kind": "string"} - ] - - temp_file.write_text(json.dumps(data_without_ids)) - - with patch.object(DLFields, "save") as mock_save: - fields = DLFields(file=str(temp_file)) - fields.load() - - # Should auto-generate ID and trigger save - loaded_fields = fields.get_all() - assert len(loaded_fields) == 1 - assert loaded_fields[0].id is not None - mock_save.assert_called_once() - - @patch("app.library.dl_fields.Config") - def test_dl_fields_clear(self, mock_config): - """Test clearing all fields.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - # Add some fields manually - fields._items = [ - DLField(name="test1", description="Test 1", field="--test1"), - DLField(name="test2", description="Test 2", field="--test2"), - ] - - assert len(fields.get_all()) == 2 - - result = fields.clear() - assert result is fields - assert len(fields.get_all()) == 0 - - @patch("app.library.dl_fields.Config") - def test_dl_fields_clear_empty(self, mock_config): - """Test clearing when already empty.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - assert len(fields.get_all()) == 0 - - result = fields.clear() - assert result is fields - assert len(fields.get_all()) == 0 - - @patch("app.library.dl_fields.Config") - def test_dl_fields_validate_valid_field(self, mock_config): - """Test validation with valid DLField.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - valid_field = DLField( - name="valid_field", description="Valid description", field="--valid-option", kind=FieldType.STRING - ) - - assert fields.validate(valid_field) is True - - @patch("app.library.dl_fields.Config") - def test_dl_fields_validate_valid_dict(self, mock_config): - """Test validation with valid dictionary.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - valid_dict = { - "id": str(uuid.uuid4()), - "name": "test_field", - "description": "Test description", - "field": "--test-field", - "kind": "text", - } - - assert fields.validate(valid_dict) is True - - @patch("app.library.dl_fields.Config") - def test_dl_fields_validate_missing_required_fields(self, mock_config): - """Test validation with missing required fields.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - # Missing name - invalid_dict = { - "id": str(uuid.uuid4()), - "description": "Test description", - "field": "--test-field", - "kind": "text", - } - - with pytest.raises(ValueError, match="Missing required key 'name'"): - fields.validate(invalid_dict) - - @patch("app.library.dl_fields.Config") - def test_dl_fields_validate_invalid_field_type(self, mock_config): - """Test validation with invalid field type.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - invalid_dict = { - "id": str(uuid.uuid4()), - "name": "test_field", - "description": "Test description", - "field": "--test-field", - "kind": "invalid_type", - } - - with pytest.raises(ValueError, match="Invalid field type"): - fields.validate(invalid_dict) - - @patch("app.library.dl_fields.Config") - def test_dl_fields_validate_invalid_yt_dlp_field(self, mock_config): - """Test validation with invalid yt-dlp field format.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - invalid_dict = { - "id": str(uuid.uuid4()), - "name": "test_field", - "description": "Test description", - "field": "invalid-field", # Missing -- - "kind": "text", - } - - with pytest.raises(ValueError, match="Invalid yt-dlp option field"): - fields.validate(invalid_dict) - - @patch("app.library.dl_fields.Config") - def test_dl_fields_validate_invalid_extras_type(self, mock_config): - """Test validation with invalid extras type.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - invalid_dict = { - "id": str(uuid.uuid4()), - "name": "test_field", - "description": "Test description", - "field": "--test-field", - "kind": "text", - "extras": "not_a_dict", - } - - with pytest.raises(ValueError, match="Extras must be a dictionary"): - fields.validate(invalid_dict) - - @patch("app.library.dl_fields.Config") - def test_dl_fields_validate_invalid_value_type(self, mock_config): - """Test validation with invalid value type.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - invalid_dict = { - "id": str(uuid.uuid4()), - "name": "test_field", - "description": "Test description", - "field": "--test-field", - "kind": "text", - "value": 123, # Should be string - } - - with pytest.raises(ValueError, match="Value must be a string"): - fields.validate(invalid_dict) - - @patch("app.library.dl_fields.Config") - def test_dl_fields_validate_invalid_order_type(self, mock_config): - """Test validation with invalid order type.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - invalid_dict = { - "id": str(uuid.uuid4()), - "name": "test_field", - "description": "Test description", - "field": "--test-field", - "kind": "text", - "order": "not_an_int", - } - - with pytest.raises(ValueError, match="Order must be an integer"): - fields.validate(invalid_dict) - - @patch("app.library.dl_fields.Config") - def test_dl_fields_validate_unexpected_type(self, mock_config): - """Test validation with unexpected item type.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - with pytest.raises(ValueError, match="Unexpected 'str' type was given"): - fields.validate("not_a_field_or_dict") - - @patch("app.library.dl_fields.Config") - def test_dl_fields_save_valid_fields(self, mock_config, temp_file): - """Test saving valid fields.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields(file=str(temp_file)) - - test_fields = [DLField(name="test_field", description="Test description", field="--test-option")] - - fields.save(test_fields) - - assert temp_file.exists(), "Verify file was written" - - # Verify content - saved_data = json.loads(temp_file.read_text()) - assert len(saved_data) == 1 - assert saved_data[0]["name"] == "test_field" - - @patch("app.library.dl_fields.Config") - def test_dl_fields_save_mixed_types(self, mock_config, temp_file): - """Test saving mix of DLField objects and dicts.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields(file=str(temp_file)) - - test_items = [ - DLField(name="field1", description="Field 1", field="--field1"), - {"id": str(uuid.uuid4()), "name": "field2", "description": "Field 2", "field": "--field2", "kind": "text"}, - ] - - fields.save(test_items) - - # Verify both items were saved - saved_data = json.loads(temp_file.read_text()) - assert len(saved_data) == 2 - - @patch("app.library.dl_fields.Config") - def test_dl_fields_get_by_id(self, mock_config): - """Test getting field by ID.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - test_id = str(uuid.uuid4()) - test_field = DLField(id=test_id, name="test_field", description="Test description", field="--test-option") - - fields._items = [test_field] - - result = fields.get(test_id) - assert result is test_field - - @patch("app.library.dl_fields.Config") - def test_dl_fields_get_by_name(self, mock_config): - """Test getting field by name.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - test_field = DLField(name="test_field", description="Test description", field="--test-option") - - fields._items = [test_field] - - result = fields.get("test_field") - assert result is test_field - - @patch("app.library.dl_fields.Config") - def test_dl_fields_get_not_found(self, mock_config): - """Test getting non-existent field.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - result = fields.get("nonexistent") - assert result is None - - @patch("app.library.dl_fields.Config") - def test_dl_fields_get_empty_string(self, mock_config): - """Test getting with empty string.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - result = fields.get("") - assert result is None - - @patch("app.library.dl_fields.Config") - def test_dl_fields_has_by_id(self, mock_config): - """Test checking field existence by ID.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - test_id = str(uuid.uuid4()) - test_field = DLField(id=test_id, name="test_field", description="Test description", field="--test-option") - - fields._items = [test_field] - - assert fields.has(test_id) is True - assert fields.has("nonexistent") is False - - @patch("app.library.dl_fields.Config") - def test_dl_fields_has_by_name(self, mock_config): - """Test checking field existence by name.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - test_field = DLField(name="test_field", description="Test description", field="--test-option") - - fields._items = [test_field] - - assert fields.has("test_field") is True - assert fields.has("nonexistent") is False - - @patch("app.library.dl_fields.Config") - def test_dl_fields_attach_method(self, mock_config): - """Test attach method calls load.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - - with patch.object(fields, "load") as mock_load: - mock_app = MagicMock() - fields.attach(mock_app) - mock_load.assert_called_once() - - @patch("app.library.dl_fields.Config") - def test_dl_fields_on_shutdown(self, mock_config): - """Test on_shutdown method.""" - mock_config.get_instance.return_value.config_path = "/tmp" - - fields = DLFields() - mock_app = MagicMock() - - # Should not raise any exceptions - import asyncio - - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - loop.run_until_complete(fields.on_shutdown(mock_app)) - finally: - loop.close() diff --git a/app/tests/test_events.py b/app/tests/test_events.py index 2b3d2912..a6d3f4a4 100644 --- a/app/tests/test_events.py +++ b/app/tests/test_events.py @@ -20,6 +20,7 @@ class TestEvents: # Connection events assert Events.CONNECTED == "connected" + assert Events.CONFIG_UPDATE == "config_update" # Log events assert Events.LOG_INFO == "log_info" @@ -48,6 +49,7 @@ class TestEvents: "started", "shutdown", "connected", + "config_update", "log_info", "log_warning", "log_error", @@ -85,6 +87,7 @@ class TestEvents: # Check some expected frontend events expected_frontend = [ Events.CONNECTED, + Events.CONFIG_UPDATE, Events.LOG_INFO, Events.LOG_WARNING, Events.LOG_ERROR, @@ -102,7 +105,6 @@ class TestEvents: assert isinstance(debug_events, list) assert Events.ITEM_UPDATED in debug_events - assert Events.CLI_OUTPUT in debug_events class TestEvent: @@ -683,8 +685,10 @@ class TestEventBus: assert result is bus # Should return self for chaining assert Events.TEST in bus._listeners - assert "test_subscriber" in bus._listeners[Events.TEST] - assert isinstance(bus._listeners[Events.TEST]["test_subscriber"], EventListener) + assert any(name == "test_subscriber" for name, _ in bus._listeners[Events.TEST]) + listener = next((listener for name, listener in bus._listeners[Events.TEST] if name == "test_subscriber"), None) + assert listener is not None + assert isinstance(listener, EventListener) @patch("app.library.config.Config") @patch("app.library.BackgroundWorker.BackgroundWorker") @@ -703,7 +707,7 @@ class TestEventBus: for event in events: assert event in bus._listeners - assert "multi_subscriber" in bus._listeners[event] + assert any(name == "multi_subscriber" for name, _ in bus._listeners[event]) @patch("app.library.config.Config") @patch("app.library.BackgroundWorker.BackgroundWorker") @@ -722,7 +726,7 @@ class TestEventBus: all_events = Events.get_all() for event in all_events: assert event in bus._listeners - assert "wildcard_subscriber" in bus._listeners[event] + assert any(name == "wildcard_subscriber" for name, _ in bus._listeners[event]) @patch("app.library.config.Config") @patch("app.library.BackgroundWorker.BackgroundWorker") @@ -741,7 +745,7 @@ class TestEventBus: frontend_events = Events.frontend() for event in frontend_events: assert event in bus._listeners - assert "frontend_subscriber" in bus._listeners[event] + assert any(name == "frontend_subscriber" for name, _ in bus._listeners[event]) @patch("app.library.config.Config") @patch("app.library.BackgroundWorker.BackgroundWorker") @@ -778,10 +782,6 @@ class TestEventBus: assert Events.TEST in bus._listeners assert len(bus._listeners[Events.TEST]) == 1 - # Name should be a UUID - subscriber_name = next(iter(bus._listeners[Events.TEST].keys())) - assert len(subscriber_name) == 36 # UUID string length - @patch("app.library.config.Config") @patch("app.library.BackgroundWorker.BackgroundWorker") def test_event_bus_unsubscribe_single_event(self, mock_bg_worker, mock_config): @@ -796,13 +796,13 @@ class TestEventBus: # First subscribe bus.subscribe(Events.TEST, test_callback, "test_subscriber") - assert "test_subscriber" in bus._listeners[Events.TEST] + assert any(name == "test_subscriber" for name, _ in bus._listeners[Events.TEST]) # Then unsubscribe result = bus.unsubscribe(Events.TEST, "test_subscriber") assert result is bus - assert "test_subscriber" not in bus._listeners[Events.TEST] + assert not any(name == "test_subscriber" for name, _ in bus._listeners[Events.TEST]) @patch("app.library.config.Config") @patch("app.library.BackgroundWorker.BackgroundWorker") diff --git a/app/tests/test_generic_task_handler.py b/app/tests/test_generic_task_handler.py deleted file mode 100644 index 49cad7f3..00000000 --- a/app/tests/test_generic_task_handler.py +++ /dev/null @@ -1,362 +0,0 @@ -import json -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -import pytest - -from app.library.task_handlers.generic import ( - ContainerDefinition, - EngineConfig, - ExtractionRule, - GenericTaskHandler, - RequestConfig, - ResponseConfig, - TaskDefinition, - load_task_definitions, -) -from app.library.Tasks import Task, TaskFailure, TaskResult - - -@pytest.fixture(autouse=True) -def reset_generic_handler(monkeypatch): - monkeypatch.setattr(GenericTaskHandler, "_definitions", []) - monkeypatch.setattr(GenericTaskHandler, "_sources_mtime", {}) - - -def test_load_task_definitions_parses_valid_file(tmp_path: Path): - tasks_dir = tmp_path / "tasks" - tasks_dir.mkdir() - - definition_content = { - "name": "example", - "match": ["https://example.com/articles/*"], - "parse": { - "link": {"type": "css", "expression": ".article a.link::attr(href)"}, - "title": {"type": "css", "expression": ".article .title", "attribute": "text"}, - }, - } - - (tasks_dir / "01-example.json").write_text(json.dumps(definition_content), encoding="utf-8") - - config = SimpleNamespace(config_path=str(tmp_path)) - definitions = load_task_definitions(config=config) - - assert len(definitions) == 1 - definition = definitions[0] - assert definition.name == "example" - assert definition.matchers[0].matches("https://example.com/articles/123") - assert definition.parsers["link"].expression == ".article a.link::attr(href)" - - -def test_load_task_definitions_handles_container(tmp_path: Path): - tasks_dir = tmp_path / "tasks" - tasks_dir.mkdir() - - definition_content = { - "name": "container", - "match": ["https://example.com/cards"], - "parse": { - "items": { - "selector": ".cards .card", - "fields": { - "link": {"type": "css", "expression": ".card-header a", "attribute": "href"}, - "title": {"type": "css", "expression": ".card-header a", "attribute": "text"}, - }, - } - }, - } - - (tasks_dir / "02-container.json").write_text(json.dumps(definition_content), encoding="utf-8") - - config = SimpleNamespace(config_path=str(tmp_path)) - definitions = load_task_definitions(config=config) - - assert len(definitions) == 1 - definition = definitions[0] - assert definition.container is not None - assert definition.container.selector == ".cards .card" - assert "link" in definition.container.fields - assert definition.parsers == {} - - -def test_load_task_definitions_handles_json(tmp_path: Path): - tasks_dir = tmp_path / "tasks" - tasks_dir.mkdir() - - definition_content = { - "name": "json-def", - "match": ["https://example.com/api"], - "response": {"type": "json"}, - "parse": { - "items": { - "type": "jsonpath", - "selector": "items", - "fields": { - "link": {"type": "jsonpath", "expression": "url"}, - "title": {"type": "jsonpath", "expression": "title"}, - }, - } - }, - } - - (tasks_dir / "03-json.json").write_text(json.dumps(definition_content), encoding="utf-8") - - config = SimpleNamespace(config_path=str(tmp_path)) - definitions = load_task_definitions(config=config) - - assert len(definitions) == 1 - definition = definitions[0] - assert definition.response.format == "json" - assert definition.container is not None - assert definition.container.selector_type == "jsonpath" - assert definition.container.fields["link"].type == "jsonpath" - - -def test_parse_items_extracts_values(): - definition = TaskDefinition( - name="example", - source=Path("example.json"), - matchers=[], - engine=EngineConfig(), - request=RequestConfig(), - parsers={ - "link": ExtractionRule(type="css", expression=".article a.link::attr(href)", attribute=None), - "title": ExtractionRule(type="css", expression=".article .title", attribute="text"), - "id": ExtractionRule(type="css", expression=".article", attribute="data-id"), - }, - ) - - html = """ -
- First - First Title -
-
- Second - Second Title -
- """ - - items = GenericTaskHandler._parse_items(definition, html, "https://example.com/base/") - - assert len(items) == 2 - assert items[0]["link"] == "https://example.com/article-101" - assert items[0]["title"] == "First Title" - assert items[0]["id"] == "101" - assert items[1]["link"] == "https://example.com/article-102" - - -def test_parse_items_handles_nested_card_layout(): - definition = TaskDefinition( - name="nested", - source=Path("nested.json"), - matchers=[], - engine=EngineConfig(), - request=RequestConfig(), - parsers={}, - container=ContainerDefinition( - selector_type="css", - selector=".columns .card", - fields={ - "link": ExtractionRule( - type="css", - expression=".card-header a[href]", - attribute="href", - ), - "title": ExtractionRule( - type="css", - expression=".card-header a[href]", - attribute="text", - ), - "poet": ExtractionRule( - type="css", - expression="footer .card-footer-item:first-child a", - attribute="text", - ), - "category": ExtractionRule( - type="css", - expression="footer .card-footer-item:nth-child(2) a", - attribute="text", - ), - }, - ), - ) - - html = """ -
-
-
-
-

- First Poem -

-
- -
-
-
-
-
-

- Second Poem -

-
- -
-
-
- """ - - items = GenericTaskHandler._parse_items(definition, html, "https://example.com") - - assert len(items) == 2 - assert items[0]["link"] == "https://example.com/poems/view/111" - assert items[0]["title"] == "First Poem" - assert items[0]["poet"] == "Poet Alpha" - assert items[0]["category"] == "Category One" - - assert items[1]["link"] == "https://example.com/poems/view/222" - assert items[1]["title"] == "Second Poem" - assert items[1]["poet"] == "Poet Beta" - assert "category" not in items[1] - - -def test_parse_items_handles_json_container(): - definition = TaskDefinition( - name="json", - source=Path("json.json"), - matchers=[], - engine=EngineConfig(), - request=RequestConfig(), - parsers={}, - container=ContainerDefinition( - selector_type="jsonpath", - selector="entries", - fields={ - "link": ExtractionRule(type="jsonpath", expression="url"), - "title": ExtractionRule(type="jsonpath", expression="title"), - "id": ExtractionRule(type="jsonpath", expression="id"), - }, - ), - response=ResponseConfig(format="json"), - ) - - payload = { - "entries": [ - {"url": "/video/1", "title": "First", "id": 1}, - {"url": "https://example.com/video/2", "title": "Second", "id": 2}, - {"title": "Missing Link", "id": 3}, - ] - } - - items = GenericTaskHandler._parse_items( - definition=definition, - html="", - base_url="https://example.com", - json_data=payload, - ) - - assert len(items) == 2 - assert items[0]["link"] == "https://example.com/video/1" - assert items[0]["title"] == "First" - assert items[0]["id"] == "1" - - assert items[1]["link"] == "https://example.com/video/2" - assert items[1]["title"] == "Second" - assert items[1]["id"] == "2" - - -@pytest.mark.asyncio -async def test_generic_task_handler_inspect(monkeypatch): - definition = TaskDefinition( - name="json-inspect", - source=Path("json-inspect.json"), - matchers=[], - engine=EngineConfig(), - request=RequestConfig(), - parsers={}, - container=ContainerDefinition( - selector_type="jsonpath", - selector="items", - fields={ - "link": ExtractionRule(type="jsonpath", expression="url"), - "title": ExtractionRule(type="jsonpath", expression="title"), - }, - ), - response=ResponseConfig(format="json"), - ) - - monkeypatch.setattr( - GenericTaskHandler, - "_find_definition", - classmethod(lambda cls, url: definition), # noqa: ARG005 - ) - - async def fake_fetch_content(url, definition, ytdlp_opts): # noqa: ARG001 - return "", {"items": [{"url": "/video/1", "title": "First"}]} - - monkeypatch.setattr(GenericTaskHandler, "_fetch_content", staticmethod(fake_fetch_content)) - - # Mock extract_info to return valid info with required fields for archive ID generation - def fake_extract_info(config, url, **kwargs): # noqa: ARG001 - return {"id": "test_video_1", "extractor_key": "Example"} - - with patch("app.library.task_handlers.generic.extract_info", side_effect=fake_extract_info): - task = Task(id="inspect", name="Inspect", url="https://example.com/api") - result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task) - - assert isinstance(result, TaskResult) - assert len(result.items) == 1 - item = result.items[0] - assert item.url == "https://example.com/video/1" - assert item.title == "First" - - -def test_parse_items_handles_json_top_level_list(): - definition = TaskDefinition( - name="json-list", - source=Path("json-list.json"), - matchers=[], - engine=EngineConfig(), - request=RequestConfig(), - parsers={}, - container=ContainerDefinition( - selector_type="jsonpath", - selector="[]", - fields={ - "link": ExtractionRule(type="jsonpath", expression="url"), - "title": ExtractionRule(type="jsonpath", expression="title"), - }, - ), - response=ResponseConfig(format="json"), - ) - - payload = [ - {"url": "/video/1", "title": "First"}, - {"url": "/video/2", "title": "Second"}, - ] - - items = GenericTaskHandler._parse_items( - definition=definition, - html="", - base_url="https://example.com", - json_data=payload, - ) - - assert len(items) == 2 - assert items[0]["link"] == "https://example.com/video/1" - assert items[0]["title"] == "First" - assert items[1]["link"] == "https://example.com/video/2" - assert items[1]["title"] == "Second" diff --git a/app/tests/test_itemdto.py b/app/tests/test_itemdto.py index e6fa577b..305402bf 100644 --- a/app/tests/test_itemdto.py +++ b/app/tests/test_itemdto.py @@ -8,7 +8,7 @@ from app.library.ItemDTO import Item, ItemDTO class TestItemFormatAndBasics: - @patch("app.library.Presets.Presets.get_instance") + @patch("app.features.presets.service.Presets.get_instance") def test_format_validates_and_normalizes(self, mock_presets_get): # Preset exists and is not default => allowed mock_presets_get.return_value.has.return_value = True @@ -42,7 +42,7 @@ class TestItemFormatAndBasics: assert item.requeued is True assert item.cli == "--embed-metadata" - @patch("app.library.Presets.Presets.get_instance") + @patch("app.features.presets.service.Presets.get_instance") def test_format_raises_for_missing_url_and_invalid_preset(self, mock_presets_get): # Missing url with pytest.raises(ValueError, match="url param is required"): @@ -282,14 +282,14 @@ class TestItemDTO: def test_get_preset_returns_preset_instance(self): """Test ItemDTO.get_preset returns the Preset instance.""" - from app.library.Presets import Preset + from app.features.presets.schemas import Preset - mock_preset = Preset(id="test-id", name="test-preset", cli="--test") + mock_preset = Preset(id=1, name="test-preset", cli="--format best") with patch.object(ItemDTO, "__post_init__", lambda _: None): dto = ItemDTO(id="vid", title="t", url="u", folder="f", preset="test-preset") - with patch("app.library.Presets.Presets.get_instance") as mock_presets: + with patch("app.features.presets.service.Presets.get_instance") as mock_presets: mock_presets.return_value.get.return_value = mock_preset result = dto.get_preset() @@ -300,14 +300,14 @@ class TestItemDTO: def test_get_preset_uses_default_when_no_preset_set(self): """Test ItemDTO.get_preset uses 'default' when preset is empty.""" - from app.library.Presets import Preset + from app.features.presets.schemas import Preset - mock_preset = Preset(id="default-id", name="default", cli="--default") + mock_preset = Preset(id=2, name="default", cli="--format best") with patch.object(ItemDTO, "__post_init__", lambda _: None): dto = ItemDTO(id="vid", title="t", url="u", folder="f", preset="") - with patch("app.library.Presets.Presets.get_instance") as mock_presets: + with patch("app.features.presets.service.Presets.get_instance") as mock_presets: mock_presets.return_value.get.return_value = mock_preset result = dto.get_preset() @@ -320,7 +320,7 @@ class TestItemDTO: with patch.object(ItemDTO, "__post_init__", lambda _: None): dto = ItemDTO(id="vid", title="t", url="u", folder="f", preset="nonexistent") - with patch("app.library.Presets.Presets.get_instance") as mock_presets: + with patch("app.features.presets.service.Presets.get_instance") as mock_presets: mock_presets.return_value.get.return_value = None result = dto.get_preset() diff --git a/app/tests/test_notifications.py b/app/tests/test_notifications.py deleted file mode 100644 index 4fd912f9..00000000 --- a/app/tests/test_notifications.py +++ /dev/null @@ -1,921 +0,0 @@ -import json -import tempfile -import uuid -from pathlib import Path -from unittest.mock import AsyncMock, Mock, patch - -import httpx -import pytest -from aiohttp import web - -from app.library.BackgroundWorker import BackgroundWorker -from app.library.encoder import Encoder -from app.library.Events import Event -from app.library.ItemDTO import ItemDTO -from app.library.Notifications import ( - Notification, - NotificationEvents, - Target, - TargetRequest, - TargetRequestHeader, -) - - -class TestTargetRequestHeader: - """Test the TargetRequestHeader dataclass.""" - - def test_target_request_header_creation(self): - """Test creating a TargetRequestHeader.""" - header = TargetRequestHeader(key="Authorization", value="Bearer token123") - - assert header.key == "Authorization" - assert header.value == "Bearer token123" - - def test_target_request_header_serialize(self): - """Test serializing a TargetRequestHeader.""" - header = TargetRequestHeader(key="Content-Type", value="application/json") - serialized = header.serialize() - - expected = {"key": "Content-Type", "value": "application/json"} - assert serialized == expected - - def test_target_request_header_json(self): - """Test JSON serialization of TargetRequestHeader.""" - header = TargetRequestHeader(key="X-API-Key", value="secret123") - json_str = header.json() - - # Should be valid JSON - parsed = json.loads(json_str) - assert parsed["key"] == "X-API-Key" - assert parsed["value"] == "secret123" - - def test_target_request_header_get(self): - """Test get method of TargetRequestHeader.""" - header = TargetRequestHeader(key="Authorization", value="Bearer token") - - assert header.get("key") == "Authorization" - assert header.get("value") == "Bearer token" - assert header.get("nonexistent", "default") == "default" - - -class TestTargetRequest: - """Test the TargetRequest dataclass.""" - - def test_target_request_creation_defaults(self): - """Test creating a TargetRequest with defaults.""" - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - - assert request.type == "json" - assert request.method == "POST" - assert request.url == "https://example.com/webhook" - assert request.headers == [] - assert request.data_key == "data" - - def test_target_request_creation_with_headers(self): - """Test creating a TargetRequest with headers.""" - headers = [ - TargetRequestHeader(key="Authorization", value="Bearer token"), - TargetRequestHeader(key="Content-Type", value="application/json"), - ] - request = TargetRequest( - type="json", method="POST", url="https://example.com/webhook", headers=headers, data_key="payload" - ) - - assert len(request.headers) == 2 - assert request.headers[0].key == "Authorization" - assert request.data_key == "payload" - - def test_target_request_serialize(self): - """Test serializing a TargetRequest.""" - headers = [TargetRequestHeader(key="X-Token", value="abc123")] - request = TargetRequest( - type="json", method="PUT", url="https://api.example.com/notify", headers=headers, data_key="content" - ) - - serialized = request.serialize() - expected = { - "type": "json", - "method": "PUT", - "url": "https://api.example.com/notify", - "data_key": "content", - "headers": [{"key": "X-Token", "value": "abc123"}], - } - assert serialized == expected - - def test_target_request_json(self): - """Test JSON serialization of TargetRequest.""" - request = TargetRequest(type="form", method="POST", url="https://webhook.site/test") - json_str = request.json() - - parsed = json.loads(json_str) - assert parsed["type"] == "form" - assert parsed["method"] == "POST" - assert parsed["url"] == "https://webhook.site/test" - - def test_target_request_get(self): - """Test get method of TargetRequest.""" - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - - assert request.get("type") == "json" - assert request.get("method") == "POST" - assert request.get("nonexistent", "default") == "default" - - -class TestTarget: - """Test the Target dataclass.""" - - def test_target_creation_minimal(self): - """Test creating a Target with minimal required fields.""" - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target(id=str(uuid.uuid4()), name="Test Webhook", request=request) - - assert target.name == "Test Webhook" - assert target.on == [] - assert target.presets == [] - assert target.enabled is True - assert target.request == request - - def test_target_creation_full(self): - """Test creating a Target with all fields.""" - target_id = str(uuid.uuid4()) - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target( - id=target_id, - name="Full Test Webhook", - on=["item_completed", "item_failed"], - presets=["default", "audio_only"], - enabled=False, - request=request, - ) - - assert target.id == target_id - assert target.name == "Full Test Webhook" - assert target.on == ["item_completed", "item_failed"] - assert target.presets == ["default", "audio_only"] - assert target.enabled is False - - def test_target_serialize(self): - """Test serializing a Target.""" - target_id = str(uuid.uuid4()) - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target(id=target_id, name="Test Target", on=["item_completed"], presets=["default"], request=request) - - serialized = target.serialize() - assert serialized["id"] == target_id - assert serialized["name"] == "Test Target" - assert serialized["on"] == ["item_completed"] - assert serialized["presets"] == ["default"] - assert isinstance(serialized["request"], dict) - - def test_target_json(self): - """Test JSON serialization of Target.""" - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target(id=str(uuid.uuid4()), name="JSON Test", request=request) - - json_str = target.json() - parsed = json.loads(json_str) - assert parsed["name"] == "JSON Test" - - def test_target_get(self): - """Test get method of Target.""" - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target(id=str(uuid.uuid4()), name="Get Test", request=request) - - assert target.get("name") == "Get Test" - assert target.get("nonexistent", "default") == "default" - - -class TestNotificationEvents: - """Test the NotificationEvents class.""" - - def test_notification_events_constants(self): - """Test that NotificationEvents has expected constants.""" - assert hasattr(NotificationEvents, "TEST") - assert hasattr(NotificationEvents, "ITEM_ADDED") - assert hasattr(NotificationEvents, "ITEM_COMPLETED") - assert hasattr(NotificationEvents, "ITEM_CANCELLED") - assert hasattr(NotificationEvents, "ITEM_DELETED") - assert hasattr(NotificationEvents, "LOG_INFO") - assert hasattr(NotificationEvents, "LOG_ERROR") - - def test_get_events(self): - """Test get_events static method.""" - events = NotificationEvents.get_events() - - assert isinstance(events, dict) - assert "TEST" in events - assert "ITEM_ADDED" in events - assert "ITEM_COMPLETED" in events - - def test_events_function(self): - """Test events function.""" - events_list = NotificationEvents.events() - - assert isinstance(events_list, list) - assert len(events_list) > 0 - - def test_is_valid(self): - """Test is_valid static method.""" - assert NotificationEvents.is_valid("test"), "Valid events" - assert NotificationEvents.is_valid("item_added") - assert NotificationEvents.is_valid("item_completed") - - assert not NotificationEvents.is_valid("invalid_event"), "Invalid events" - assert not NotificationEvents.is_valid("") - assert not NotificationEvents.is_valid(None) - - -class TestNotification: - """Test the Notification singleton class.""" - - def setup_method(self): - """Set up test environment before each test.""" - # Reset singleton instance to ensure clean state - Notification._reset_singleton() - - def teardown_method(self): - """Clean up after each test.""" - # Reset singleton instance to prevent test pollution - Notification._reset_singleton() - - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - def test_notification_singleton(self, mock_background_worker, mock_config): - """Test that Notification follows singleton pattern.""" - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_background_worker.get_instance.return_value = Mock() - - instance1 = Notification.get_instance() - instance2 = Notification.get_instance() - - assert instance1 is instance2 - assert isinstance(instance1, Notification) - - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - def test_notification_init_default_params(self, mock_background_worker, mock_config): - """Test Notification initialization with default parameters.""" - mock_config_instance = Mock(debug=False, config_path="/tmp/test", app_version="1.0.0") - mock_config.get_instance.return_value = mock_config_instance - mock_background_worker.get_instance.return_value = Mock() - - with patch("pathlib.Path.exists", return_value=False): - notification = Notification.get_instance() - - assert notification._debug is False - assert notification._version == "1.0.0" - assert isinstance(notification._targets, list) - assert len(notification._targets) == 0 - - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - def test_notification_init_custom_params(self, mock_background_worker, mock_config): - """Test Notification initialization with custom parameters.""" - _ = mock_background_worker, mock_config # Suppress unused variable warnings - mock_config_instance = Mock(debug=True, config_path="/custom/path", app_version="2.0.0") - mock_client = Mock(spec=httpx.AsyncClient) - mock_encoder = Mock(spec=Encoder) - mock_worker = Mock(spec=BackgroundWorker) - - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_path = temp_file.name - - notification = Notification.get_instance( - file=temp_path, - client=mock_client, - encoder=mock_encoder, - config=mock_config_instance, - background_worker=mock_worker, - ) - - assert notification._debug is True - assert notification._version == "2.0.0" - assert notification._client == mock_client - assert notification._encoder == mock_encoder - assert notification._offload == mock_worker - - def test_get_targets_empty(self): - """Test get_targets when no targets are loaded.""" - with ( - patch("app.library.Notifications.Config") as mock_config, - patch("app.library.Notifications.BackgroundWorker") as mock_worker, - ): - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - notification = Notification.get_instance() - targets = notification.get_targets() - - assert isinstance(targets, list) - assert len(targets) == 0 - - def test_clear_targets(self): - """Test clearing notification targets.""" - with ( - patch("app.library.Notifications.Config") as mock_config, - patch("app.library.Notifications.BackgroundWorker") as mock_worker, - ): - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - notification = Notification.get_instance() - - # Add a dummy target - notification._targets = ["dummy_target"] - assert len(notification._targets) == 1 - - # Clear targets - result = notification.clear() - - assert result == notification # Should return self - assert len(notification._targets) == 0 - - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - def test_save_targets(self, mock_worker, mock_config): - """Test saving targets to file.""" - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: - temp_path = temp_file.name - - # Create a test target - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target(id=str(uuid.uuid4()), name="Test Target", request=request) - - notification = Notification.get_instance(file=temp_path) - result = notification.save([target]) - - assert result == notification # Should return self - - # Verify file was written - with open(temp_path) as f: - saved_data = json.load(f) - - assert isinstance(saved_data, list) - assert len(saved_data) == 1 - assert saved_data[0]["name"] == "Test Target" - - # Clean up - Path(temp_path).unlink() - - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - def test_load_targets_empty_file(self, mock_worker, mock_config): - """Test loading targets from empty file.""" - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: - temp_file.write("") # Empty file - temp_path = temp_file.name - - notification = Notification.get_instance(file=temp_path) - result = notification.load() - - assert result == notification # Should return self - assert len(notification._targets) == 0 - - # Clean up - Path(temp_path).unlink() - - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - @patch("app.library.Notifications.Presets") - def test_load_targets_schema_update(self, mock_presets, mock_worker, mock_config): - """Test that load automatically updates file schema when enabled field is missing.""" - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - mock_preset = Mock() - mock_preset.name = "default" - mock_presets.get_instance.return_value.get_all.return_value = [mock_preset] - - # Create target data without enabled field (old schema) - target_data = [ - { - "id": str(uuid.uuid4()), - "name": "Old Schema Target", - "on": ["item_completed"], - "presets": ["default"], - "request": {"type": "json", "method": "POST", "url": "https://example.com/webhook"}, - } - ] - - with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: - json.dump(target_data, temp_file) - temp_path = temp_file.name - - notification = Notification.get_instance(file=temp_path) - - # Mock the save method to verify it's called - with patch.object(notification, "save") as mock_save: - notification.load() - - # Verify save was called due to schema update - mock_save.assert_called_once() - - assert len(notification._targets) == 1, "Verify the target has enabled=True by default" - assert notification._targets[0].enabled is True - - # Clean up - Path(temp_path).unlink() - - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - @patch("app.library.Notifications.Presets") - def test_load_targets_valid_file(self, mock_presets, mock_worker, mock_config): - """Test loading targets from valid file.""" - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - # Mock preset with name attribute - mock_preset = Mock() - mock_preset.name = "default" - mock_presets.get_instance.return_value.get_all.return_value = [mock_preset] - - target_data = [ - { - "id": str(uuid.uuid4()), - "name": "Test Webhook", - "on": ["item_completed"], - "presets": ["default"], - "request": { - "type": "json", - "method": "POST", - "url": "https://example.com/webhook", - "data_key": "data", - "headers": [], - }, - } - ] - - with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: - json.dump(target_data, temp_file) - temp_path = temp_file.name - - notification = Notification.get_instance(file=temp_path) - result = notification.load() - - assert result == notification # Should return self - assert len(notification._targets) == 1 - assert notification._targets[0].name == "Test Webhook" - - # Clean up - Path(temp_path).unlink() - - def test_make_target(self): - """Test make_target method.""" - with ( - patch("app.library.Notifications.Config") as mock_config, - patch("app.library.Notifications.BackgroundWorker") as mock_worker, - ): - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - notification = Notification.get_instance() - - target_dict = { - "id": str(uuid.uuid4()), - "name": "Test Target", - "on": ["item_completed"], - "presets": ["default"], - "request": { - "type": "json", - "method": "POST", - "url": "https://example.com/webhook", - "data_key": "payload", - "headers": [{"key": "Authorization", "value": "Bearer token"}], - }, - } - - target = notification.make_target(target_dict) - - assert isinstance(target, Target) - assert target.name == "Test Target" - assert target.on == ["item_completed"] - assert target.presets == ["default"] - assert target.request.url == "https://example.com/webhook" - assert target.request.data_key == "payload" - assert len(target.request.headers) == 1 - assert target.request.headers[0].key == "Authorization" - - def test_validate_target_valid(self): - """Test validate method with valid target.""" - target_dict = { - "id": str(uuid.uuid4()), - "name": "Valid Target", - "request": {"url": "https://example.com/webhook"}, - } - - with ( - patch("app.library.Notifications.NotificationEvents.get_events") as mock_events, - patch("app.library.Notifications.Presets") as mock_presets, - ): - mock_events.return_value.values.return_value = ["item_completed"] - mock_presets.get_instance.return_value.get_all.return_value = [] - - result = Notification.validate(target_dict) - assert result is True - - def test_validate_target_missing_id(self): - """Test validate method with missing ID.""" - target_dict = {"name": "Missing ID Target", "request": {"url": "https://example.com/webhook"}} - - with pytest.raises(ValueError, match=r"Invalid notification target\. No ID found\."): - Notification.validate(target_dict) - - def test_validate_target_invalid_id(self): - """Test validate method with invalid UUID.""" - target_dict = { - "id": "invalid-uuid", - "name": "Invalid ID Target", - "request": {"url": "https://example.com/webhook"}, - } - - with pytest.raises(ValueError, match=r"Invalid notification target\. No ID found\."): - Notification.validate(target_dict) - - def test_validate_target_missing_name(self): - """Test validate method with missing name.""" - target_dict = {"id": str(uuid.uuid4()), "request": {"url": "https://example.com/webhook"}} - - with pytest.raises(ValueError, match=r"Invalid notification target\. No name found\."): - Notification.validate(target_dict) - - def test_validate_target_missing_request(self): - """Test validate method with missing request.""" - target_dict = {"id": str(uuid.uuid4()), "name": "Missing Request Target"} - - with pytest.raises(ValueError, match=r"Invalid notification target\. No request details found\."): - Notification.validate(target_dict) - - def test_validate_target_missing_url(self): - """Test validate method with missing URL.""" - target_dict = {"id": str(uuid.uuid4()), "name": "Missing URL Target", "request": {}} - - with pytest.raises(ValueError, match=r"Invalid notification target\. No URL found\."): - Notification.validate(target_dict) - - def test_validate_target_invalid_method(self): - """Test validate method with invalid HTTP method.""" - target_dict = { - "id": str(uuid.uuid4()), - "name": "Invalid Method Target", - "request": { - "url": "https://example.com/webhook", - "method": "GET", # Only POST and PUT are allowed - }, - } - - with pytest.raises(ValueError, match=r"Invalid notification target\. Invalid method found\."): - Notification.validate(target_dict) - - def test_validate_target_invalid_type(self): - """Test validate method with invalid request type.""" - target_dict = { - "id": str(uuid.uuid4()), - "name": "Invalid Type Target", - "request": { - "url": "https://example.com/webhook", - "type": "xml", # Only json and form are allowed - }, - } - - with pytest.raises(ValueError, match=r"Invalid notification target\. Invalid type found\."): - Notification.validate(target_dict) - - def test_validate_target_invalid_enabled(self): - """Test validating a target with invalid enabled field.""" - target_dict = { - "id": str(uuid.uuid4()), - "name": "Test Target", - "enabled": "yes", # Should be boolean - "request": { - "url": "https://example.com/webhook", - "method": "POST", - "type": "json", - }, - } - - with pytest.raises(ValueError, match=r"Enabled must be a boolean\."): - Notification.validate(target_dict) - - def test_validate_target_enabled_defaults_to_true(self): - """Test that enabled defaults to True when not provided.""" - target_dict = { - "id": str(uuid.uuid4()), - "name": "Test Target", - "request": { - "url": "https://example.com/webhook", - "method": "POST", - "type": "json", - }, - } - - result = Notification.validate(target_dict) - assert result is True - assert target_dict.get("enabled") is True - - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - @patch("app.library.Notifications.EventBus") - def test_attach(self, mock_eventbus, mock_worker, mock_config): - """Test attach method.""" - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - - notification = Notification.get_instance() - app = Mock(spec=web.Application) - - with patch.object(notification, "load") as mock_load: - notification.attach(app) - - mock_load.assert_called_once() - mock_eventbus_instance.subscribe.assert_called_once() - - @pytest.mark.asyncio - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - async def test_send_no_targets(self, mock_worker, mock_config): - """Test send method with no targets.""" - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - notification = Notification.get_instance() - event = Event(event="test", data={"test": "data"}) - - result = await notification.send(event) - assert result == [] - - @pytest.mark.asyncio - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - async def test_send_invalid_event_data(self, mock_worker, mock_config): - """Test send method with invalid event data.""" - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - notification = Notification.get_instance() - # Add a target - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target(id=str(uuid.uuid4()), name="Test Target", request=request) - notification._targets = [target] - - event = Event( - event="test", - data="invalid_string_data", # Should be ItemDTO or dict - ) - - result = await notification.send(event) - assert result == [] - - @pytest.mark.asyncio - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - async def test_send_with_http_target(self, mock_worker, mock_config): - """Test send method with HTTP target.""" - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - # Mock HTTP client - mock_response = Mock() - mock_response.status_code = 200 - mock_response.text = "OK" - mock_client = AsyncMock() - mock_client.request = AsyncMock(return_value=mock_response) - - notification = Notification.get_instance(client=mock_client) - - # Add HTTP target - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target(id=str(uuid.uuid4()), name="Test HTTP Target", request=request) - notification._targets = [target] - - # Create test event - item_dto = ItemDTO( - id="test_id", url="https://youtube.com/watch?v=test", title="Test Video", folder="/downloads" - ) - event = Event(event="item_completed", data=item_dto) - - result = await notification.send(event) - - assert len(result) == 1 - assert result[0]["status"] == 200 - assert result[0]["text"] == "OK" - mock_client.request.assert_called_once() - - @pytest.mark.asyncio - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - async def test_send_skips_disabled_targets(self, mock_worker, mock_config): - """Test that send method skips disabled targets.""" - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - # Mock HTTP client - mock_response = Mock() - mock_response.status_code = 200 - mock_response.text = "OK" - mock_client = AsyncMock() - mock_client.request = AsyncMock(return_value=mock_response) - - notification = Notification.get_instance(client=mock_client) - - # Add enabled and disabled targets - enabled_request = TargetRequest(type="json", method="POST", url="https://example.com/enabled") - enabled_target = Target(id=str(uuid.uuid4()), name="Enabled Target", enabled=True, request=enabled_request) - - disabled_request = TargetRequest(type="json", method="POST", url="https://example.com/disabled") - disabled_target = Target(id=str(uuid.uuid4()), name="Disabled Target", enabled=False, request=disabled_request) - - notification._targets = [enabled_target, disabled_target] - - # Create test event - item_dto = ItemDTO( - id="test_id", url="https://youtube.com/watch?v=test", title="Test Video", folder="/downloads" - ) - event = Event(event="item_completed", data=item_dto) - - result = await notification.send(event) - - assert len(result) == 1, "Only enabled target should be called" - assert result[0]["status"] == 200 - mock_client.request.assert_called_once() - - @pytest.mark.asyncio - @patch("app.library.Notifications.Config") - @patch("app.library.Notifications.BackgroundWorker") - async def test_send_with_apprise_target(self, mock_worker, mock_config): - """Test send method with Apprise target.""" - mock_config.get_instance.return_value = Mock( - debug=False, config_path="/tmp", app_version="1.0.0", apprise_config="/tmp/apprise.conf" - ) - mock_worker.get_instance.return_value = Mock() - - notification = Notification.get_instance() - - # Add Apprise target (non-HTTP URL) - request = TargetRequest(type="json", method="POST", url="discord://webhook_id/webhook_token") - target = Target(id=str(uuid.uuid4()), name="Test Discord Target", request=request) - notification._targets = [target] - - # Create test event - event = Event(event="item_completed", data={"test": "data"}) - - with patch("app.library.Notifications.Path") as mock_path, patch("builtins.__import__") as mock_import: - # Mock apprise config file doesn't exist - mock_path.return_value.exists.return_value = False - - # Mock apprise import - mock_apprise = Mock() - mock_notify = Mock() - # Mock async_notify as an AsyncMock that returns True - mock_notify.async_notify = AsyncMock(return_value=True) - mock_apprise.Apprise.return_value = mock_notify - mock_import.return_value = mock_apprise - - result = await notification.send(event) - - assert len(result) == 1, "Should return empty dict from _apprise method" - assert result[0] == {} - - def test_check_preset_no_presets(self): - """Test _check_preset method with target having no preset filters.""" - with ( - patch("app.library.Notifications.Config") as mock_config, - patch("app.library.Notifications.BackgroundWorker") as mock_worker, - ): - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - notification = Notification.get_instance() - - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target( - id=str(uuid.uuid4()), - name="No Preset Filter", - presets=[], # No preset filter - request=request, - ) - - event = Event(event="item_completed", data={"preset": "default"}) - - result = notification._check_preset(target, event) - assert result is True # Should pass when no preset filter - - def test_check_preset_with_matching_preset(self): - """Test _check_preset method with matching preset.""" - with ( - patch("app.library.Notifications.Config") as mock_config, - patch("app.library.Notifications.BackgroundWorker") as mock_worker, - ): - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - notification = Notification.get_instance() - - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target( - id=str(uuid.uuid4()), name="Preset Filter", presets=["default", "audio_only"], request=request - ) - - # Test with ItemDTO - item_dto = ItemDTO( - id="test_id", url="https://youtube.com/test", title="Test Video", folder="/downloads", preset="default" - ) - event = Event(event="item_completed", data=item_dto) - - result = notification._check_preset(target, event) - assert result is True - - def test_check_preset_with_non_matching_preset(self): - """Test _check_preset method with non-matching preset.""" - with ( - patch("app.library.Notifications.Config") as mock_config, - patch("app.library.Notifications.BackgroundWorker") as mock_worker, - ): - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - notification = Notification.get_instance() - - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target(id=str(uuid.uuid4()), name="Preset Filter", presets=["audio_only"], request=request) - - event = Event( - event="item_completed", - data={"preset": "video_only"}, # Different preset - ) - - result = notification._check_preset(target, event) - assert result is False - - def test_emit_invalid_event(self): - """Test emit method with invalid event.""" - with ( - patch("app.library.Notifications.Config") as mock_config, - patch("app.library.Notifications.BackgroundWorker") as mock_worker, - ): - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker_instance = Mock() - mock_worker.get_instance.return_value = mock_worker_instance - - notification = Notification.get_instance() - - # Add a target - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target(id=str(uuid.uuid4()), name="Test Target", request=request) - notification._targets = [target] - - # Create invalid event - event = Event(event="invalid_event", data={"test": "data"}) - - with patch("app.library.Notifications.NotificationEvents.is_valid", return_value=False): - result = notification.emit(event, None) - - assert result is None, "Should return None and not submit to background worker" - mock_worker_instance.submit.assert_not_called() - - def test_emit_valid_event(self): - """Test emit method with valid event.""" - with ( - patch("app.library.Notifications.Config") as mock_config, - patch("app.library.Notifications.BackgroundWorker") as mock_worker, - ): - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker_instance = Mock() - mock_worker.get_instance.return_value = mock_worker_instance - - notification = Notification.get_instance() - - # Add a target - request = TargetRequest(type="json", method="POST", url="https://example.com/webhook") - target = Target(id=str(uuid.uuid4()), name="Test Target", request=request) - notification._targets = [target] - - # Create valid event - event = Event(event="item_completed", data={"test": "data"}) - - with patch("app.library.Notifications.NotificationEvents.is_valid", return_value=True): - result = notification.emit(event, None) - - assert result is None, "Should return None but submit to background worker" - mock_worker_instance.submit.assert_called_once_with(notification.send, event) - - @pytest.mark.asyncio - async def test_noop(self): - """Test noop method.""" - with ( - patch("app.library.Notifications.Config") as mock_config, - patch("app.library.Notifications.BackgroundWorker") as mock_worker, - ): - mock_config.get_instance.return_value = Mock(debug=False, config_path="/tmp", app_version="1.0.0") - mock_worker.get_instance.return_value = Mock() - - notification = Notification.get_instance() - result = await notification.noop() - - assert result is None diff --git a/app/tests/test_presets.py b/app/tests/test_presets.py deleted file mode 100644 index 65395ff5..00000000 --- a/app/tests/test_presets.py +++ /dev/null @@ -1,692 +0,0 @@ -import json -import tempfile -import uuid -from pathlib import Path -from unittest.mock import Mock, patch - -import pytest - -from app.library.Presets import DEFAULT_PRESETS, Preset, Presets - - -class TestPreset: - """Test the Preset dataclass.""" - - def test_preset_creation_with_defaults(self): - """Test creating a preset with default values.""" - preset = Preset(name="test_preset") - - assert preset.name == "test_preset" - assert preset.description == "" - assert preset.folder == "" - assert preset.template == "" - assert preset.cookies == "" - assert preset.cli == "" - assert preset.default is False - assert len(preset.id) == 36, "ID should be auto-generated UUID" # UUID4 string length - assert "-" in preset.id - - def test_preset_creation_with_all_fields(self): - """Test creating a preset with all fields specified.""" - preset_id = str(uuid.uuid4()) - preset = Preset( - id=preset_id, - name="test_preset", - description="Test description", - folder="test_folder", - template="test_template", - cookies="test_cookies", - cli="--test-option", - default=True, - priority=10, - ) - - assert preset.id == preset_id - assert preset.name == "test_preset" - assert preset.description == "Test description" - assert preset.folder == "test_folder" - assert preset.template == "test_template" - assert preset.cookies == "test_cookies" - assert preset.cli == "--test-option" - assert preset.priority == 10 - assert preset.default is True - - def test_preset_serialize(self): - """Test preset serialization to dictionary.""" - preset = Preset(name="test_preset", description="Test description", cli="--test-option") - - serialized = preset.serialize() - - assert isinstance(serialized, dict) - assert serialized["name"] == "test_preset" - assert serialized["description"] == "Test description" - assert serialized["cli"] == "--test-option" - assert "id" in serialized - - def test_preset_json(self): - """Test preset JSON serialization.""" - preset = Preset(name="test_preset", cli="--test-option") - - json_str = preset.json() - - assert isinstance(json_str, str) - # Should be valid JSON - parsed = json.loads(json_str) - assert parsed["name"] == "test_preset" - assert parsed["cli"] == "--test-option" - - def test_preset_get_method(self): - """Test preset get method for accessing fields.""" - preset = Preset(name="test_preset", description="Test description") - - assert preset.get("name") == "test_preset" - assert preset.get("description") == "Test description" - assert preset.get("nonexistent") is None - assert preset.get("nonexistent", "default_value") == "default_value" - - def test_preset_id_generation(self): - """Test that each preset gets a unique ID.""" - preset1 = Preset(name="preset1") - preset2 = Preset(name="preset2") - - assert preset1.id != preset2.id - assert len(preset1.id) == 36 - assert len(preset2.id) == 36 - - -class TestPresets: - """Test the Presets singleton manager.""" - - def setup_method(self): - """Set up test environment before each test.""" - # Reset singleton completely - Presets._reset_singleton() - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_singleton(self, mock_eventbus, mock_config): - """Test that Presets follows singleton pattern.""" - mock_config.get_instance.return_value = Mock(config_path="/tmp", default_preset="default") - mock_eventbus.get_instance.return_value = Mock() - - instance1 = Presets.get_instance() - instance2 = Presets.get_instance() - - assert instance1 is instance2 - assert isinstance(instance1, Presets) - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_initialization(self, mock_eventbus, mock_config): - """Test Presets initialization with default presets.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - # Should have default presets loaded - default_presets = presets._default - assert len(default_presets) > 0 - - # Check that default presets are Preset instances - for preset in default_presets: - assert isinstance(preset, Preset) - assert preset.default is True - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_load_empty_file(self, mock_eventbus, mock_config): - """Test loading presets from empty or non-existent file.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - presets.clear() # Ensure we start with empty items - - result = presets.load() - - assert result is presets - assert len(presets._items) == 0 - assert len(presets._default) > 0 # Should still have default presets - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - @patch("app.library.Presets.arg_converter") - def test_presets_load_valid_file(self, mock_arg_converter, mock_eventbus, mock_config): - """Test loading presets from valid JSON file.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_arg_converter.return_value = [] # Mock successful CLI parsing - - test_presets = [ - {"id": str(uuid.uuid4()), "name": "test_preset_1", "description": "Test preset 1", "cli": "--format best"}, - {"id": str(uuid.uuid4()), "name": "test_preset_2", "description": "Test preset 2", "cli": "--format worst"}, - ] - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets_file.write_text(json.dumps(test_presets, indent=2)) - - presets = Presets(file=presets_file, config=mock_config_instance) - presets._items.clear() # Ensure we start with empty items - result = presets.load() - - assert result is presets - assert len(presets._items) == 2 - assert presets._items[0].name == "test_preset_1" - assert presets._items[1].name == "test_preset_2" - - @patch("app.library.Presets.EventBus") - @patch("app.library.Presets.Config") - def test_presets_load_invalid_json(self, mock_eventbus, mock_config): - """Test loading presets from invalid JSON file.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets_file.write_text("invalid json content") - - presets = Presets(file=presets_file, config=mock_config_instance) - - # Should handle invalid JSON gracefully - result = presets.load() - assert result is presets - assert len(presets._items) == 0 - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - @patch("app.library.Presets.arg_converter") - def test_presets_load_with_format_migration(self, mock_arg_converter, mock_eventbus, mock_config): - """Test loading presets with old format that needs migration.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_arg_converter.return_value = [] # Mock successful CLI parsing - - # Old format preset with 'format' field instead of 'cli' - old_preset = {"name": "old_preset", "format": "best[height<=720]", "description": "Old format preset"} - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets_file.write_text(json.dumps([old_preset])) - - presets = Presets(file=presets_file, config=mock_config_instance) - result = presets.load() - - assert result is presets - assert len(presets._items) == 1 - loaded_preset = presets._items[0] - assert loaded_preset.name == "old_preset" - assert "best[height<=720]" in loaded_preset.cli, "Should have migrated format to cli" - assert "--format" in loaded_preset.cli - # Should have generated ID - assert loaded_preset.id is not None - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_validate_valid_preset(self, mock_eventbus, mock_config): - """Test validating a valid preset.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - valid_preset = {"id": str(uuid.uuid4()), "name": "valid_preset", "cli": "--format best"} - - result = presets.validate(valid_preset) - assert result is True - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_validate_invalid_preset(self, mock_eventbus, mock_config): - """Test validating invalid presets.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - # Test missing ID - with pytest.raises(ValueError, match="No id found"): - presets.validate({"name": "test"}) - - # Test missing name - with pytest.raises(ValueError, match="No name found"): - presets.validate({"id": str(uuid.uuid4())}) - - # Test wrong type - with pytest.raises(ValueError, match="Unexpected"): - presets.validate("invalid_type") - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - @patch("app.library.Presets.arg_converter") - def test_presets_save(self, mock_arg_converter, mock_eventbus, mock_config): - """Test saving presets to file.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_arg_converter.return_value = [] # Mock successful CLI parsing - - test_presets = [ - Preset(name="test1", cli="--format best"), - Preset(name="test2", cli="--format worst", default=True), # This should be excluded from save - ] - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - result = presets.save(test_presets) - - assert result is presets - assert presets_file.exists() - - # Should only save non-default presets - saved_data = json.loads(presets_file.read_text()) - assert len(saved_data) == 1 - assert saved_data[0]["name"] == "test1" - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_get_all(self, mock_eventbus, mock_config): - """Test getting all presets (default + custom).""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - # Add a custom preset - custom_preset = Preset(name="custom", cli="--custom") - presets._items.append(custom_preset) - - all_presets = presets.get_all() - - assert len(all_presets) > 1, "Should include both default and custom presets" - assert any(p.name == "custom" for p in all_presets) - assert any(p.default is True for p in all_presets) - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_get_by_id(self, mock_eventbus, mock_config): - """Test getting preset by ID.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - test_id = str(uuid.uuid4()) - test_preset = Preset(id=test_id, name="test_preset") - presets._items.append(test_preset) - - found_preset = presets.get(test_id) - - assert found_preset is not None - assert found_preset.id == test_id - assert found_preset.name == "test_preset" - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_get_by_name(self, mock_eventbus, mock_config): - """Test getting preset by name.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - test_preset = Preset(name="test_preset", cli="--test") - presets._items.append(test_preset) - - found_preset = presets.get("test_preset") - - assert found_preset is not None - assert found_preset.name == "test_preset" - assert found_preset.cli == "--test" - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_get_nonexistent(self, mock_eventbus, mock_config): - """Test getting non-existent preset returns None.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - found_preset = presets.get("nonexistent") - - assert found_preset is None - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_get_empty_parameter(self, mock_eventbus, mock_config): - """Test getting preset with empty string returns None.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - assert presets.get("") is None - assert presets.get(None) is None - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_has_method(self, mock_eventbus, mock_config): - """Test has method for checking preset existence.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - test_preset = Preset(name="existing_preset") - presets._items.append(test_preset) - - assert presets.has("existing_preset") is True - assert presets.has(test_preset.id) is True - assert presets.has("nonexistent") is False - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_clear(self, mock_eventbus, mock_config): - """Test clearing all custom presets.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - # Add some custom presets - presets._items.append(Preset(name="custom1")) - presets._items.append(Preset(name="custom2")) - - assert len(presets._items) == 2 - - result = presets.clear() - - assert result is presets - assert len(presets._items) == 0 - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_file_permissions(self, mock_eventbus, mock_config): - """Test that presets file gets correct permissions.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - # Create file with different permissions - presets_file.write_text("{}") - presets_file.chmod(0o644) - - # Creating Presets instance should attempt to fix permissions - presets = Presets(file=presets_file, config=mock_config_instance) - - # Note: The actual chmod might fail in test environment, - # but we're testing that the code attempts it - assert presets is not None, "but we're testing that the code attempts it" - - def test_default_presets_validation(self): - """Test that all default presets are valid.""" - # This test verifies that the DEFAULT_PRESETS constant contains valid presets - for i, preset_data in enumerate(DEFAULT_PRESETS): - assert "id" in preset_data, f"Default preset {i} missing id" - assert "name" in preset_data, f"Default preset {i} missing name" - assert "default" in preset_data, f"Default preset {i} missing default flag" - assert preset_data["default"] is True, f"Default preset {i} should have default=True" - - # Should be able to create Preset instance - preset = Preset( - id=preset_data["id"], - name=preset_data["name"], - description=preset_data.get("description", ""), - folder=preset_data.get("folder", ""), - template=preset_data.get("template", ""), - cli=preset_data.get("cli", ""), - default=preset_data["default"], - ) - assert isinstance(preset, Preset) - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_attach_method(self, mock_eventbus, mock_config): - """Test the attach method for aiohttp integration.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_app = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - # Mock the get method to return a preset for default_preset check - with patch.object(presets, "get", return_value=Mock()) as mock_get: - presets.attach(mock_app) - mock_get.assert_called_once_with("default") - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_presets_attach_default_preset_not_found(self, mock_eventbus, mock_config): - """Test attach method when default preset is not found.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="nonexistent") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_app = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - with patch.object(presets, "get", return_value=None): - # The actual config instance stored in presets should be checked - presets.attach(mock_app) - assert presets._config.default_preset == "default", "Should have reset default_preset to 'default'" - - @pytest.mark.asyncio - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - async def test_presets_on_shutdown(self, mock_eventbus, mock_config): - """Test the on_shutdown method.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_app = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - # Should not raise an exception - await presets.on_shutdown(mock_app) - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - @patch("app.library.Presets.LOG") - @patch("app.library.Presets.arg_converter") - def test_presets_load_invalid_preset_in_file(self, mock_arg_converter, mock_log, mock_eventbus, mock_config): - """Test loading file with some invalid presets.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_arg_converter.return_value = [] # Mock successful CLI parsing - - # Mix of valid and invalid presets - test_presets = [ - {"id": str(uuid.uuid4()), "name": "valid_preset", "cli": "--format best"}, - { - # Missing required fields - "description": "Invalid preset" - }, - ] - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets_file.write_text(json.dumps(test_presets)) - - # Clear any existing items first - presets = Presets(file=presets_file, config=mock_config_instance) - presets.load() - - assert len(presets._items) == 1, "Should only load the valid preset" - assert presets._items[0].name == "valid_preset", f"Expected 'valid_preset', got '{presets._items}'" - - # Should have logged an error for the invalid preset - mock_log.error.assert_called() - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - @patch("app.library.Presets.LOG") - def test_presets_save_with_invalid_preset(self, mock_log, mock_eventbus, mock_config): - """Test saving presets with some invalid ones.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - # Create preset with invalid CLI that will fail validation - valid_preset = Preset(name="valid", cli="--valid-option") - - # Create invalid preset data - invalid_preset_data = {"name": "invalid"} # Missing ID - - presets_to_save = [valid_preset, invalid_preset_data] - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - presets.save(presets_to_save) - - # Should have logged errors for invalid presets - mock_log.error.assert_called() - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_preset_priority_validation_integer(self, mock_eventbus, mock_config): - """Test that priority must be an integer.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - # Test with non-integer priority - invalid_preset = {"id": str(uuid.uuid4()), "name": "test", "priority": "not_an_int"} - - with pytest.raises(ValueError, match="Priority must be an integer"): - presets.validate(invalid_preset) - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_preset_priority_validation_negative(self, mock_eventbus, mock_config): - """Test that priority must be >= 0.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - # Test with negative priority - invalid_preset = {"id": str(uuid.uuid4()), "name": "test", "priority": -5} - - with pytest.raises(ValueError, match="Priority must be >= 0"): - presets.validate(invalid_preset) - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_preset_priority_sorting(self, mock_eventbus, mock_config): - """Test that presets are sorted by priority in descending order.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - presets = Presets(file=presets_file, config=mock_config_instance) - - # Create presets with different priorities - preset1 = Preset(name="low", priority=1) - preset2 = Preset(name="high", priority=10) - preset3 = Preset(name="medium", priority=5) - - presets.save([preset1, preset2, preset3]).load() - - all_presets = presets.get_all() - non_default = [p for p in all_presets if not p.default] - - assert non_default[0].name == "high", "Should be sorted by priority descending" - assert non_default[0].priority == 10 - assert non_default[1].name == "medium" - assert non_default[1].priority == 5 - assert non_default[2].name == "low" - assert non_default[2].priority == 1 - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_preset_priority_default_zero(self, mock_eventbus, mock_config): - """Test that priority defaults to 0 if not specified.""" - preset = Preset(name="test") - assert preset.priority == 0 - - @patch("app.library.Presets.Config") - @patch("app.library.Presets.EventBus") - def test_preset_priority_migration(self, mock_eventbus, mock_config): - """Test that old presets without priority get it added on load.""" - mock_config_instance = Mock(config_path="/tmp", default_preset="default") - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - presets_file = Path(temp_dir) / "presets.json" - - # Create a preset file without priority field - old_preset_data = [{"id": str(uuid.uuid4()), "name": "old_preset", "cli": ""}] - presets_file.write_text(json.dumps(old_preset_data)) - - # Load presets - should migrate by adding priority - presets = Presets(file=presets_file, config=mock_config_instance) - presets.load() - - # Check that priority was added - loaded_data = json.loads(presets_file.read_text()) - assert "priority" in loaded_data[0] - assert loaded_data[0]["priority"] == 0 diff --git a/app/tests/test_services.py b/app/tests/test_services.py index be64623a..e151bba6 100644 --- a/app/tests/test_services.py +++ b/app/tests/test_services.py @@ -3,7 +3,7 @@ from unittest.mock import MagicMock, patch import pytest -from app.library.Services import Services +from app.library.Services import ServiceEntry, Services class TestServices: @@ -92,9 +92,9 @@ class TestServices: services.add("test", "value") all_services = services.get_all() - all_services["injected"] = "malicious" + all_services.append(ServiceEntry(name="injected", declared_type=str, instance="malicious")) - assert "injected" not in services.get_all(), "Modifying returned dict should not affect internal state" + assert services.get("injected") is None, "Modifying returned dict should not affect internal state" def test_handle_sync_with_matching_args(self): """Test synchronous handler with matching arguments.""" @@ -138,8 +138,9 @@ class TestServices: # Should still log error about missing arguments mock_logger.error.assert_called_once() error_call = mock_logger.error.call_args[0][0] - assert "Missing arguments" in error_call - assert "missing_service_param" in error_call + assert "Missing arguments for handler" in error_call, ( + f"Expected 'Missing arguments for handler' in log, got: {error_call}" + ) def test_handle_sync_no_args_handler(self): """Test synchronous handler that takes no arguments.""" @@ -197,8 +198,9 @@ class TestServices: # Should still log error about missing arguments mock_logger.error.assert_called_once() error_call = mock_logger.error.call_args[0][0] - assert "Missing arguments" in error_call - assert "missing_service_param" in error_call + assert "Missing arguments for handler" in error_call, ( + f"Expected 'Missing arguments for handler' in log, got: {error_call}" + ) @pytest.mark.asyncio async def test_handle_async_no_args_handler(self): diff --git a/app/tests/test_sqlite_store.py b/app/tests/test_sqlite_store.py index a03d7e15..3a52c3d8 100644 --- a/app/tests/test_sqlite_store.py +++ b/app/tests/test_sqlite_store.py @@ -1,6 +1,7 @@ from datetime import UTC, datetime, timedelta import pytest +from sqlalchemy import text from app.library.ItemDTO import ItemDTO from app.library.operations import Operation @@ -11,8 +12,8 @@ async def make_store() -> SqliteStore: """Create an isolated in-memory SqliteStore instance with schema.""" SqliteStore._reset_singleton() store = SqliteStore.get_instance(db_path=":memory:") - await store._ensure_conn() - assert store._conn is not None + await store.get_connection() + assert store._engine is not None, "Engine should be initialized after _ensure_conn" return store @@ -26,6 +27,53 @@ def make_item(idx: int, *, status: str = "finished") -> ItemDTO: ) +@pytest.mark.asyncio +async def test_sessionmaker_returns_valid_sessionmaker() -> None: + """Test that sessionmaker() returns a working async_sessionmaker.""" + SqliteStore._reset_singleton() + store = SqliteStore.get_instance(db_path=":memory:") + + # Ensure connection is initialized + await store.get_connection() + + # Get sessionmaker + sessionmaker = store.sessionmaker() + assert sessionmaker is not None, "Sessionmaker should not be None after connection initialization" + + # Verify we can create a session + async with sessionmaker() as session: + assert session is not None, "Session should be created successfully" + + await store.close() + + +@pytest.mark.asyncio +async def test_sessionmaker_raises_before_init() -> None: + """Test that sessionmaker() raises error before connection initialization.""" + SqliteStore._reset_singleton() + store = SqliteStore.get_instance(db_path=":memory:") + + with pytest.raises(RuntimeError, match="Database connection not initialized"): + store.sessionmaker() + + await store.close() + + +@pytest.mark.asyncio +async def test_sqlalchemy_engine_disposed_on_close() -> None: + """Test that SQLAlchemy engine is properly disposed on close.""" + SqliteStore._reset_singleton() + store = SqliteStore.get_instance(db_path=":memory:") + + await store.get_connection() + assert store._engine is not None, "Engine should be created" + assert store._sessionmaker is not None, "Sessionmaker should be created" + + await store.close() + assert store._engine is None, "Engine should be None after close" + assert store._sessionmaker is None, "Sessionmaker should be None after close" + + @pytest.mark.asyncio async def test_enqueue_upsert_and_fetch_saved(): store = await make_store() @@ -87,8 +135,10 @@ async def test_paginate_order_and_bounds(): created_at = (base + timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:%S") _list.append(itm) await store._conn.execute( - 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (?, ?, ?, ?, ?)', - (itm._id, "history", itm.url, encoded, created_at), + text( + 'INSERT INTO "history" ("id", "type", "url", "data", "created_at") VALUES (:id, :type, :url, :data, :created_at)' + ), + {"id": itm._id, "type": "history", "url": itm.url, "data": encoded, "created_at": created_at}, ) await store._conn.commit() @@ -224,6 +274,7 @@ async def test_on_shutdown_closes_connection(): await store.on_shutdown(None) assert store._conn is None + # Note: on_shutdown already closes the connection, so no need to call close() again @pytest.mark.asyncio diff --git a/app/tests/test_system_routes.py b/app/tests/test_system_routes.py index 4c9087c8..209ccd7c 100644 --- a/app/tests/test_system_routes.py +++ b/app/tests/test_system_routes.py @@ -25,7 +25,7 @@ class TestCheckUpdatesEndpoint: response = await check_updates(config, encoder, update_checker) - assert 400 == response.status, "Should return 400 when update checking is disabled" + assert 200 == response.status, "Should work even if disabled as it's manual check." body = response.body assert b"disabled" in body.lower(), "Response should mention update checking is disabled" @@ -39,7 +39,7 @@ class TestCheckUpdatesEndpoint: update_checker = UpdateChecker.get_instance() with patch.object(update_checker, "check_for_updates", new_callable=AsyncMock) as mock_check: - mock_check.return_value = ("up_to_date", None) + mock_check.return_value = (("up_to_date", None), ("up_to_date", None)) response = await check_updates(config, encoder, update_checker) assert 200 == response.status, "Should return 200" @@ -57,7 +57,7 @@ class TestCheckUpdatesEndpoint: update_checker = UpdateChecker.get_instance() with patch.object(update_checker, "check_for_updates", new_callable=AsyncMock) as mock_check: - mock_check.return_value = ("up_to_date", None) + mock_check.return_value = (("up_to_date", None), ("up_to_date", None)) response = await check_updates(config, encoder, update_checker) assert 200 == response.status, "Should return 200" @@ -74,7 +74,7 @@ class TestCheckUpdatesEndpoint: update_checker = UpdateChecker.get_instance() with patch.object(update_checker, "check_for_updates", new_callable=AsyncMock) as mock_check: - mock_check.return_value = ("update_available", "v1.0.5") + mock_check.return_value = (("update_available", "v1.0.5"), ("up_to_date", None)) response = await check_updates(config, encoder, update_checker) assert 200 == response.status, "Should return 200" @@ -92,7 +92,7 @@ class TestCheckUpdatesEndpoint: update_checker = UpdateChecker.get_instance() with patch.object(update_checker, "check_for_updates", new_callable=AsyncMock) as mock_check: - mock_check.return_value = ("up_to_date", None) + mock_check.return_value = (("up_to_date", None), ("up_to_date", None)) response = await check_updates(config, encoder, update_checker) assert 200 == response.status, "Should return 200" @@ -109,7 +109,7 @@ class TestCheckUpdatesEndpoint: update_checker = UpdateChecker.get_instance() with patch.object(update_checker, "check_for_updates", new_callable=AsyncMock) as mock_check: - mock_check.return_value = ("error", None) + mock_check.return_value = (("error", None), ("error", None)) response = await check_updates(config, encoder, update_checker) assert 200 == response.status, "Should return 200" diff --git a/app/tests/test_task_definitions.py b/app/tests/test_task_definitions.py deleted file mode 100644 index 28a42bb8..00000000 --- a/app/tests/test_task_definitions.py +++ /dev/null @@ -1,315 +0,0 @@ -import json -import uuid -from pathlib import Path -from typing import Any -from unittest.mock import AsyncMock, MagicMock, Mock, patch - -import pytest -from aiohttp import web -from aiohttp.web import Request -from jsonschema import Draft7Validator - -from app.library.encoder import Encoder -from app.library.TaskDefinitions import TaskDefinitionRecord, TaskDefinitions -from app.routes.api import task_definitions as api - - -def _load_validator() -> Draft7Validator: - schema_path = Path(__file__).resolve().parent.parent / "schema" / "task_definition.json" - schema = json.loads(schema_path.read_text(encoding="utf-8")) - return Draft7Validator(schema) - - -def _sample_definition(name: str = "example", *, priority: int = 0) -> dict[str, Any]: - return { - "name": name, - "match": ["https://example.com/*"], - "priority": priority, - "parse": { - "items": { - "type": "css", - "selector": ".card", - "fields": { - "link": {"type": "css", "expression": "a", "attribute": "href"}, - "title": {"type": "css", "expression": "a", "attribute": "text"}, - }, - } - }, - } - - -class TestTaskDefinitionsManager: - def setup_method(self) -> None: - TaskDefinitions._reset_singleton() - - def teardown_method(self) -> None: - TaskDefinitions._reset_singleton() - - def test_load_populates_records(self, tmp_path: Path) -> None: - validator = _load_validator() - config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) - - first_identifier = "b5c6ad5f-4745-4c05-88c8-dde1deae3b51" - second_identifier = "ae38a6b0-2c22-4763-ba60-801ae8ce1218" - - first = tmp_path / f"{first_identifier}.json" - second = tmp_path / f"{second_identifier}.json" - - first.write_text(json.dumps(_sample_definition("First", priority=5)), encoding="utf-8") - second.write_text(json.dumps(_sample_definition("Second", priority=1)), encoding="utf-8") - - (tmp_path / "not-a-uuid.json").write_text(json.dumps(_sample_definition("Ignored")), encoding="utf-8") - (tmp_path / "0f9184de-5d3c-2111-8c21-6d3f0be1bd3d.json").write_text( - json.dumps(_sample_definition("Ignored2")), - encoding="utf-8", - ) - - manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) - - records = manager.list() - assert len(records) == 2 - assert [record.name for record in records] == ["Second", "First"] - assert [record.priority for record in records] == [1, 5] - - def test_create_writes_file_and_refreshes(self, tmp_path: Path) -> None: - validator = _load_validator() - config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) - manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) - - with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh: - definition = _sample_definition("My Definition") - record = manager.create(definition) - - refresh.assert_called_once_with(force=True) - identifier_uuid = uuid.UUID(record.identifier) - assert 4 == identifier_uuid.version - expected_filename = f"{record.identifier}.json" - saved_path = tmp_path / expected_filename - assert saved_path.exists() - saved_content = json.loads(saved_path.read_text(encoding="utf-8")) - assert saved_content["name"] == "My Definition" - assert saved_content["priority"] == 0 - - def test_update_missing_definition_raises(self, tmp_path: Path) -> None: - validator = _load_validator() - config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) - manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) - - with pytest.raises(ValueError, match="does not exist"): - manager.update("missing", _sample_definition("Updated")) - - def test_update_overwrites_file(self, tmp_path: Path) -> None: - validator = _load_validator() - config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) - - identifier = "c59ec7cf-6291-4f0f-86f8-d8cb12c325a4" - initial = _sample_definition("Original", priority=4) - path = tmp_path / f"{identifier}.json" - path.write_text(json.dumps(initial), encoding="utf-8") - - manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) - - with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh: - updated_record = manager.update(identifier, _sample_definition("Updated", priority=2)) - - refresh.assert_called_once_with(force=True) - assert updated_record.name == "Updated" - assert updated_record.priority == 2 - saved = json.loads(path.read_text(encoding="utf-8")) - assert saved["name"] == "Updated" - assert saved["priority"] == 2 - - def test_delete_removes_file(self, tmp_path: Path) -> None: - validator = _load_validator() - config = Mock(config_path=str(tmp_path), app_path=str(tmp_path)) - - identifier = "f0b71f47-6b65-4b6d-89fd-6b87ce47d3bc" - definition_path = tmp_path / f"{identifier}.json" - definition_path.write_text(json.dumps(_sample_definition("Delete")), encoding="utf-8") - - manager = TaskDefinitions.get_instance(directory=tmp_path, config=config, validator=validator) - - with patch("app.library.task_handlers.generic.GenericTaskHandler.refresh_definitions") as refresh: - manager.delete(identifier) - - refresh.assert_called_once_with(force=True) - assert not definition_path.exists() - assert manager.get(identifier) is None - - -@pytest.mark.asyncio -class TestTaskDefinitionRoutes: - def setup_method(self) -> None: - TaskDefinitions._reset_singleton() - - def teardown_method(self) -> None: - TaskDefinitions._reset_singleton() - - async def test_list_definitions(self) -> None: - request = MagicMock(spec=Request) - request.query = {} - - identifier = "9af7018f-8659-4d2a-a42b-b5d2c5f0a6e2" - record = TaskDefinitionRecord( - identifier=identifier, - filename=f"{identifier}.json", - name="Sample", - priority=0, - path=Path("/tmp/sample.json"), - data=_sample_definition("Sample"), - updated_at=123.0, - ) - - task_definitions = MagicMock(spec=TaskDefinitions) - task_definitions.list.return_value = [record] - - response = await api.task_definitions_list(request, Encoder(), task_definitions) - payload = json.loads(response.text) - - assert response.status == web.HTTPOk.status_code - assert payload == [record.serialize()] - assert "filename" not in payload[0] - - async def test_list_definitions_includes_definition(self) -> None: - request = MagicMock(spec=Request) - request.query = {"include": "definition"} - - identifier = "f5b5e88d-5c6b-4a27-8453-1b6a4fb8a8d1" - record = TaskDefinitionRecord( - identifier=identifier, - filename=f"{identifier}.json", - name="Sample", - priority=0, - path=Path("/tmp/sample.json"), - data=_sample_definition("Sample"), - updated_at=123.0, - ) - - task_definitions = MagicMock(spec=TaskDefinitions) - task_definitions.list.return_value = [record] - - response = await api.task_definitions_list(request, Encoder(), task_definitions) - payload = json.loads(response.text) - - assert payload[0]["definition"]["name"] == "Sample" - - async def test_get_definition_not_found(self) -> None: - request = MagicMock(spec=Request) - request.match_info = {"identifier": "unknown"} - - task_definitions = MagicMock(spec=TaskDefinitions) - task_definitions.get.return_value = None - - response = await api.task_definitions_get(request, Encoder(), task_definitions) - payload = json.loads(response.text) - - assert response.status == web.HTTPNotFound.status_code - assert "error" in payload - - async def test_create_definition_success(self) -> None: - payload_definition = _sample_definition("New", priority=1) - payload = {"definition": payload_definition} - - request = MagicMock(spec=Request) - request.json = AsyncMock(return_value=payload) - - identifier = "4f08b8af-b87a-4d6e-9289-39e5172898aa" - record = TaskDefinitionRecord( - identifier=identifier, - filename=f"{identifier}.json", - name="New", - priority=1, - path=Path("/tmp/new.json"), - data=payload["definition"], - updated_at=123.0, - ) - - task_definitions = MagicMock(spec=TaskDefinitions) - task_definitions.create.return_value = record - - response = await api.task_definitions_create(request, Encoder(), task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPCreated.status_code - assert body["id"] == identifier - assert body["priority"] == 1 - assert "filename" not in body - task_definitions.create.assert_called_once_with(payload_definition) - - async def test_create_definition_invalid_payload(self) -> None: - request = MagicMock(spec=Request) - request.json = AsyncMock(return_value=[]) # type: ignore[arg-type] - - task_definitions = MagicMock(spec=TaskDefinitions) - - response = await api.task_definitions_create(request, Encoder(), task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPBadRequest.status_code - assert "error" in body - - async def test_update_definition_success(self) -> None: - request = MagicMock(spec=Request) - identifier = "6d8d5719-95ae-4478-bb05-986f5b72b6c1" - request.match_info = {"identifier": identifier} - definition = _sample_definition("Updated", priority=4) - request.json = AsyncMock(return_value={"definition": definition}) - record = TaskDefinitionRecord( - identifier=identifier, - filename=f"{identifier}.json", - name="Updated", - priority=4, - path=Path("/tmp/existing.json"), - data=definition, - updated_at=456.0, - ) - - task_definitions = MagicMock(spec=TaskDefinitions) - task_definitions.update.return_value = record - - response = await api.task_definitions_update(request, Encoder(), task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPOk.status_code - assert body["name"] == "Updated" - assert body["priority"] == 4 - task_definitions.update.assert_called_once_with(identifier, definition) - - async def test_update_definition_missing_identifier(self) -> None: - request = MagicMock(spec=Request) - request.match_info = {"identifier": ""} - request.json = AsyncMock(return_value={}) - - task_definitions = MagicMock(spec=TaskDefinitions) - - response = await api.task_definitions_update(request, Encoder(), task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPBadRequest.status_code - assert "error" in body - - async def test_delete_definition_success(self) -> None: - request = MagicMock(spec=Request) - identifier = "c9f4ac6c-a4ab-4d1a-8d25-764dc0c8a3f0" - request.match_info = {"identifier": identifier} - - task_definitions = MagicMock(spec=TaskDefinitions) - - response = await api.task_definitions_delete(request, task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPOk.status_code - assert body["status"] == "deleted" - task_definitions.delete.assert_called_once_with(identifier) - - async def test_delete_definition_missing_identifier(self) -> None: - request = MagicMock(spec=Request) - request.match_info = {"identifier": ""} - - task_definitions = MagicMock(spec=TaskDefinitions) - - response = await api.task_definitions_delete(request, task_definitions) - body = json.loads(response.text) - - assert response.status == web.HTTPBadRequest.status_code - assert "error" in body diff --git a/app/tests/test_tasks.py b/app/tests/test_tasks.py deleted file mode 100644 index 92163ca3..00000000 --- a/app/tests/test_tasks.py +++ /dev/null @@ -1,1069 +0,0 @@ -import tempfile -from pathlib import Path -from unittest.mock import AsyncMock, Mock, patch - -import pytest - -from app.library.Tasks import Task, Tasks - - -class TestTask: - """Test the Task dataclass.""" - - def test_task_creation_with_required_fields(self): - """Test creating a task with only required fields.""" - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - assert task.id == "test_id" - assert task.name == "test_task" - assert task.url == "https://example.com/video" - assert task.folder == "" - assert task.preset == "" - assert task.timer == "" - assert task.template == "" - assert task.cli == "" - assert task.auto_start is True - assert task.handler_enabled is True - assert task.enabled is True - - def test_task_creation_with_all_fields(self): - """Test creating a task with all fields specified.""" - task = Task( - id="test_id", - name="test_task", - url="https://example.com/video", - folder="test_folder", - preset="test_preset", - timer="0 */6 * * *", - template="%(title)s.%(ext)s", - cli="--extract-flat", - auto_start=False, - handler_enabled=False, - enabled=False, - ) - - assert task.id == "test_id" - assert task.name == "test_task" - assert task.url == "https://example.com/video" - assert task.folder == "test_folder" - assert task.preset == "test_preset" - assert task.timer == "0 */6 * * *" - assert task.template == "%(title)s.%(ext)s" - assert task.cli == "--extract-flat" - assert task.auto_start is False - assert task.handler_enabled is False - assert task.enabled is False - - def test_task_serialize(self): - """Test task serialization to dictionary.""" - task = Task( - id="test_id", - name="test_task", - url="https://example.com/video", - preset="test_preset", - ) - - serialized = task.serialize() - - assert isinstance(serialized, dict) - assert serialized["id"] == "test_id" - assert serialized["name"] == "test_task" - assert serialized["url"] == "https://example.com/video" - assert serialized["preset"] == "test_preset" - assert serialized["folder"] == "" - assert serialized["auto_start"] is True - - @patch("app.library.Tasks.Encoder") - def test_task_json(self, mock_encoder): - """Test task JSON serialization.""" - mock_encoder_instance = Mock() - mock_encoder_instance.encode.return_value = '{"id": "test_id"}' - mock_encoder.return_value = mock_encoder_instance - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - json_str = task.json() - - assert json_str == '{"id": "test_id"}' - mock_encoder_instance.encode.assert_called_once() - - def test_task_get_method(self): - """Test task get method for accessing fields.""" - task = Task( - id="test_id", - name="test_task", - url="https://example.com/video", - preset="test_preset", - ) - - assert task.get("id") == "test_id" - assert task.get("name") == "test_task" - assert task.get("preset") == "test_preset" - assert task.get("nonexistent") is None - assert task.get("nonexistent", "default_value") == "default_value" - - @patch("app.library.Tasks.YTDLPOpts") - def test_task_get_ytdlp_opts_without_preset_or_cli(self, mock_ytdlp_opts): - """Test getting YTDLPOpts without preset or CLI options.""" - mock_opts_instance = Mock() - mock_ytdlp_opts.get_instance.return_value = mock_opts_instance - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - result = task.get_ytdlp_opts() - - assert result == mock_opts_instance - mock_ytdlp_opts.get_instance.assert_called_once() - - @patch("app.library.Tasks.YTDLPOpts") - def test_task_get_ytdlp_opts_with_preset_and_cli(self, mock_ytdlp_opts): - """Test getting YTDLPOpts with preset and CLI options.""" - mock_opts_instance = Mock() - mock_preset_opts = Mock() - mock_final_opts = Mock() - - mock_opts_instance.preset.return_value = mock_preset_opts - mock_preset_opts.add_cli.return_value = mock_final_opts - mock_ytdlp_opts.get_instance.return_value = mock_opts_instance - - task = Task( - id="test_id", - name="test_task", - url="https://example.com/video", - preset="test_preset", - cli="--extract-flat", - ) - - result = task.get_ytdlp_opts() - - assert result == mock_final_opts - mock_opts_instance.preset.assert_called_once_with(name="test_preset") - mock_preset_opts.add_cli.assert_called_once_with("--extract-flat", from_user=True) - - @patch("app.library.Tasks.archive_add") - @patch("app.library.Tasks.extract_info") - def test_task_mark_success(self, mock_extract_info, mock_archive_add): - """Test successful task marking.""" - # Mock extract_info to return playlist data - mock_extract_info.return_value = { - "_type": "playlist", - "entries": [ - { - "_type": "video", - "id": "video1", - "ie_key": "Youtube", - }, - { - "_type": "video", - "id": "video2", - "ie_key": "Youtube", - }, - ], - } - mock_archive_add.return_value = True - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {"download_archive": "/tmp/archive.txt"} - - task = Task(id="test_id", name="test_task", url="https://example.com/playlist") - - success, message = task.mark() - - assert success is True - assert "marked as downloaded" in message - mock_archive_add.assert_called_once() - # Check that archive_add was called with the correct items - call_args = mock_archive_add.call_args[0] - archive_file = call_args[0] - items = call_args[1] - assert str(archive_file) == "/tmp/archive.txt" - assert "youtube video1" in items - assert "youtube video2" in items - - def test_task_mark_no_url(self): - """Test task marking with no URL.""" - task = Task(id="test_id", name="test_task", url="") - - success, message = task.mark() - - assert success is False - assert "No URL found" in message - - def test_task_mark_no_archive_file(self): - """Test task marking with no archive file configured.""" - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {} - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - success, message = task.mark() - - assert success is False - assert "No archive file found" in message - - @patch("app.library.Tasks.archive_delete") - @patch("app.library.Tasks.extract_info") - def test_task_unmark_success(self, mock_extract_info, mock_archive_delete): - """Test successful task unmarking.""" - # Mock extract_info to return playlist data - mock_extract_info.return_value = { - "_type": "playlist", - "entries": [ - { - "_type": "video", - "id": "video1", - "ie_key": "Youtube", - }, - ], - } - mock_archive_delete.return_value = True - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {"download_archive": "/tmp/archive.txt"} - - task = Task(id="test_id", name="test_task", url="https://example.com/playlist") - - success, message = task.unmark() - - assert success is True - assert "Removed" in message - assert "items from archive file" in message - mock_archive_delete.assert_called_once() - - @patch("app.library.Tasks.extract_info") - def test_task_mark_logic_invalid_extract_info(self, mock_extract_info): - """Test _mark_logic with invalid extract_info response.""" - mock_extract_info.return_value = None - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {"download_archive": "/tmp/archive.txt"} - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - result = task._mark_logic() - - assert isinstance(result, tuple) - success, message = result - assert success is False - assert "Failed to extract information" in message - - @patch("app.library.Tasks.extract_info") - def test_task_mark_logic_not_playlist(self, mock_extract_info): - """Test _mark_logic with non-playlist type.""" - mock_extract_info.return_value = { - "_type": "video", - "id": "video1", - } - - with patch.object(Task, "get_ytdlp_opts") as mock_get_opts: - mock_get_opts.return_value.get_all.return_value = {"download_archive": "/tmp/archive.txt"} - - task = Task(id="test_id", name="test_task", url="https://example.com/video") - - result = task._mark_logic() - - assert isinstance(result, tuple) - success, message = result - assert success is False - assert "Expected a playlist type" in message - - -class TestTasks: - """Test the Tasks singleton manager.""" - - def setup_method(self): - """Set up test environment before each test.""" - # Reset singleton instance to ensure clean state - Tasks._reset_singleton() - - def teardown_method(self): - """Clean up after each test.""" - # Reset singleton instance to prevent test pollution - Tasks._reset_singleton() - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_singleton(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test that Tasks follows singleton pattern.""" - mock_config.get_instance.return_value = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - instance1 = Tasks.get_instance() - instance2 = Tasks.get_instance() - - assert instance1 is instance2 - assert isinstance(instance1, Tasks) - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_initialization(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test Tasks initialization with proper dependencies.""" - mock_config_instance = Mock( - debug=True, default_preset="test_preset", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - mock_scheduler_instance = Mock() - mock_scheduler.get_instance.return_value = mock_scheduler_instance - mock_download_queue_instance = Mock() - mock_download_queue.get_instance.return_value = mock_download_queue_instance - - with tempfile.TemporaryDirectory() as temp_dir: - tasks_file = Path(temp_dir) / "tasks.json" - tasks = Tasks(file=tasks_file, config=mock_config_instance) - - assert tasks._debug is True, "Check initialization" - assert tasks._default_preset == "test_preset" - assert tasks._file == tasks_file - assert tasks._tasks == [] - assert tasks._scheduler == mock_scheduler_instance - assert tasks._notify == mock_eventbus_instance - assert tasks._downloadQueue == mock_download_queue_instance - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_get_all_empty(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test getting all tasks when no tasks exist.""" - mock_config.get_instance.return_value = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks.get_instance() - - result = tasks.get_all() - - assert result == [] - assert isinstance(result, list) - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_get_by_id_not_found(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test getting task by ID when task doesn't exist.""" - mock_config.get_instance.return_value = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks.get_instance() - - result = tasks.get("nonexistent_id") - - assert result is None - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_load_empty_file(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test loading tasks from empty or non-existent file.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - tasks_file = Path(temp_dir) / "nonexistent.json" - tasks = Tasks(file=tasks_file, config=mock_config_instance) - - result = tasks.load() - - assert result == tasks # Should return self - assert tasks.get_all() == [] - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_load_valid_file(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test loading tasks from valid JSON file.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler_instance = Mock() - mock_scheduler.get_instance.return_value = mock_scheduler_instance - mock_download_queue.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - tasks_file = Path(temp_dir) / "tasks.json" - - # Create valid JSON file with task data - task_data = [ - { - "id": "task1", - "name": "Test Task 1", - "url": "https://example.com/video1", - "preset": "default", - "timer": "0 6 * * *", - "folder": "", - "template": "", - "cli": "", - "auto_start": True, - "handler_enabled": True, - } - ] - tasks_file.write_text("[\n" + "\n".join(f" {line}" for line in str(task_data).split("\n")) + "\n]") - - # Mock json.loads to return our test data - with patch("json.loads", return_value=task_data): - tasks = Tasks(file=tasks_file, config=mock_config_instance) - - # Mock the scheduler.add method - mock_scheduler_instance.add = Mock() - - result = tasks.load() - - assert result == tasks - loaded_tasks = tasks.get_all() - assert len(loaded_tasks) == 1 - assert loaded_tasks[0].id == "task1" - assert loaded_tasks[0].name == "Test Task 1" - assert loaded_tasks[0].url == "https://example.com/video1" - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_load_invalid_json(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test loading tasks from file with invalid JSON.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - tasks_file = Path(temp_dir) / "tasks.json" - tasks_file.write_text("invalid json {") - - tasks = Tasks(file=tasks_file, config=mock_config_instance) - - result = tasks.load() - - assert result == tasks - assert tasks.get_all() == [] # Should be empty due to invalid JSON - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_save_valid_tasks(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test saving valid tasks to file.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - with tempfile.TemporaryDirectory() as temp_dir: - tasks_file = Path(temp_dir) / "tasks.json" - tasks = Tasks(file=tasks_file, config=mock_config_instance) - - task_data = [ - { - "id": "task1", - "name": "Test Task", - "url": "https://example.com/video", - "preset": "default", - "folder": "", - "template": "", - "cli": "", - "timer": "", - "auto_start": True, - "handler_enabled": True, - } - ] - - result = tasks.save(task_data) - - assert result == tasks - assert tasks_file.exists() - assert tasks_file.stat().st_size > 0, "Verify file content was written" - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_get_by_id_found(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test getting task by ID when task exists.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks(file=None, config=mock_config_instance) - - # Manually add a task to the internal list - test_task = Task(id="test_id", name="Test Task", url="https://example.com/video") - tasks._tasks.append(test_task) - - result = tasks.get("test_id") - - assert result is not None - assert result.id == "test_id" - assert result.name == "Test Task" - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_get_all_with_tasks(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test getting all tasks when tasks exist.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks(file=None, config=mock_config_instance) - - # Manually add tasks to the internal list - task1 = Task(id="task1", name="Task 1", url="https://example.com/video1") - task2 = Task(id="task2", name="Task 2", url="https://example.com/video2") - tasks._tasks.extend([task1, task2]) - - result = tasks.get_all() - - assert len(result) == 2 - assert result[0].id == "task1" - assert result[1].id == "task2" - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_clear_with_scheduler_cleanup(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test clearing tasks with scheduler cleanup.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler_instance = Mock() - mock_scheduler_instance.has.return_value = True - mock_scheduler_instance.remove = Mock() - mock_scheduler.get_instance.return_value = mock_scheduler_instance - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks(file=None, config=mock_config_instance) - - # Add a task with timer to the internal list - task_with_timer = Task(id="task1", name="Task 1", url="https://example.com/video1", timer="0 6 * * *") - tasks._tasks.append(task_with_timer) - - result = tasks.clear() - - assert result == tasks - assert len(tasks.get_all()) == 0 - mock_scheduler_instance.has.assert_called_with("task1") - mock_scheduler_instance.remove.assert_called_with("task1") - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_clear_no_tasks(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test clearing when no tasks exist.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler_instance = Mock() - mock_scheduler.get_instance.return_value = mock_scheduler_instance - mock_download_queue.get_instance.return_value = Mock() - - tasks = Tasks(file=None, config=mock_config_instance) - - result = tasks.clear() - - assert result == tasks - assert len(tasks.get_all()) == 0 - # Should not call scheduler methods when no tasks exist - mock_scheduler_instance.has.assert_not_called() - mock_scheduler_instance.remove.assert_not_called() - - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - def test_tasks_validate_valid_task_dict(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test task validation with valid task dictionary.""" - mock_config.get_instance.return_value = Mock() - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - valid_task = { - "name": "Test Task", - "url": "https://example.com/video", - "timer": "0 6 * * *", - "cli": "--write-info-json", - } - - with patch("app.library.Tasks.validate_url") as mock_validate_url: - mock_validate_url.return_value = True - - result = Tasks.validate(valid_task) - - assert result is True - mock_validate_url.assert_called_once_with("https://example.com/video", allow_internal=True) - - def test_tasks_validate_invalid_task_type(self): - """Test task validation with invalid task type.""" - invalid_task = "not a dict or Task" - - with pytest.raises(ValueError, match="Invalid task type"): - Tasks.validate(invalid_task) - - def test_tasks_validate_missing_name(self): - """Test task validation with missing name.""" - invalid_task = { - "url": "https://example.com/video", - } - - with pytest.raises(ValueError, match="No name found"): - Tasks.validate(invalid_task) - - def test_tasks_validate_missing_url(self): - """Test task validation with missing URL.""" - invalid_task = { - "name": "Test Task", - } - - with pytest.raises(ValueError, match="No URL found"): - Tasks.validate(invalid_task) - - def test_tasks_validate_invalid_url(self): - """Test task validation with invalid URL.""" - invalid_task = { - "name": "Test Task", - "url": "invalid-url", - } - - with patch("app.library.Tasks.validate_url") as mock_validate_url: - mock_validate_url.side_effect = ValueError("Invalid URL") - - with pytest.raises(ValueError, match="Invalid URL format"): - Tasks.validate(invalid_task) - - def test_tasks_validate_invalid_timer(self): - """Test task validation with invalid timer format.""" - invalid_task = { - "name": "Test Task", - "url": "https://example.com/video", - "timer": "invalid timer", - } - - with patch("app.library.Tasks.validate_url") as mock_validate_url: - mock_validate_url.return_value = True - - # Import error will be raised by cronsim for invalid format - with patch("cronsim.CronSim") as mock_cronsim: - mock_cronsim.side_effect = Exception("Invalid cron format") - - with pytest.raises(ValueError, match="Invalid timer format"): - Tasks.validate(invalid_task) - - def test_tasks_validate_invalid_cli(self): - """Test task validation with invalid CLI options.""" - invalid_task = { - "name": "Test Task", - "url": "https://example.com/video", - "cli": "invalid --cli options", - } - - with patch("app.library.Tasks.validate_url") as mock_validate_url: - mock_validate_url.return_value = True - - with patch("app.library.Utils.arg_converter") as mock_arg_converter: - mock_arg_converter.side_effect = Exception("Invalid CLI args") - - with pytest.raises(ValueError, match="Invalid command options for yt-dlp"): - Tasks.validate(invalid_task) - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.Item") - @patch("app.library.Tasks.datetime") - @patch("app.library.Tasks.time") - async def test_tasks_runner_success( - self, mock_time, mock_datetime, mock_item, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test successful task runner execution.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - mock_scheduler.get_instance.return_value = Mock() - - # Mock download queue - mock_download_queue_instance = Mock() - mock_download_queue_instance.add = AsyncMock(return_value={"success": True, "id": "download123"}) - mock_download_queue.get_instance.return_value = mock_download_queue_instance - - # Mock time and datetime - mock_time.time.side_effect = [1000.0, 1005.5] # start and end times - mock_datetime_instance = Mock() - mock_datetime_instance.isoformat.return_value = "2024-01-01T12:00:00Z" - mock_datetime.now.return_value = mock_datetime_instance - - # Mock Item.format - mock_item.format.return_value = {"formatted": "item"} - - tasks = Tasks(file=None, config=mock_config_instance) - - test_task = Task( - id="task1", - name="Test Task", - url="https://example.com/video", - preset="test_preset", - folder="test_folder", - template="test_template", - cli="--write-info-json", - auto_start=True, - enabled=True, - ) - - # Add task to tasks list so it can be found by _runner - tasks._tasks.append(test_task) - - await tasks._runner(test_task) - - # Verify download queue was called - mock_download_queue_instance.add.assert_called_once() - - # Verify Item.format was called with correct parameters - mock_item.format.assert_called_once_with( - { - "url": "https://example.com/video", - "preset": "test_preset", - "folder": "test_folder", - "template": "test_template", - "cli": "--write-info-json", - "auto_start": True, - "extras": { - "source_name": "Test Task", - "source_id": "task1", - "source_handler": "Tasks", - }, - } - ) - - assert mock_eventbus_instance.emit.call_count == 2, "Verify events were emitted" - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.datetime") - async def test_tasks_runner_no_url( - self, mock_datetime, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test task runner with no URL.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue_instance = Mock() - mock_download_queue.get_instance.return_value = mock_download_queue_instance - - mock_datetime_instance = Mock() - mock_datetime_instance.isoformat.return_value = "2024-01-01T12:00:00Z" - mock_datetime.now.return_value = mock_datetime_instance - - tasks = Tasks(file=None, config=mock_config_instance) - - test_task = Task( - id="task1", - name="Test Task", - url="", # Empty URL to trigger error - enabled=True, - ) - - # Add task to tasks list so it can be found by _runner - tasks._tasks.append(test_task) - - await tasks._runner(test_task) - - # Verify download queue was NOT called - mock_download_queue_instance.add.assert_not_called() - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.Item") - @patch("app.library.Tasks.datetime") - async def test_tasks_runner_download_queue_failure( - self, mock_datetime, mock_item, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test task runner when download queue add fails.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - mock_scheduler.get_instance.return_value = Mock() - - # Mock download queue to raise exception - mock_download_queue_instance = Mock() - mock_download_queue_instance.add = AsyncMock(side_effect=Exception("Queue error")) - mock_download_queue.get_instance.return_value = mock_download_queue_instance - - mock_datetime_instance = Mock() - mock_datetime_instance.isoformat.return_value = "2024-01-01T12:00:00Z" - mock_datetime.now.return_value = mock_datetime_instance - - mock_item.format.return_value = {"formatted": "item"} - - tasks = Tasks(file=None, config=mock_config_instance) - - test_task = Task( - id="task1", - name="Test Task", - url="https://example.com/video", - enabled=True, - ) - - # Add task to tasks list so it can be found by _runner - tasks._tasks.append(test_task) - - await tasks._runner(test_task) - - # Verify error event was emitted (check last call) - calls = mock_eventbus_instance.emit.call_args_list - assert len(calls) > 0 - # Verify the last call was an error event - last_call = calls[-1] - assert "Task failed" in str(last_call) - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - async def test_tasks_runner_disabled_task(self, mock_download_queue, mock_scheduler, mock_eventbus, mock_config): - """Test that disabled tasks are skipped by the runner.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus_instance = Mock() - mock_eventbus.get_instance.return_value = mock_eventbus_instance - mock_scheduler.get_instance.return_value = Mock() - - # Mock download queue - mock_download_queue_instance = Mock() - mock_download_queue_instance.add = AsyncMock(return_value={"success": True, "id": "download123"}) - mock_download_queue.get_instance.return_value = mock_download_queue_instance - - tasks = Tasks(file=None, config=mock_config_instance) - - test_task = Task( - id="task1", - name="Test Task", - url="https://example.com/video", - enabled=False, # Task is disabled - ) - - # Add task to tasks list so it can be found by _runner - tasks._tasks.append(test_task) - - await tasks._runner(test_task) - - # Verify download queue was NOT called (task was skipped) - mock_download_queue_instance.add.assert_not_called() - - -class TestHandleTaskInspect: - """Test the HandleTask inspect method with static_only parameter.""" - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.Services") - async def test_inspect_with_static_only_true( - self, mock_services, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test inspect with static_only=True returns immediately after can_handle check.""" - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - # Mock Services to simulate successful can_handle - mock_services_instance = Mock() - mock_services_instance.handle_sync = Mock(return_value=True) - mock_services_instance.handle_async = AsyncMock() # Should NOT be called with static_only=True - mock_services.get_instance.return_value = mock_services_instance - - # Create a mock handler - mock_handler = Mock() - mock_handler.__name__ = "TestHandler" - mock_handler.can_handle = Mock(return_value=True) - mock_handler.extract = AsyncMock() # Should NOT be called with static_only=True - - tasks = Tasks(file=None, config=mock_config_instance) - handler = tasks.get_handler() - handler._handlers = [mock_handler] - - # Call inspect with static_only=True - result = await handler.inspect( - url="https://example.com/feed", preset="default", handler_name=None, static_only=True - ) - - assert hasattr(result, "items"), "Verify result structure" - assert hasattr(result, "metadata") - assert result.items == [] - assert result.metadata["matched"] is True - assert result.metadata["handler"] == "TestHandler" - - # Verify handle_async (extract) was NOT called - mock_services_instance.handle_async.assert_not_called() - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.Services") - async def test_inspect_with_static_only_false( - self, mock_services, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test inspect with static_only=False performs full extraction.""" - from app.library.Tasks import TaskResult - - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - # Mock Services - mock_services_instance = Mock() - mock_services_instance.handle_sync = Mock(return_value=True) - - # Mock successful extract result - mock_extract_result = TaskResult( - items=[{"url": "https://example.com/video1", "title": "Video 1"}], - metadata={"raw_count": 1}, - ) - mock_services_instance.handle_async = AsyncMock(return_value=mock_extract_result) - mock_services.get_instance.return_value = mock_services_instance - - # Create a mock handler - mock_handler = Mock() - mock_handler.__name__ = "TestHandler" - mock_handler.can_handle = Mock(return_value=True) - mock_handler.extract = AsyncMock(return_value=mock_extract_result) - - tasks = Tasks(file=None, config=mock_config_instance) - handler = tasks.get_handler() - handler._handlers = [mock_handler] - - # Call inspect with static_only=False (default) - result = await handler.inspect(url="https://example.com/feed", preset="default", handler_name=None) - - assert hasattr(result, "items"), "Verify result structure includes extracted items" - assert hasattr(result, "metadata") - assert len(result.items) > 0 - assert result.metadata["matched"] is True - assert result.metadata["handler"] == "TestHandler" - assert result.metadata["supported"] is True - - # Verify handle_async (extract) WAS called - mock_services_instance.handle_async.assert_called_once() - - @pytest.mark.asyncio - @patch("app.library.Tasks.Config") - @patch("app.library.Tasks.EventBus") - @patch("app.library.Tasks.Scheduler") - @patch("app.library.Tasks.DownloadQueue") - @patch("app.library.Tasks.Services") - async def test_inspect_static_only_with_no_handler_match( - self, mock_services, mock_download_queue, mock_scheduler, mock_eventbus, mock_config - ): - """Test inspect with static_only=True when no handler matches.""" - from app.library.Tasks import TaskFailure - - mock_config_instance = Mock( - debug=False, default_preset="default", config_path="/tmp", tasks_handler_timer="15 */1 * * *" - ) - mock_config.get_instance.return_value = mock_config_instance - mock_eventbus.get_instance.return_value = Mock() - mock_scheduler.get_instance.return_value = Mock() - mock_download_queue.get_instance.return_value = Mock() - - # Mock Services to simulate failed can_handle - mock_services_instance = Mock() - mock_services_instance.handle_sync = Mock(return_value=False) - mock_services.get_instance.return_value = mock_services_instance - - # Create a mock handler that won't match - mock_handler = Mock() - mock_handler.__name__ = "TestHandler" - mock_handler.can_handle = Mock(return_value=False) - - tasks = Tasks(file=None, config=mock_config_instance) - handler = tasks.get_handler() - handler._handlers = [mock_handler] - - # Call inspect with static_only=True - result = await handler.inspect( - url="https://unsupported.com/feed", preset="default", handler_name=None, static_only=True - ) - - # Verify result is a TaskFailure - assert isinstance(result, TaskFailure) - assert result.metadata["matched"] is False - assert result.metadata["handler"] is None diff --git a/app/tests/test_update_checker.py b/app/tests/test_update_checker.py index 18cebf34..f876b865 100644 --- a/app/tests/test_update_checker.py +++ b/app/tests/test_update_checker.py @@ -53,23 +53,19 @@ class TestUpdateChecker: config = Config.get_instance() config.check_for_updates = True - # Create scheduler with a fresh event loop loop = asyncio.new_event_loop() scheduler = Scheduler.get_instance(loop=loop) notify = EventBus.get_instance() - # Mock APP_VERSION to be a release version with patch("app.library.UpdateChecker.APP_VERSION", "v1.0.0"): checker = UpdateChecker.get_instance(config=config, scheduler=scheduler) app_mock = MagicMock() try: checker.attach(app_mock) - assert checker._job_id is not None, "Should have scheduled a job" - assert "update_checker" == checker._job_id, "Job ID should be 'update_checker'" + assert "UpdateChecker.check_for_updates" == checker._job_id, "Job ID should be 'update_checker'" finally: - # Clean up if checker._job_id: scheduler.remove(checker._job_id) loop.close() @@ -118,7 +114,6 @@ class TestUpdateChecker: config = Config.get_instance() config.check_for_updates = True - # Create scheduler with a fresh event loop loop = asyncio.new_event_loop() scheduler = Scheduler.get_instance(loop=loop) notify = EventBus.get_instance() @@ -126,15 +121,11 @@ class TestUpdateChecker: with patch("app.library.UpdateChecker.APP_VERSION", "v1.0.0"): checker = UpdateChecker.get_instance(config=config, scheduler=scheduler) app_mock = MagicMock() - try: checker.attach(app_mock) initial_job_id = checker._job_id - assert initial_job_id is not None, "Should have job ID before shutdown" - await checker.on_shutdown(app_mock) - assert checker._job_id is None, "Should clear job ID after shutdown" finally: loop.close() @@ -148,15 +139,18 @@ class TestUpdateChecker: config = Config.get_instance() config.check_for_updates = False config.new_version = "" + config.yt_new_version = "" checker = UpdateChecker.get_instance(config=config) - # Should return disabled status - status, new_version = await checker.check_for_updates() + (app_status, app_version), (ytdlp_status, ytdlp_version) = await checker.check_for_updates() - assert "disabled" == status, "Should return disabled status" - assert new_version is None, "Should return None for new_version when disabled" + assert "disabled" == app_status, "Should return disabled status for app" + assert app_version is None, "Should return None for app new_version when disabled" + assert "disabled" == ytdlp_status, "Should return disabled status for yt-dlp" + assert ytdlp_version is None, "Should return None for yt-dlp new_version when disabled" assert "" == config.new_version, "Should not update new_version when disabled" + assert "" == config.yt_new_version, "Should not update yt_new_version when disabled" @pytest.mark.asyncio @patch("app.library.UpdateChecker.async_client") @@ -168,25 +162,32 @@ class TestUpdateChecker: config = Config.get_instance() config.check_for_updates = True config.new_version = "" + config.yt_new_version = "" - # Mock HTTP response - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = {"tag_name": "v99.0.0"} + mock_app_response = MagicMock() + mock_app_response.status_code = 200 + mock_app_response.json.return_value = {"tag_name": "v99.0.0"} + mock_ytdlp_response = MagicMock() + mock_ytdlp_response.status_code = 200 + mock_ytdlp_response.json.return_value = {"tag_name": "9999.12.31"} + + mock_get = AsyncMock(side_effect=[mock_app_response, mock_ytdlp_response]) mock_context = AsyncMock() - mock_context.__aenter__.return_value.get = AsyncMock(return_value=mock_response) + mock_context.__aenter__.return_value.get = mock_get mock_client.return_value = mock_context - # Mock current version to be older with patch("app.library.UpdateChecker.APP_VERSION", "v1.0.0"): checker = UpdateChecker.get_instance(config=config) - status, new_version = await checker.check_for_updates() + (app_status, app_version), (ytdlp_status, ytdlp_version) = await checker.check_for_updates() - assert "update_available" == status, "Should return update_available status" - assert "v99.0.0" == new_version, "Should return new version tag" - assert "v99.0.0" == config.new_version, "Should store new version tag" - assert checker._job_id is None, "Should stop scheduled task after finding update" + assert "update_available" == app_status, "Should return update_available status for app" + assert "v99.0.0" == app_version, "Should return new app version tag" + assert "v99.0.0" == config.new_version, "Should store new app version tag" + assert "update_available" == ytdlp_status, "Should return update_available status for yt-dlp" + assert "9999.12.31" == ytdlp_version, "Should return new yt-dlp version tag" + assert "9999.12.31" == config.yt_new_version, "Should store new yt-dlp version tag" + assert checker._job_id is None, "Should stop scheduled task after finding app update" @pytest.mark.asyncio @patch("app.library.UpdateChecker.async_client") @@ -198,24 +199,32 @@ class TestUpdateChecker: config = Config.get_instance() config.check_for_updates = True config.new_version = "" + config.yt_new_version = "" - # Mock HTTP response with older version (current is hardcoded as 1.0.14 in the code) - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = {"tag_name": "v1.0.0"} + mock_app_response = MagicMock() + mock_app_response.status_code = 200 + mock_app_response.json.return_value = {"tag_name": "v1.0.0"} + mock_ytdlp_response = MagicMock() + mock_ytdlp_response.status_code = 200 + mock_ytdlp_response.json.return_value = {"tag_name": "2020.01.01"} + + mock_get = AsyncMock(side_effect=[mock_app_response, mock_ytdlp_response]) mock_context = AsyncMock() - mock_context.__aenter__.return_value.get = AsyncMock(return_value=mock_response) + mock_context.__aenter__.return_value.get = mock_get mock_client.return_value = mock_context checker = UpdateChecker.get_instance(config=config) - checker._job_id = "test-job" # Set a job ID to verify it's not removed + checker._job_id = "test-job" - status, new_version = await checker.check_for_updates() + (app_status, app_version), (ytdlp_status, ytdlp_version) = await checker.check_for_updates() - assert "up_to_date" == status, "Should return up_to_date status" - assert new_version is None, "Should return None for new_version" - assert "" == config.new_version, "Should clear new_version when no update available" + assert "up_to_date" == app_status, "Should return up_to_date status for app" + assert app_version is None, "Should return None for app new_version" + assert "" == config.new_version, "Should clear new_version when no app update available" + assert "up_to_date" == ytdlp_status, "Should return up_to_date status for yt-dlp" + assert ytdlp_version is None, "Should return None for yt-dlp new_version" + assert "" == config.yt_new_version, "Should clear yt_new_version when no yt-dlp update available" assert "test-job" == checker._job_id, "Should keep scheduled task running" @pytest.mark.asyncio @@ -228,8 +237,8 @@ class TestUpdateChecker: config = Config.get_instance() config.check_for_updates = True config.new_version = "" + config.yt_new_version = "" - # Mock HTTP error mock_response = MagicMock() mock_response.status_code = 404 @@ -239,11 +248,14 @@ class TestUpdateChecker: checker = UpdateChecker.get_instance(config=config) - status, new_version = await checker.check_for_updates() + (app_status, app_version), (ytdlp_status, ytdlp_version) = await checker.check_for_updates() - assert "error" == status, "Should return error status" - assert new_version is None, "Should return None for new_version on error" + assert "error" == app_status, "Should return error status for app" + assert app_version is None, "Should return None for app new_version on error" assert "" == config.new_version, "Should not set new_version on HTTP error" + assert "error" == ytdlp_status, "Should return error status for yt-dlp" + assert ytdlp_version is None, "Should return None for yt-dlp new_version on error" + assert "" == config.yt_new_version, "Should not set yt_new_version on HTTP error" @pytest.mark.asyncio @patch("app.library.UpdateChecker.async_client") @@ -255,19 +267,22 @@ class TestUpdateChecker: config = Config.get_instance() config.check_for_updates = True config.new_version = "" + config.yt_new_version = "" - # Mock exception during HTTP request mock_context = AsyncMock() mock_context.__aenter__.return_value.get = AsyncMock(side_effect=Exception("Network error")) mock_client.return_value = mock_context checker = UpdateChecker.get_instance(config=config) - status, new_version = await checker.check_for_updates() + (app_status, app_version), (ytdlp_status, ytdlp_version) = await checker.check_for_updates() - assert "error" == status, "Should return error status on exception" - assert new_version is None, "Should return None for new_version on exception" + assert "error" == app_status, "Should return error status on exception for app" + assert app_version is None, "Should return None for app new_version on exception" assert "" == config.new_version, "Should not set new_version on exception" + assert "error" == ytdlp_status, "Should return error status on exception for yt-dlp" + assert ytdlp_version is None, "Should return None for yt-dlp new_version on exception" + assert "" == config.yt_new_version, "Should not set yt_new_version on exception" def test_compare_versions_newer_available(self): """Test version comparison detects newer version.""" @@ -311,13 +326,19 @@ class TestUpdateChecker: config = Config.get_instance() config.check_for_updates = True config.new_version = "" + config.yt_new_version = "" - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = {"tag_name": "v2.0.0"} + mock_app_response = MagicMock() + mock_app_response.status_code = 200 + mock_app_response.json.return_value = {"tag_name": "v2.0.0"} + mock_ytdlp_response = MagicMock() + mock_ytdlp_response.status_code = 200 + mock_ytdlp_response.json.return_value = {"tag_name": "2026.01.01"} + + mock_get = AsyncMock(side_effect=[mock_app_response, mock_ytdlp_response]) mock_context = AsyncMock() - mock_context.__aenter__.return_value.get = AsyncMock(return_value=mock_response) + mock_context.__aenter__.return_value.get = mock_get with patch("app.library.UpdateChecker.async_client", return_value=mock_context): with patch("app.library.UpdateChecker.APP_VERSION", "v1.0.0"): @@ -325,6 +346,7 @@ class TestUpdateChecker: await checker.check_for_updates() assert "v2.0.0" == config.new_version, "Should store full tag_name including 'v' prefix" + assert "2026.01.01" == config.yt_new_version, "Should store yt-dlp tag_name" def test_subscribe_to_started_event(self): """Test that attach subscribes to Events.STARTED.""" @@ -350,13 +372,128 @@ class TestUpdateChecker: checker.attach(app_mock) - # Verify subscription was created - subscriptions = notify._listeners.get("started", {}) + subscriptions = notify._listeners.get("started", []) assert len(subscriptions) > 0, "Should have subscribed to STARTED event" - assert any("UpdateChecker.attach" in name for name in subscriptions.keys()), ( + assert any("UpdateChecker.attach" in name for name, _ in subscriptions), ( "Should have UpdateChecker.attach subscription" ) finally: if checker and checker._job_id: scheduler.remove(checker._job_id) loop.close() + + @pytest.mark.asyncio + @patch("app.library.UpdateChecker.async_client") + async def test_check_ytdlp_version_finds_newer_version(self, mock_client): + """Test that yt-dlp check detects when a newer version is available.""" + from app.library.config import Config + from app.library.UpdateChecker import UpdateChecker + + config = Config.get_instance() + config.check_for_updates = True + config.yt_new_version = "" + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"tag_name": "9999.12.31"} + + mock_context = AsyncMock() + mock_context.__aenter__.return_value.get = AsyncMock(return_value=mock_response) + mock_client.return_value = mock_context + + checker = UpdateChecker.get_instance(config=config) + status, new_version = await checker._check_ytdlp_version() + + assert "update_available" == status, "Should return update_available status for yt-dlp" + assert "9999.12.31" == new_version, "Should return new yt-dlp version tag" + assert "9999.12.31" == config.yt_new_version, "Should store new yt-dlp version tag" + + @pytest.mark.asyncio + @patch("app.library.UpdateChecker.async_client") + async def test_check_ytdlp_version_no_update_available(self, mock_client): + """Test that yt-dlp check correctly handles when no update is available.""" + from app.library.config import Config + from app.library.UpdateChecker import UpdateChecker + + config = Config.get_instance() + config.check_for_updates = True + config.yt_new_version = "" + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"tag_name": "2020.01.01"} + + mock_context = AsyncMock() + mock_context.__aenter__.return_value.get = AsyncMock(return_value=mock_response) + mock_client.return_value = mock_context + + checker = UpdateChecker.get_instance(config=config) + status, new_version = await checker._check_ytdlp_version() + + assert "up_to_date" == status, "Should return up_to_date status for yt-dlp" + assert new_version is None, "Should return None for yt-dlp new_version" + assert "" == config.yt_new_version, "Should clear yt_new_version when no update available" + + @pytest.mark.asyncio + @patch("app.library.UpdateChecker.async_client") + async def test_check_ytdlp_version_handles_http_error(self, mock_client): + """Test that yt-dlp check handles HTTP errors gracefully.""" + from app.library.config import Config + from app.library.UpdateChecker import UpdateChecker + + config = Config.get_instance() + config.check_for_updates = True + config.yt_new_version = "" + + mock_response = MagicMock() + mock_response.status_code = 500 + + mock_context = AsyncMock() + mock_context.__aenter__.return_value.get = AsyncMock(return_value=mock_response) + mock_client.return_value = mock_context + + checker = UpdateChecker.get_instance(config=config) + status, new_version = await checker._check_ytdlp_version() + + assert "error" == status, "Should return error status for yt-dlp on HTTP error" + assert new_version is None, "Should return None for yt-dlp new_version on error" + assert "" == config.yt_new_version, "Should not set yt_new_version on HTTP error" + + @pytest.mark.asyncio + @patch("app.library.UpdateChecker.async_client") + async def test_check_for_updates_caches_separately(self, mock_client): + """Test that app and yt-dlp checks are cached separately.""" + from app.library.config import Config + from app.library.UpdateChecker import UpdateChecker + + config = Config.get_instance() + config.check_for_updates = True + config.new_version = "" + config.yt_new_version = "" + + mock_app_response = MagicMock() + mock_app_response.status_code = 200 + mock_app_response.json.return_value = {"tag_name": "v2.0.0"} + + mock_ytdlp_response = MagicMock() + mock_ytdlp_response.status_code = 200 + mock_ytdlp_response.json.return_value = {"tag_name": "2026.01.01"} + + mock_get = AsyncMock(side_effect=[mock_app_response, mock_ytdlp_response]) + mock_context = AsyncMock() + mock_context.__aenter__.return_value.get = mock_get + mock_client.return_value = mock_context + + with patch("app.library.UpdateChecker.APP_VERSION", "v1.0.0"): + checker = UpdateChecker.get_instance(config=config) + (app_status1, app_version1), (ytdlp_status1, ytdlp_version1) = await checker.check_for_updates() + + assert "update_available" == app_status1, "Should find app update on first call" + assert "update_available" == ytdlp_status1, "Should find yt-dlp update on first call" + + (app_status2, app_version2), (ytdlp_status2, ytdlp_version2) = await checker.check_for_updates() + + assert "update_available" == app_status2, "Should return cached app result" + assert "update_available" == ytdlp_status2, "Should return cached yt-dlp result" + assert app_version1 == app_version2, "App versions should match from cache" + assert ytdlp_version1 == ytdlp_version2, "yt-dlp versions should match from cache" diff --git a/app/tests/test_ytdlpopts.py b/app/tests/test_ytdlpopts.py index 916aad42..107a078f 100644 --- a/app/tests/test_ytdlpopts.py +++ b/app/tests/test_ytdlpopts.py @@ -4,7 +4,7 @@ from unittest.mock import Mock, patch import pytest -from app.library.Presets import Preset +from app.features.presets.schemas import Preset from app.library.YTDLPOpts import YTDLPOpts @@ -116,7 +116,7 @@ class TestYTDLPOpts: """Test applying a valid preset.""" with ( patch("app.library.YTDLPOpts.Config") as mock_config, - patch("app.library.YTDLPOpts.Presets") as mock_presets, + patch("app.features.presets.service.Presets") as mock_presets, patch("app.library.YTDLPOpts.arg_converter") as mock_converter, ): # Mock config @@ -155,7 +155,7 @@ class TestYTDLPOpts: def test_preset_with_nonexistent_preset(self): """Test applying a nonexistent preset returns self.""" - with patch("app.library.YTDLPOpts.Config"), patch("app.library.YTDLPOpts.Presets") as mock_presets: + with patch("app.library.YTDLPOpts.Config"), patch("app.features.presets.service.Presets") as mock_presets: mock_presets_instance = Mock() mock_presets_instance.get.return_value = None mock_presets.get_instance.return_value = mock_presets_instance @@ -171,7 +171,7 @@ class TestYTDLPOpts: """Test that preset with cookies creates cookie file.""" with ( patch("app.library.YTDLPOpts.Config") as mock_config, - patch("app.library.YTDLPOpts.Presets") as mock_presets, + patch("app.features.presets.service.Presets") as mock_presets, ): # Mock config mock_config_instance = Mock() @@ -209,7 +209,7 @@ class TestYTDLPOpts: """Test that preset with invalid CLI raises ValueError.""" with ( patch("app.library.YTDLPOpts.Config"), - patch("app.library.YTDLPOpts.Presets") as mock_presets, + patch("app.features.presets.service.Presets") as mock_presets, patch("app.library.YTDLPOpts.arg_converter") as mock_converter, ): mock_preset = Mock(spec=Preset) @@ -401,7 +401,7 @@ class TestYTDLPOpts: with ( patch("app.library.YTDLPOpts.Config"), patch("app.library.YTDLPOpts.arg_converter"), - patch("app.library.YTDLPOpts.Presets") as mock_presets, + patch("app.features.presets.service.Presets") as mock_presets, ): mock_presets_instance = Mock() mock_presets_instance.get.return_value = None @@ -441,7 +441,7 @@ class TestYTDLPOpts: """Test error handling when cookie loading fails.""" with ( patch("app.library.YTDLPOpts.Config") as mock_config, - patch("app.library.YTDLPOpts.Presets") as mock_presets, + patch("app.features.presets.service.Presets") as mock_presets, patch("app.library.YTDLPOpts.LOG") as mock_log, ): # Mock config @@ -741,7 +741,7 @@ class TestARGSMerger: class TestYTDLPCli: """Test the YTDLPCli class.""" - @patch("app.library.Presets.Presets") + @patch("app.features.presets.service.Presets") @patch("app.library.YTDLPOpts.Config") def test_constructor_with_valid_item(self, mock_config, mock_presets): """Test YTDLPCli constructor with valid Item.""" @@ -770,7 +770,7 @@ class TestYTDLPCli: with pytest.raises(ValueError, match="Expected Item instance"): YTDLPCli(item="not an item") # type: ignore - @patch("app.library.Presets.Presets") + @patch("app.features.presets.service.Presets") @patch("app.library.YTDLPOpts.Config") def test_build_with_user_fields_only(self, mock_config, mock_presets): """Test build with only user-provided fields.""" @@ -805,12 +805,12 @@ class TestYTDLPCli: assert info["merged"]["save_path"] == "/downloads/myfolder" assert "--format best" in command - @patch("app.library.Presets.Presets") + @patch("app.features.presets.service.Presets") @patch("app.library.YTDLPOpts.Config") def test_build_with_preset_fields_fallback(self, mock_config, mock_presets): """Test build falls back to preset fields when user doesn't provide them.""" from app.library.ItemDTO import Item - from app.library.Presets import Preset + from app.features.presets.schemas import Preset from app.library.YTDLPOpts import YTDLPCli mock_config_instance = Mock() @@ -839,12 +839,12 @@ class TestYTDLPCli: assert info["merged"]["save_path"] == "/downloads/preset_folder" assert "--format 720p" in command - @patch("app.library.Presets.Presets") + @patch("app.features.presets.service.Presets") @patch("app.library.YTDLPOpts.Config") def test_build_user_fields_override_preset(self, mock_config, mock_presets): """Test that user fields override preset fields.""" from app.library.ItemDTO import Item - from app.library.Presets import Preset + from app.features.presets.schemas import Preset from app.library.YTDLPOpts import YTDLPCli mock_config_instance = Mock() @@ -878,7 +878,7 @@ class TestYTDLPCli: assert info["merged"]["save_path"] == "/downloads/user_folder" assert "--format best" in command, "User CLI should appear after preset CLI in command" - @patch("app.library.Presets.Presets") + @patch("app.features.presets.service.Presets") @patch("app.library.YTDLPOpts.Config") def test_build_with_default_fallback(self, mock_config, mock_presets): """Test build falls back to defaults when neither user nor preset provide fields.""" @@ -903,7 +903,7 @@ class TestYTDLPCli: assert info["merged"]["template"] == "%(title)s.%(ext)s" assert info["merged"]["save_path"] == "/default/downloads" - @patch("app.library.Presets.Presets") + @patch("app.features.presets.service.Presets") @patch("app.library.YTDLPOpts.create_cookies_file") @patch("app.library.YTDLPOpts.Config") def test_build_with_cookies_from_user(self, mock_config, mock_create_cookies, mock_presets): @@ -933,12 +933,12 @@ class TestYTDLPCli: assert "/tmp/cookies.txt" in command assert info["merged"]["cookie_file"] == "/tmp/cookies.txt" - @patch("app.library.Presets.Presets") + @patch("app.features.presets.service.Presets") @patch("app.library.YTDLPOpts.Config") def test_build_with_cookies_from_preset(self, mock_config, mock_presets): """Test build with cookies from preset when user doesn't provide.""" from app.library.ItemDTO import Item - from app.library.Presets import Preset + from app.features.presets.schemas import Preset from app.library.YTDLPOpts import YTDLPCli mock_config_instance = Mock() @@ -969,7 +969,7 @@ class TestYTDLPCli: assert "--cookies" in command assert "/preset/cookies.txt" in command - @patch("app.library.Presets.Presets") + @patch("app.features.presets.service.Presets") @patch("app.library.YTDLPOpts.Config") def test_build_with_absolute_folder_path(self, mock_config, mock_presets): """Test build with absolute folder path from user.""" @@ -996,7 +996,7 @@ class TestYTDLPCli: ) assert "--paths" in command - @patch("app.library.Presets.Presets") + @patch("app.features.presets.service.Presets") @patch("app.library.YTDLPOpts.Config") def test_build_includes_url_in_command(self, mock_config, mock_presets): """Test that build includes the URL in the final command.""" @@ -1018,12 +1018,12 @@ class TestYTDLPCli: assert "https://youtube.com/watch?v=test123" in command - @patch("app.library.Presets.Presets") + @patch("app.features.presets.service.Presets") @patch("app.library.YTDLPOpts.Config") def test_build_cli_args_priority_order(self, mock_config, mock_presets): """Test that CLI args are added in correct priority order (preset first, user last).""" from app.library.ItemDTO import Item - from app.library.Presets import Preset + from app.features.presets.schemas import Preset from app.library.YTDLPOpts import YTDLPCli mock_config_instance = Mock() diff --git a/pyproject.toml b/pyproject.toml index 80549294..398d8c23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ dependencies = [ # Non‐Windows "python-magic>=0.4.27; sys_platform != 'win32'", # Cross‐platform - "python-socketio>=5.11.1", "aiohttp>=3.9.3", "coloredlogs>=15.0.1", "aiocron>=1.8", @@ -44,6 +43,8 @@ dependencies = [ "jmespath>=1.0.1", "jsonschema>=4.23.0", "aiosqlite>=0.22.0", + "sqlalchemy>=2.0.45", + "pydantic>=2.12.5", ] [tool.ruff] @@ -161,6 +162,7 @@ ignore = [ "PT011", "RUF001", "S311", + "TC001" ] fixable = ["ALL"] @@ -181,10 +183,10 @@ link-mode = "copy" [tool.pytest.ini_options] pythonpath = ["."] -testpaths = ["app/tests"] +testpaths = ["app/tests", "app/features"] addopts = "-v --tb=short" filterwarnings = [ - "ignore:Parsing dates involving a day of month without a year:DeprecationWarning", + "ignore:Parsing dates involving a day of month without a year:DeprecationWarning" ] [dependency-groups] diff --git a/ui/app/components/ConditionForm.vue b/ui/app/components/ConditionForm.vue index dfe0bd2c..aafda99c 100644 --- a/ui/app/components/ConditionForm.vue +++ b/ui/app/components/ConditionForm.vue @@ -2,7 +2,7 @@
-
+
@@ -21,7 +21,7 @@
-
+
@@ -335,17 +335,18 @@ import 'assets/css/bulma-switch.css' import { useStorage } from '@vueuse/core' import TextareaAutocomplete from '~/components/TextareaAutocomplete.vue' import type { AutoCompleteOptions } from '~/types/autocomplete'; -import type { ConditionItem, ImportedConditionItem } from '~/types/conditions' +import type { Condition } from '~/types/conditions' import { useConfirm } from '~/composables/useConfirm' +import type { ImportedItem } from '~/types'; const emitter = defineEmits<{ (e: 'cancel'): void - (e: 'submit', payload: { reference: string | null | undefined, item: ConditionItem }): void + (e: 'submit', payload: { reference: number | null | undefined, item: Condition }): void }>() const props = defineProps<{ - reference?: string | null - item: ConditionItem + reference?: number | null + item: Condition addInProgress?: boolean }>() @@ -354,7 +355,7 @@ const showImport = useStorage('showImport', false) const box = useConfirm() const config = useConfigStore() -const form = reactive(JSON.parse(JSON.stringify(props.item))) +const form = reactive(JSON.parse(JSON.stringify(props.item))) const import_string = ref('') const newExtraKey = ref('') const newExtraValue = ref('') @@ -394,7 +395,7 @@ watch(() => config.ytdlp_options, newOptions => ytDlpOpt.value = newOptions watch(() => form.filter, () => test_data.value.changed = true) const checkInfo = async (): Promise => { - const required: (keyof ConditionItem)[] = ['name', 'filter'] + const required: (keyof Condition)[] = ['name', 'filter'] for (const key of required) { if (!form[key]) { @@ -416,7 +417,7 @@ const checkInfo = async (): Promise => { form.cli = form.cli.trim() } - const copy: ConditionItem = JSON.parse(JSON.stringify(form)) + const copy: Condition = JSON.parse(JSON.stringify(form)) for (const key in copy) { if (typeof (copy as any)[key] !== 'string') { @@ -483,9 +484,9 @@ const importItem = async (): Promise => { } try { - const item = decode(val) as ImportedConditionItem + const item = decode(val) as Condition & ImportedItem - if (!item?._type || item._type !== 'condition') { + if (!item._type || item._type !== 'condition') { toast.error(`Invalid import string. Expected type 'condition', got '${item._type ?? 'unknown'}'.`) return } diff --git a/ui/app/components/Connection.vue b/ui/app/components/Connection.vue index 2aa7587e..5ff63e0b 100644 --- a/ui/app/components/Connection.vue +++ b/ui/app/components/Connection.vue @@ -1,10 +1,14 @@