Compare commits

..

No commits in common. "master" and "v1.2.5" have entirely different histories.

332 changed files with 65889 additions and 48731 deletions

View file

@ -15,7 +15,7 @@ on:
env:
BUN_VERSION: latest
NODE_VERSION: 22
NODE_VERSION: 20
PYTHON_VERSION: "3.13"
jobs:

View file

@ -29,7 +29,7 @@ env:
DOCKERHUB_SLUG: arabcoders/ytptube
GHCR_SLUG: ghcr.io/arabcoders/ytptube
BUN_VERSION: latest
NODE_VERSION: 22
NODE_VERSION: 20
PYTHON_VERSION: "3.13"
jobs:

View file

@ -37,9 +37,9 @@ jobs:
contents: write
env:
PYTHON_VERSION: "3.13"
PYTHON_VERSION: 3.11
BUN_VERSION: latest
NODE_VERSION: 22
NODE_VERSION: 20
TAG_NAME: ${{ github.event.inputs.tag || github.ref_name }}
steps:
@ -63,37 +63,6 @@ jobs:
shell: bash
run: cp temp-spec/app.spec ./app.spec
- name: Update app/library/version.py
shell: bash
run: |
VERSION="${TAG_NAME}"
SHA=$(git rev-parse HEAD)
DATE=$(date -u +"%Y%m%d")
BRANCH=$(git branch --show-current)
if [ -z "${BRANCH}" ]; then
BRANCH="${TAG_NAME}"
fi
BRANCH=$(echo "${BRANCH}" | sed 's#/#-#g')
echo "APP_VERSION=${VERSION}" >> "$GITHUB_ENV"
echo "APP_SHA=${SHA}" >> "$GITHUB_ENV"
echo "APP_DATE=${DATE}" >> "$GITHUB_ENV"
echo "APP_BRANCH=${BRANCH}" >> "$GITHUB_ENV"
sed -i.bak \
-e "s/^APP_VERSION = \".*\"/APP_VERSION = \"${VERSION}\"/" \
-e "s/^APP_COMMIT_SHA = \".*\"/APP_COMMIT_SHA = \"${SHA}\"/" \
-e "s/^APP_BUILD_DATE = \".*\"/APP_BUILD_DATE = \"${DATE}\"/" \
-e "s/^APP_BRANCH = \".*\"/APP_BRANCH = \"${BRANCH}\"/" \
app/library/version.py
rm -f app/library/version.py.bak
echo "Updated version info:"
cat app/library/version.py
- name: Cache Python venv
id: cache-python
uses: actions/cache@v4
@ -113,9 +82,7 @@ jobs:
shell: bash
run: |
pip install uv
if [ ! -d .venv ]; then
uv venv --system-site-packages --relocatable
fi
uv venv --system-site-packages --relocatable
uv sync --link-mode=copy --active --extra installer
- name: Cache bun installation
@ -194,7 +161,7 @@ jobs:
rm -rf "dist/${PKG_DIR}"
mv "dist/YTPTube" "dist/${PKG_DIR}"
zip -yr "release/${ZIP_NAME}" "dist/${PKG_DIR}"
zip -r "release/${ZIP_NAME}" "dist/${PKG_DIR}"
- name: Package artifact (Windows)
if: startsWith(env.TAG_NAME, 'v') && runner.os == 'Windows'

6
.vscode/launch.json vendored
View file

@ -2,7 +2,7 @@
"version": "0.2.0",
"configurations": [
{
"name": "UI: Dev",
"name": "Node: Nuxt3",
"request": "launch",
"runtimeArgs": [
"run",
@ -21,7 +21,7 @@
"outputCapture": "std"
},
{
"name": "Python: main.py",
"name": "Python: main.py ",
"type": "debugpy",
"request": "launch",
"program": "app/main.py",
@ -38,7 +38,7 @@
},
},
{
"name": "UI: Generate",
"name": "Node: Generate UI",
"request": "launch",
"runtimeArgs": [
"run",

594
API.md
View file

@ -24,8 +24,6 @@ This document describes the available endpoints and their usage. All endpoints r
- [DELETE /api/history](#delete-apihistory)
- [POST /api/history/{id}](#post-apihistoryid)
- [GET /api/history/{id}](#get-apihistoryid)
- [GET /api/history/{id}/thumbnail](#get-apihistoryidthumbnail)
- [POST /api/history/{id}/rename](#post-apihistoryidrename)
- [GET /api/history](#get-apihistory)
- [GET /api/history/live](#get-apihistorylive)
- [POST /api/history/start](#post-apihistorystart)
@ -57,8 +55,7 @@ This document describes the available endpoints and their usage. All endpoints r
- [GET /api/player/m3u8/{mode}/{file:.\*}.m3u8](#get-apiplayerm3u8modefilem3u8)
- [GET /api/player/segments/{segment}/{file:.\*}.ts](#get-apiplayersegmentssegmentfilets)
- [GET /api/player/subtitle/{file:.\*}.vtt](#get-apiplayersubtitlefilevtt)
- [GET /api/player/subtitles/manifest/{file:.\*}](#get-apiplayersubtitlesmanifestfile)
- [GET /api/player/subtitles/{source\_format}/{file:.\*}](#get-apiplayersubtitlessource_formatfile)
- [GET /api/thumbnail](#get-apithumbnail)
- [GET /api/file/ffprobe/{file:.\*}](#get-apifileffprobefile)
- [GET /api/file/info/{file:.\*}](#get-apifileinfofile)
- [GET /api/file/browser/{path:.\*}](#get-apifilebrowserpath)
@ -88,8 +85,6 @@ This document describes the available endpoints and their usage. All endpoints r
- [DELETE /api/conditions/{id}](#delete-apiconditionsid)
- [GET /api/logs](#get-apilogs)
- [GET /api/logs/stream](#get-apilogsstream)
- [GET /api/logs/level](#get-apilogslevel)
- [POST /api/logs/level/{level}](#post-apilogslevellevel)
- [GET /api/notifications/](#get-apinotifications)
- [GET /api/notifications/events/](#get-apinotificationsevents)
- [POST /api/notifications/](#post-apinotifications)
@ -101,15 +96,7 @@ This document describes the available endpoints and their usage. All endpoints r
- [POST /api/notifications/test](#post-apinotificationstest)
- [GET /api/yt-dlp/options](#get-apiyt-dlpoptions)
- [GET /api/system/configuration](#get-apisystemconfiguration)
- [GET /api/system/folders](#get-apisystemfolders)
- [GET /api/system/diagnostics](#get-apisystemdiagnostics)
- [GET /api/system/limits](#get-apisystemlimits)
- [POST /api/system/terminal](#post-apisystemterminal)
- [GET /api/system/terminal](#get-apisystemterminal)
- [GET /api/system/terminal/active](#get-apisystemterminalactive)
- [GET /api/system/terminal/{session\_id}](#get-apisystemterminalsession_id)
- [DELETE /api/system/terminal/{session\_id}](#delete-apisystemterminalsession_id)
- [GET /api/system/terminal/{session\_id}/stream](#get-apisystemterminalsession_idstream)
- [POST /api/system/pause](#post-apisystempause)
- [POST /api/system/resume](#post-apisystemresume)
- [POST /api/system/shutdown](#post-apisystemshutdown)
@ -141,7 +128,6 @@ This document describes the available endpoints and their usage. All endpoints r
- [`item_updated`](#item_updated)
- [`item_cancelled`](#item_cancelled)
- [`item_deleted`](#item_deleted)
- [`item_bulk_deleted`](#item_bulk_deleted)
- [`item_moved`](#item_moved)
- [`item_status`](#item_status)
- [`paused`](#paused)
@ -356,10 +342,7 @@ or an error:
"cookies": "...", // -- optional. If provided, it MUST BE in Netscape HTTP Cookie format.
"template": "%(title)s.%(ext)s", // -- optional. The filename template to use for this item.
"cli": "--write-subs --embed-subs", // -- optional. Additional command options for yt-dlp to apply to this item.
"auto_start": true, // -- optional. Whether to auto-start the download after adding it. Defaults to true.
"extras": {
"ignore_conditions": ["123", "My Condition"] // -- optional. Skip matching conditions by id string or exact name for this request only. Use ["*"] to ignore all conditions.
}
"auto_start": true // -- optional. Whether to auto-start the download after adding it. Defaults to true.
}
// Or multiple items (array of objects)
@ -394,7 +377,7 @@ or an error:
**Body Parameters**:
- `type` (string, required): Type of items - `"queue"` or `"done"`
- `ids` (array, optional): List of specific item IDs to delete. If provided, `status` filter is ignored
- `status` (string, optional): Filter by status (e.g., `"finished"`, `"!finished"`, `"finished,skip"`, `"!finished,!skip"`). Required if `ids` not provided
- `status` (string, optional): Filter by status (e.g., `"finished"`, `"!finished"`). Required if `ids` not provided
- `remove_file` (boolean, optional): Whether to delete files from disk. Default: `true`.
> [!NOTE]
@ -429,15 +412,6 @@ or an error:
}
```
**Delete all completed and skipped items in one request:**
```json
{
"type": "done",
"status": "finished,skip",
"remove_file": false
}
```
**Response**:
```json
{
@ -470,7 +444,6 @@ or an error:
**Notes**:
- When using filter mode, all matching items will be deleted.
- Filter mode with `{ "status": "!finished" }` is useful for cleaning up failed/pending downloads.
- `status` also accepts comma-separated include filters (`finished,skip`) and comma-separated exclude filters (`!finished,!skip`).
- Filter mode returns a `deleted` count indicating how many items were removed.
---
@ -537,61 +510,6 @@ or an error:
---
### GET /api/history/{id}/thumbnail
**Purpose**: Return thumbnail for a downloaded item.
**Path Parameter**:
- `id` = item ID.
**Behavior**:
- Returns an existing local sidecar or artwork image when available.
- Otherwise generates a representative frame thumbnail with `ffmpeg` and caches it if enabled.
**Response**:
- `200 OK` with an image file response.
- `404 Not Found` if the item, downloaded file, or local thumbnail is not available.
- `400 Bad Request` if `id` is missing.
**Notes**:
- Audio-only files return `404` unless a local image sidecar already exists.
---
### POST /api/history/{id}/rename
**Purpose**: Rename a downloaded history file and its sidecars.
**Path Parameter**:
- `id` = Unique history item ID.
**Body**:
```json
{
"new_name": "renamed_video.mp4"
}
```
**Response**:
```json
{
"_id": "<uuid>",
"title": "Video Title",
"url": "https://youtube.com/watch?v=...",
....
}
```
**Error Responses**:
- `400 Bad Request` if `id` or `new_name` is missing, or the item has no downloaded file.
- `404 Not Found` if the item does not exist.
- `409 Conflict` if the rename destination already exists.
- `500 Internal Server Error` if the filesystem rename fails unexpectedly.
**Notes**:
- Uses the same file rename behavior as the file browser actions.
- Renames matching sidecar files together with the main media file.
---
### GET /api/history
**Purpose**: Returns the download queue and/or download history with optional pagination support.
@ -700,16 +618,10 @@ GET /api/history?type=queue&status=pending&order=ASC
This endpoint returns the current state of active downloads from memory.
**Query Parameters**:
- `limit` (optional): Override the configured queue display limit for this request. `0` means unlimited.
**Response**:
```json
{
"history_count": 0, // total number of completed items in history
"queue_count": 250, // total number of queued items
"queue_loaded": 100, // number of queued items included in this response
"queue_limit": 100, // effective display limit, 0 means unlimited
"queue":{
"id": "abc123",
"url": "https://example.com/video",
@ -1619,48 +1531,14 @@ Binary TS data (`Content-Type: video/mpegts`).
---
### GET /api/player/subtitles/manifest/{file:.*}
**Purpose**: Returns subtitle track metadata for a local media file.
### GET /api/thumbnail
**Purpose**: Proxy/fetch a remote thumbnail image.
**Path Parameter**:
- `file` = Relative path of the media file within the `download_path`.
**Query Parameter**:
- `?url=<remote-thumbnail-url>`
**Response**:
```json
{
"subtitles": [
{
"lang": "en",
"name": "VTT (0) - en",
"source_format": "vtt",
"delivery_format": "vtt",
"renderer": "native",
"url": "/api/player/subtitles/vtt/path/to/video.en.vtt"
},
{
"lang": "en",
"name": "ASS (1) - en",
"source_format": "ass",
"delivery_format": "ass",
"renderer": "assjs",
"url": "/api/player/subtitles/ass/path/to/video.en.ass"
}
]
}
```
---
### GET /api/player/subtitles/{source_format}/{file:.*}
**Purpose**: Delivers a subtitle file using its preferred playback format.
**Path Parameters**:
- `source_format` = `vtt`, `srt`, or `ass`.
- `file` = Relative path of the subtitle file.
**Response**:
- `text/vtt; charset=UTF-8` for `vtt` and `srt` sources.
- `text/x-ssa; charset=UTF-8` for `ass` sources.
**Response**:
Binary image data with the appropriate `Content-Type`.
---
@ -1696,16 +1574,21 @@ Binary TS data (`Content-Type: video/mpegts`).
},
"mimetype": "video/mp4",
"sidecar": {
"subtitle": [
"subtitles": [
{
"file": "filename.xxx.ass",
"file": "filename.xxx.vtt",
"lang": "xxx",
"name": "ASS (0) - xxx"
"name": "VTT 0 - XXX|end",
},
...
}
],
"video": [],
"audio": [],
"image": [],
"text": []
"text": [],
"metadata": [],
...
}
}
```
@ -1888,9 +1771,6 @@ Binary image data with appropriate headers
**Query Parameters**:
- `page` (optional): Page number (1-indexed). Default: `1`.
- `per_page` (optional): Items per page. Default: `config.default_pagination`.
- `sort` (optional): Comma-separated sort fields. Accepted values: `id`, `name`, `priority`, `default`, `created_at`, `updated_at`. Default: `priority,name`.
- `order` (optional): Comma-separated sort directions matching `sort`, or a single direction applied to every requested sort field. Accepted values: `asc`, `desc`. Default: `desc,asc`.
- `exclude_defaults` (optional): When `true`, excludes system presets from the results. Default: `false`.
**Response**:
```json
@ -1921,9 +1801,6 @@ Binary image data with appropriate headers
**Notes**:
- `default: true` indicates this is a system default preset (cannot be modified or deleted)
**Error Responses**:
- `400 Bad Request` - Invalid data was provided.
---
### GET /api/presets/{id}
@ -2323,46 +2200,12 @@ Binary image data with appropriate headers
{
"logs": [
{
"id": "<uuid>",
"datetime": "2026-05-18T12:00:00.000+00:00",
"level": "error",
"levelno": 40,
"logger": "ytptube",
"message": "Failed to download 'Example Video'.",
"exception": {
"type": "ValueError",
"message": "bad",
"file": "/app/library/downloads/queue_manager.py",
"line": 123,
"stack": [
{
"path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py",
"module": "queue_manager",
"function": "start",
"line": 123
}
]
},
"source": {
"path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py",
"module": "queue_manager",
"function": "start",
"line": 123
},
"fields": {
"download": {
"download_id": "abc123",
"media_id": "video-id",
"title": "Example Video",
"url": "https://example.test/video",
"preset": "default",
"status": "error",
"has_cookies": false
}
}
}
"timestamp": "2023-01-01T12:00:00Z",
"level": "INFO",
"message": "...",
...
},
...
],
"offset": 0,
"limit": 100,
@ -2384,45 +2227,9 @@ Binary image data with appropriate headers
**Event Payload**:
```json
{
"id": "<uuid>",
"datetime": "2026-05-18T12:00:00.000+00:00",
"level": "error",
"levelno": 40,
"logger": "ytptube",
"message": "Failed to download 'Example Video'.",
"exception": {
"type": "ValueError",
"message": "bad",
"file": "/app/library/downloads/queue_manager.py",
"line": 123,
"stack": [
{
"path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py",
"module": "queue_manager",
"function": "start",
"line": 123
}
]
},
"source": {
"path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py",
"module": "queue_manager",
"function": "start",
"line": 123
},
"fields": {
"download": {
"download_id": "abc123",
"media_id": "video-id",
"title": "Example Video",
"url": "https://example.test/video",
"preset": "default",
"status": "error",
"has_cookies": false
}
}
"id": "<sha256>",
"line": "<log line>",
"datetime": "2024-01-01T12:00:00.000000+00:00"
}
```
@ -2430,31 +2237,6 @@ Binary image data with appropriate headers
---
### GET /api/logs/level
**Purpose**: Read the active runtime log level.
**Response**:
```json
{
"conf": "info",
"active": "info",
"levels": ["debug", "info", "warning", "error"]
}
```
---
### POST /api/logs/level/{level}
**Purpose**: Change the active runtime log level.
**Path Parameter**:
- `level`: One of `debug`, `info`, `warning`, `error`.
**Response**:
- `204 No Content` on success.
---
### GET /api/notifications/
**Purpose**: Retrieve notification targets with pagination.
@ -2635,139 +2417,60 @@ or an error:
---
### GET /api/system/configuration
**Purpose**: Retrieve system configuration.
**Purpose**: Retrieve comprehensive system configuration including app settings, presets, download fields, queue status, and folder structure.
**Response**:
```json
{
"app": {...},
"presets": [...],
"dl_fields": [...],
"app": {
"version": "...",
"download_path": "/path/to/downloads",
"base_path": "/",
...
},
"presets": [
{
"id": 1,
"name": "default",
"description": "...",
...
}
],
"dl_fields": [
{
"id": 1,
"name": "Title",
"field": "title",
"kind": "text",
...
}
],
"paused": false,
"history_count": 150
}
```
---
### GET /api/system/folders
**Purpose**: List child directories for a given relative path within the download directory.
**Query Parameters**:
- `path=<relative-path>` (optional, default: root) - Relative path within the download directory.
**Response**:
```json
{
"path": "videos",
"folders": ["archive", "shorts", "2024"]
}
```
**Notes**:
- Results are cached server-side for a short time.
- Non-existent paths return an empty folder list.
---
### GET /api/system/diagnostics
**Purpose**: View system information.
**Response**:
```json
{
"status": "error",
"generated_at": 1713000000,
"summary": {
"total": 10,
"pass": 5,
"fail": 2,
"warn": 1,
"skip": 2,
"required_failed": 2
},
"runtime": {
"app_version": "1.0.0",
"app_branch": "main",
"app_commit_sha": "abcdef12",
"app_build_date": "20260526",
"started": 1712999900,
"uptime_seconds": 100,
"platform": "linux",
"platform_release": "6.8.0",
"platform_machine": "x86_64",
"python_version": "3.13.1",
"python_minimum": "3.13",
"is_native": false,
"console_enabled": false
},
"requirements": {
"python": {
"current": "3.13.1",
"required": "3.13",
"supported": true,
"note": ""
"folders": [
{"name": "folder1", "path": "folder1"},
{"name": "folder2", "path": "folder2"}
],
"history_count": 150,
"queue": [
{
"id": "abc123",
"url": "https://example.com/video",
"status": "pending",
...
}
},
"checks": []
]
}
```
**Notes**:
- Unexpected collection errors are returned as an `error`.
---
### GET /api/system/limits
**Purpose**: Get the system limits.
**Response**:
```json
{
"downloads": {
"paused": false,
"global": {
"limit": 20,
"active": 3,
"available": 17,
"live_active": 1,
"queued": 8
},
"per_extractor": {
"default_limit": 2,
"items": [
{
"name": "youtube",
"limit": 3,
"source": "env_override",
"active": 2,
"queued": 4,
"available": 1
}
]
}
},
"extraction": {
"concurrency": 4,
"timeout_seconds": 70,
"info_cache_ttl_seconds": 10800
},
"live": {
"prevent_premiere": true,
"premiere_buffer_minutes": 5
}
}
```
**Notes**:
- `downloads.global` counts only non-live downloads against the worker limit.
- `downloads.global.live_active` is reported separately because live downloads bypass the global and per-extractor worker limits.
- `downloads.per_extractor.items[*].source` is `default` unless an override was provided through `YTP_MAX_WORKERS_FOR_<EXTRACTOR>`.
- This endpoint combines multiple data sources into a single response for efficient initialization
- The `folders` array includes available download folders up to the configured depth limit
- The `queue` array contains active download items
---
### POST /api/system/terminal
**Purpose**: Start a yt-dlp terminal session. Requires `YTP_CONSOLE_ENABLED=true`.
**Purpose**: Stream yt-dlp CLI output via Server-Sent Events (SSE). Requires `YTP_CONSOLE_ENABLED=true`.
**Body**:
```json
@ -2776,147 +2479,9 @@ or an error:
}
```
**Response**:
```json
{
"session_id": "3a8c5f7e2d3b4a8f9c0d1e2f3a4b5c6d",
"command": "--help",
"status": "starting",
"created_at": 1713000000.0,
"started_at": 1713000000.0,
"finished_at": null,
"expires_at": null,
"exit_code": null,
"last_sequence": 0
}
```
**Notes**:
- Starts the command in an application-owned background task.
- Only one active terminal session is allowed at a time.
- The command continues running if the frontend disconnects or reloads.
- `403 Forbidden` if console is disabled.
- `400 Bad Request` if the request body is invalid.
- `409 Conflict` if another terminal session is already active.
---
### GET /api/system/terminal
**Purpose**: List recent terminal sessions.
**Response**:
```json
{
"items": [
{
"session_id": "3a8c5f7e2d3b4a8f9c0d1e2f3a4b5c6d",
"command": "--help",
"status": "completed",
"created_at": 1713000000.0,
"started_at": 1713000000.0,
"finished_at": 1713000004.0,
"expires_at": 1713086404.0,
"available_until": 1713086404.0,
"exit_code": 0,
"last_sequence": 15
}
]
}
```
**Notes**:
- Expired sessions are removed automatically later.
- `403 Forbidden` if console is disabled.
---
### GET /api/system/terminal/active
**Purpose**: Return the currently active terminal session metadata, or `null` if no session is active.
**Response**:
```json
null
```
or
```json
{
"session_id": "3a8c5f7e2d3b4a8f9c0d1e2f3a4b5c6d",
"command": "--help",
"status": "running",
"created_at": 1713000000.0,
"started_at": 1713000000.0,
"finished_at": null,
"expires_at": null,
"exit_code": null,
"last_sequence": 12
}
```
- `403 Forbidden` if console is disabled.
---
### GET /api/system/terminal/{session_id}
**Purpose**: Return metadata for a specific terminal session.
**Response**:
```json
{
"session_id": "3a8c5f7e2d3b4a8f9c0d1e2f3a4b5c6d",
"command": "--help",
"status": "completed",
"created_at": 1713000000.0,
"started_at": 1713000000.0,
"finished_at": 1713000004.0,
"expires_at": 1713086404.0,
"exit_code": 0,
"last_sequence": 15
}
```
- `403 Forbidden` if console is disabled.
- `404 Not Found` if the session does not exist or has already expired.
---
### DELETE /api/system/terminal/{session_id}
**Purpose**: Request cancellation for the active terminal session.
**Response**:
```json
{
"message": "Terminal session cancellation requested.",
"session_id": "3a8c5f7e2d3b4a8f9c0d1e2f3a4b5c6d"
}
```
**Notes**:
- This only applies to the currently active session.
- Cancelled sessions are marked as `interrupted`.
- `403 Forbidden` if console is disabled.
- `404 Not Found` if the session does not exist or has already expired.
- `409 Conflict` if the session exists but is no longer active.
---
### GET /api/system/terminal/{session_id}/stream
**Purpose**: Replay a terminal session transcript over SSE and tail live output when the session is still running.
**Query Parameters**:
- `since` (optional): Resume after the provided integer event id.
**Headers**:
- `Last-Event-ID` (optional): Resume after the provided integer event id.
If both `since` and `Last-Event-ID` are present, the larger value is used.
**Response**:
- `Content-Type: text/event-stream`
- Replays transcript events with monotonic integer SSE `id` values.
- Emits `output` events for stdout/stderr and a final `close` event when available.
- Emits `output` events for stdout/stderr and a final `close` event when the process exits.
**Event Payloads**:
```json
@ -2927,8 +2492,7 @@ If both `since` and `Last-Event-ID` are present, the larger value is used.
```
- `403 Forbidden` if console is disabled.
- `400 Bad Request` if the replay cursor is invalid.
- `404 Not Found` if the session does not exist or has already expired.
- `400 Bad Request` if the request body is invalid.
---
@ -3439,24 +3003,6 @@ Emitted when a download item is deleted from the queue or history.
---
##### `item_bulk_deleted`
Emitted when multiple items are cleared in bulk operation.
**Event**:
```json
{
"event": "item_bulk_deleted",
"data": {
"count": 10001,
"status": "finished,skip",
"ids": ["id1", "id2", "..."] // optional, included when clear is driven by explicit ids
}
}
```
---
##### `item_moved`
Emitted when a download item is moved between queue and history.
@ -3468,7 +3014,7 @@ Emitted when a download item is moved between queue and history.
"data": {
"id": "abc123",
"from": "queue",
"to": "history"
"to": "done"
}
}
```

189
FAQ.md
View file

@ -3,9 +3,6 @@
Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line,
or the `environment:` section in `compose.yaml` file.
<details>
<summary>Click to expand</summary>
| Environment Variable | Description | Default |
| ------------------------------- | ------------------------------------------------------------------- | --------------------- |
| TZ | The timezone to use for the application | `(not_set)` |
@ -33,6 +30,7 @@ or the `environment:` section in `compose.yaml` file.
| YTP_DEBUG | Whether to turn on debug mode | `false` |
| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` |
| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` |
| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` |
| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `(not_set)` |
| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` |
| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use | `(not_set)` |
@ -40,17 +38,16 @@ or the `environment:` section in `compose.yaml` file.
| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` |
| YTP_YTDLP_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` |
| YTP_YTDLP_VERSION | The version of yt-dlp to use. Defaults to latest version | `(not_set)` |
| YTP_BROWSER_URL | Remote browser endpoint for the bundled `browser` extractor | `(not_set)` |
| YTP_FLARESOLVERR_URL | FlareSolverr endpoint URL. | `(not_set)` |
| YTP_FLARESOLVERR_MAX_TIMEOUT | Max FlareSolverr challenge timeout in seconds | `120` |
| YTP_FLARESOLVERR_CLIENT_TIMEOUT | HTTP client timeout (seconds) when calling FlareSolverr | `120` |
| YTP_FLARESOLVERR_CACHE_TTL | The cache TTL (in seconds) for FlareSolverr solutions | `600` |
| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` |
| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` |
| YTP_QUEUE_DISPLAY_LIMIT | Max queued downloads returned to the UI. `0` = unlimited | `100` |
| YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` |
| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` |
| YTP_TEMP_DISABLED | Disable temp files handling. | `false` |
| YTP_DOWNLOAD_PATH_DEPTH | How many subdirectories to show in auto complete. | `1` |
| YTP_ALLOW_INTERNAL_URLS | Allow requests to internal URLs | `false` |
| YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` |
| YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` |
@ -60,23 +57,19 @@ or the `environment:` section in `compose.yaml` file.
| YTP_IGNORE_ARCHIVED_ITEMS | Don't report archived items in the download history. | `false` |
| YTP_CHECK_FOR_UPDATES | Whether to check for application updates. | `true` |
| YTP_EXTRACT_INFO_CONCURRENCY | The number of concurrent extract info operations. | `4` |
| YTP_THUMB_CONCURRENCY | The number of concurrent ffmpeg thumbnail generations allowed. | `2` |
| YTP_THUMB_GENERATE | Enable ffmpeg thumbnail generation when no local thumbnail exists. | `true` |
| YTP_THUMB_SIDECAR | Save generated thumbnails next to media instead of temp cache. | `false` |
| YTP_DISABLE_EXEC | Strip some dangerous yt-dlp options. | `false` |
</details>
> [!NOTE]
> To raise the worker limit for a specific extractor, set an env variable using this format: `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`
> The extractor name must be uppercase. You can find the extractor name in the download logs. This value cannot be
> higher than `YTP_MAX_WORKERS`; higher values are ignored.
>
> `YTP_SIMPLE_MODE=true` only applies when the browser has no saved layout choice yet. Users can still choose a layout in
> WebUI Settings. `/?simple=1` forces and saves Simple for that browser.
>
> `YTP_AUTO_CLEAR_HISTORY_DAYS` `0` days means no automatic clearing of the download history. lowest value that will
> trigger the clearing is `1` day. This setting will **NOT** delete the downloaded files, it will only clear the
> history from the database.
> To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`.
> The extractor name must be in uppercase, to know the extractor name, check the log for the specific extractor used for the download.
> The limit should not exceed the `YTP_MAX_WORKERS` value as it will be ignored.
> [!IMPORTANT]
> The env variable `YTP_SIMPLE_MODE` only control what being displayed for first time visitor, the users can still switch between the two modes via the WebUI settings page.
## Notes about YTP_AUTO_CLEAR_HISTORY_DAYS
- `0` days means no automatic clearing of the download history. lowest value that will trigger the clearing is `1` day.
- This setting will **NOT** delete the downloaded files, it will only clear the history from the database.
# Browser extensions & bookmarklets
@ -123,35 +116,6 @@ As this is a simple basic authentication, if your browser doesn't show the promp
`http://username:password@your_ytptube_url:port`
# Security recommendations
YTPTube is designed for LAN and home-lab use behind a firewall or reverse proxy. The web interface and API are
unauthenticated by default because in a trusted network, auth adds friction without meaningful benefit. However,
if you expose YTPTube to the internet directly or via port forwarding **YOU MUST enable authentication**.
### Without auth, anyone who can reach the API can:
- Download arbitrary content through your IP and server.
- Delete or modify your downloaded files and database.
- Run arbitrary `yt-dlp` options, including `--exec`, which executes shell commands inside the container.
This is not a vulnerability, it's the intended design. The `cli` field passes options directly to `yt-dlp`,
a tool that by design can execute commands. Auth is the mechanism that controls who gets to use that power.
**If you expose YTPTube to untrusted networks**, do one of the following:
1. **Enable authentication** — set both `YTP_AUTH_USERNAME` and `YTP_AUTH_PASSWORD`.
2. **Put it behind a reverse proxy** with its own authentication layer (see [Run behind reverse proxy](#run-behind-reverse-proxy)).
3. **Keep it on a private network** with no public exposure.
YTPTube already gates other powerful features behind explicit opt-in: the built-in terminal, file browser actions and internal
URL requests for example. The `cli` field is no different, its power is by design, and access control is your responsibility.
> [!NOTE]
> If you choose to run without authentication but still want to reduce at least some impact, you can set
> `YTP_DISABLE_EXEC=true`. This strips some dangerous options at run time. However, understand that this is not a
> substitute for auth an unauthenticated API is still fully open for all other operations.
# I cant download anything
If you are receiving errors like:
@ -204,28 +168,6 @@ YTP_YTDLP_VERSION=2025.07.21 or master or nightly
Then restart the container to apply the changes.
# Custom output template placeholders
YTPTube supports custom `ytp_*` placeholders in `yt-dlp` output template via the following syntax `%(ytp_*:<args>)s`.
## Currently available extra placeholders are:
- `ytp_random`: random mixed letters and digits,
- `N` A number is required to specify the length of the random string, for example `%(ytp_random:8)s` will generate a random string of 8 characters.
- if the args followed by `:s` it will generate random letters only, if followed by `:d` it will generate random digits only.
## Examples of the custom placeholders in action:
- Template: `%(title)s [%(ytp_random:8)s].%(ext)s`
- Example result: `My Video [A7k2Pq9Z].mp4`
- Template: `%(uploader)s/%(ytp_random:6:d)s - %(title)s.%(ext)s`
- Example result: `MyChannel/483920 - My Video.mp4`
- Template: `%(playlist)s/%(ytp_random:10:s)s/%(title)s.%(ext)s`
- Example result: `Favorites/QwErTyUiOp/My Video.mp4`
> [!NOTE]
> `%(ytp_` placeholders are a YTPTube extension and not avaliable via console or directly via yt-dlp.
# How can I monitor sites without RSS feeds?
YTPTube includes a **generic task handler** that turns JSON definitions into site-specific scrapers. You can use it
@ -651,108 +593,3 @@ services:
```
For more information please visit [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) project.
# How to use the browser extractor?
YTPTube ships with a bundled `browser` extractor plugin for `yt-dlp`. It opens the target page in a remote browser,
waits for the page network activity, then tries to extract media urls from the captured requests and media elements.
This is mainly useful for sites where the regular extractor or generic page parsing does not expose the final media url,
but a real browser session does.
## Environment variables
You can define the remote browser endpoint globally with the following environment variable.
### Selenium example
```env
YTP_BROWSER_URL=selenium+http://selenium:4444/wd/hub
```
### Playwright example
For a native Playwright server, point to the Playwright websocket endpoint:
```env
YTP_BROWSER_URL=playwright+ws://playwright:3000/
```
### Playwright CDP example
To connect Playwright over the Chrome DevTools Protocol, use either the shorthand `playwright+cdp://` form or an explicit transport form:
```env
YTP_BROWSER_URL=playwright+cdp://chrome:9222/
# same as playwright+cdp+http://chrome:9222/
```
- `YTP_BROWSER_URL` is the full backend selector and endpoint in one value.
- Supported forms are:
- `selenium+http://...`
- `selenium+https://...`
- `playwright+ws://...`
- `playwright+wss://...`
- `playwright+cdp://...` which is treated as `playwright+cdp+http://...`
- `playwright+cdp+http://...`
- `playwright+cdp+https://...`
- `playwright+cdp+ws://...`
- `playwright+cdp+wss://...`
## yt-dlp usage
If you want to set the browser extractor options directly on the yt-dlp side, you can also use `--extractor-args` with `generic:url=...`:
### Selenium example
```bash
--use-extractors "generic" --extractor-args "generic:url=selenium+http://selenium:4444/wd/hub"
```
### Playwright example
```bash
--use-extractors "generic" --extractor-args "generic:url=playwright+ws://playwright:3000/"
```
### Playwright CDP example
```bash
--use-extractors "generic" --extractor-args "generic:url=playwright+cdp://chrome:9222/"
```
The explicit `--extractor-args` value takes priority over `YTP_BROWSER_URL`.
## Example compose setup
```yaml
services:
ytptube:
user: "${UID:-1000}:${UID:-1000}"
image: ghcr.io/arabcoders/ytptube:latest
container_name: ytptube
restart: unless-stopped
environment:
- YTP_BROWSER_URL=selenium+http://selenium:4444/wd/hub # or playwright+ws://playwright:3000/ or playwright+cdp://chrome:9222/
ports:
- "8081:8081"
volumes:
- ./config:/config:rw
- ./downloads:/downloads:rw
selenium:
image: selenium/standalone-chrome:latest
container_name: selenium
restart: unless-stopped
shm_size: 2gb
playwright:
image: mcr.microsoft.com/playwright:v1.59.0-noble
container_name: playwright
restart: unless-stopped
command: /bin/sh -c "npx -y playwright@1.59.0 run-server --port 3000 --host 0.0.0.0"
```
> [!NOTE]
> The browser extractor is slower than the normal extractor flow and should only be used when a site actually needs a real browser session.
> playwright require same version for both the server and the client, so make sure to use the same version in the container and in your local environment if you want to test it locally.

View file

@ -1,9 +1,8 @@
# YTPTube
![Build Status](https://github.com/arabcoders/ytptube/actions/workflows/main.yml/badge.svg)
![Build Status](https://github.com/ArabCoders/ytptube/actions/workflows/main.yml/badge.svg)
![MIT License](https://img.shields.io/github/license/arabcoders/ytptube.svg)
![Docker Pull](https://img.shields.io/docker/pulls/arabcoders/ytptube.svg)
![gchr Pull](https://ghcr-badge.elias.eu.org/shield/arabcoders/ytptube/ytptube)
![Docker pull](https://ghcr-badge.elias.eu.org/shield/arabcoders/ytptube/ytptube)
**YTPTube** is a web-based GUI for [yt-dlp](https://github.com/yt-dlp/yt-dlp), designed to make downloading videos from
video platforms easier and user-friendly. It supports downloading playlists, channels, live streams and
@ -38,35 +37,26 @@ Example of the Simple mode interface.
* Conditions feature to apply custom options based on `yt-dlp` returned info.
* Custom browser extensions, bookmarklets and iOS shortcuts to send links to YTPTube instance.
* A bundled executable version for Windows, macOS and Linux. `MacOS version is untested`.
* Use playwright or selenium for extractors that require a browser. see [related FAQ](FAQ.md#how-to-use-the-browser-extractor).
Please read the [FAQ](FAQ.md) for more information.
# Installation
> [!IMPORTANT]
> By default YTPTube runs without authentication. If you expose it to the internet, **enable auth**. See [security recommendations](FAQ.md#security-recommendations).
## Run using docker command
```bash
mkdir -p ./{config,downloads/{files,tmp}} && docker run -itd --rm --user "${UID}:${UID}" --name ytptube \
mkdir -p ./{config,downloads/files,downloads/tmp} && docker run -itd --rm --user "${UID}:${UID}" --name ytptube \
-e YTP_TEMP_PATH=/downloads/tmp -e YTP_DOWNLOAD_PATH=/downloads/files \
-p 8081:8081 -v ./config:/config:rw -v ./downloads:/downloads:rw \
ghcr.io/arabcoders/ytptube:latest
```
## Run using podman
```bash
mkdir -p ./{config,downloads/{files,tmp}} && podman run -itd --rm --userns=keep-id --name ytptube \
-e YTP_TEMP_PATH=/downloads/tmp -e YTP_DOWNLOAD_PATH=/downloads/files \
-p 8081:8081 -v ./config:/config:rw -v ./downloads:/downloads:rw \
arabcoders/ytptube:latest
```
Then you can access the WebUI at `http://localhost:8081`.
> [!NOTE]
> If you are using `podman` instead of `docker`, you can use the same command, but you need to change the user to `0:0`
> it will appears to be running as root, but it will run as the user who started the container.
## Using compose file
The following is an example of a `compose.yaml` file that can be used to run YTPTube.
@ -75,8 +65,6 @@ The following is an example of a `compose.yaml` file that can be used to run YTP
services:
ytptube:
user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id.
# comment out the above line and uncomment the below line if you are using podman-compose.
#userns_mode: keep-id
image: ghcr.io/arabcoders/ytptube:latest
container_name: ytptube
restart: unless-stopped
@ -91,14 +79,18 @@ services:
```
> [!IMPORTANT]
> Make sure to change the `user` line to match your user id and group id in docker setups, or use `userns_mode: keep-id` in podman setups.
> Make sure to change the `user` line to match your user id and group id.
```bash
mkdir -p ./{config,downloads/{files,tmp}} && docker compose -f compose.yaml up -d
mkdir -p ./{config,downloads/files,downloads/tmp} && docker compose -f compose.yaml up -d
```
Then you can access the WebUI at `http://localhost:8081`.
> [!NOTE]
> you can use podman-compose instead of docker-compose, as it supports the same syntax. However, you should change the
> user to `0:0` it will appears to be running as root, but it will run as the user who started the container.
## Unraid
For `Unraid` users You can install the `Community Applications` plugin, and search for **ytptube** it comes
@ -112,14 +104,15 @@ For simple API documentation, you can refer to the [API documentation](API.md).
This project is not affiliated with yt-dlp or any other service.
This is a personal project designed to make downloading videos from the internet more convenient for me. It is not
intended for piracy or any unlawful use. This project was built primarily for my own use and preferences.
Its a personal project designed to make downloading videos from the internet more convenient. Its not intended for
piracy or any unlawful use.
AI-assisted tools are used in this project. If you are uncomfortable with this, you should not use this project.
This project was built primarily for my own needs and preferences. The UI might not be the most polished or visually
refined, but Im happy with it as it is. You can, however, create and load your own UI for complete customization. I
plan to refactor the UI/UX in the future using [Nuxt/ui](https://ui.nuxt.com/).
Contributions are welcome, but I may decline changes that do not interest me or do not align with my vision for this
project. Unsolicited pull requests will be closed. For suggestions or feature requests, please open a discussion or
join the Discord server.
Contributions are welcome, but I may decline changes that dont align with my vision for the project. Unsolicited pull
requests may be ignored. For suggestions or feature requests, please open a discussion or join the Discord server.
# Social contact

View file

@ -1,25 +0,0 @@
from __future__ import annotations
import os
import tempfile
from app.tests.helpers import cleanup_test_run_root, get_test_run_root, get_test_system_temp_root
def pytest_configure(config) -> None:
temp_root = get_test_system_temp_root()
for env_name in ("TMPDIR", "TEMP", "TMP"):
os.environ[env_name] = str(temp_root)
tempfile.tempdir = None
if getattr(config.option, "basetemp", None) is None:
config.option.basetemp = str(get_test_run_root() / "pytest")
os.environ["YTP_FILE_LOGGING"] = "false"
def pytest_unconfigure(config) -> None:
del config
os.environ.pop("YTP_FILE_LOGGING", None)
cleanup_test_run_root()

View file

@ -1,18 +1,18 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
from app.features.conditions.models import ConditionModel
from app.features.core.migration import Migration as FeatureMigration
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.conditions.repository import ConditionsRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -36,11 +36,7 @@ class Migration(FeatureMigration):
try:
items: list[dict] | None = json.loads(self._source_file.read_text())
except Exception as exc:
LOG.exception(
"Failed to read conditions migration file '%s'. Ignoring.",
self._source_file,
extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
await self._move_file(self._source_file)
return
@ -58,11 +54,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert condition '%s'.",
normalized.name,
extra={"condition_name": normalized.name, "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert condition '%s': %s", normalized.name, exc)
LOG.info("Migrated %s condition(s) from %s.", inserted, self._source_file)
await self._move_file(self._source_file)

View file

@ -1,50 +1,31 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from app.features.conditions.migration import Migration
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
from contextlib import AbstractAsyncContextManager
from collections.abc import AsyncGenerator, Iterable
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
from sqlalchemy import delete, func, or_, select
from app.features.conditions.models import ConditionModel
from app.features.core.deps import get_session
from app.library.log import get_logger
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> ConditionModel:
model = ConditionModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for ConditionModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: ConditionModel | dict[str, Any]) -> ConditionModel:
if isinstance(payload, ConditionModel):
return payload
return _model_from_payload(payload)
LOG: logging.Logger = logging.getLogger(__name__)
class ConditionsRepository(metaclass=Singleton):
def __init__(self, session: SessionFactory | None = None) -> None:
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: SessionFactory = session or get_session
self.session: AsyncGenerator[AsyncSession] = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -57,7 +38,7 @@ class ConditionsRepository(metaclass=Singleton):
def get_instance() -> ConditionsRepository:
return ConditionsRepository()
async def all(self) -> list[ConditionModel]:
async def list(self) -> list[ConditionModel]:
async with self.session() as session:
result: Result[tuple[ConditionModel]] = await session.execute(
select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc())
@ -112,11 +93,11 @@ class ConditionsRepository(metaclass=Singleton):
result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: ConditionModel | dict[str, Any]) -> ConditionModel:
async def create(self, payload: ConditionModel | dict) -> ConditionModel:
async with self.session() as session:
model = _coerce_model(payload)
model: ConditionModel = ConditionModel(**payload) if isinstance(payload, dict) else payload
if model.id is not None:
model.id = None # ty: ignore
model.id = None
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Condition with name '{model.name}' already exists."
@ -179,11 +160,13 @@ class ConditionsRepository(metaclass=Singleton):
await session.commit()
return model
async def replace_all(self, items: Iterable[dict[str, Any] | ConditionModel]) -> list[ConditionModel]:
async def replace_all(self, items: Iterable[dict | ConditionModel]) -> list[ConditionModel]:
async with self.session() as session:
try:
await session.execute(delete(ConditionModel))
models: list[ConditionModel] = [_coerce_model(item) for item in items]
models: list[ConditionModel] = [
ConditionModel(**item) if isinstance(item, dict) else item for item in items
]
session.add_all(models)
await session.commit()
except Exception:

View file

@ -1,4 +1,4 @@
import asyncio
import logging
from collections import OrderedDict
from typing import Any
@ -14,11 +14,9 @@ from app.library.cache import Cache
from app.library.config import Config
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.router import route
from app.library.Utils import validate_url
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def _model(model: Any) -> Condition:
@ -85,11 +83,6 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf
if not (cond := params.get("condition")):
return web.json_response({"error": "condition is required."}, status=web.HTTPBadRequest.status_code)
try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
except ValueError as e:
return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code)
try:
preset: str = params.get("preset", config.default_preset)
key: str = cache.hash(url + str(preset))
@ -115,44 +108,18 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf
else:
data = cache.get(key)
except Exception as e:
LOG.exception(
"Failed to extract video info for condition check '%s'.",
cond,
extra={
"route": "conditions.match",
"condition": cond,
"url": url,
"preset": preset,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
return web.json_response(
data={"error": f"Failed to extract video info. '{e!s}'"},
status=web.HTTPInternalServerError.status_code,
)
if not isinstance(data, dict):
return web.json_response(
data={"error": "Failed to extract video info."},
status=web.HTTPInternalServerError.status_code,
)
try:
from app.features.ytdlp.mini_filter import match_str
status: bool = match_str(cond, data)
except Exception as e:
LOG.exception(
"Failed to evaluate condition '%s'.",
cond,
extra={
"route": "conditions.match",
"condition": cond,
"url": url,
"preset": preset,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
return web.json_response(
data={"error": str(e)},
status=web.HTTPBadRequest.status_code,

View file

@ -1,40 +1,14 @@
from collections.abc import Iterable
import logging
from aiohttp import web
from app.features.conditions.models import ConditionModel
from app.features.conditions.repository import ConditionsRepository
from app.features.conditions.schemas import Condition
from app.features.ytdlp.mini_filter import match_str
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.Singleton import Singleton
LOG = get_logger()
def _ignored_identifiers(ignore_conditions: Iterable[str | int | float] | None) -> tuple[set[str], bool]:
ignored: set[str] = set()
ignore_all = False
if not ignore_conditions:
return ignored, ignore_all
for value in ignore_conditions:
if isinstance(value, bool) or not isinstance(value, (str, int, float)):
continue
identifier = str(value).strip()
if not identifier:
continue
if "*" == identifier:
ignore_all = True
continue
ignored.add(identifier)
return ignored, ignore_all
LOG: logging.Logger = logging.getLogger("feature.conditions")
class Conditions(metaclass=Singleton):
@ -55,7 +29,7 @@ class Conditions(metaclass=Singleton):
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations")
async def get_all(self) -> list[ConditionModel]:
return await self._repo.all()
return await self._repo.list()
async def save(self, item: ConditionModel | dict) -> ConditionModel:
"""
@ -80,7 +54,7 @@ class Conditions(metaclass=Singleton):
if item.id is None or 0 == item.id:
model = await repo.create(item)
else:
model = await repo.update(item.id, Condition.model_validate(item).model_dump())
model = await repo.update(item.id, item.serialize())
except KeyError as exc:
raise ValueError(str(exc)) from exc
@ -113,15 +87,12 @@ class Conditions(metaclass=Singleton):
repo = self._repo
return await repo.get(identifier)
async def match(
self, info: dict, ignore_conditions: Iterable[str | int | float] | None = None
) -> ConditionModel | None:
async def match(self, info: dict) -> ConditionModel | None:
"""
Check if any condition matches the info dict.
Args:
info (dict): The info dict to check.
ignore_conditions (Iterable[str | int | float] | None): Condition ids or names to skip for this match.
Returns:
Condition|None: The condition if found, None otherwise.
@ -130,12 +101,8 @@ class Conditions(metaclass=Singleton):
if not info or not isinstance(info, dict) or len(info) < 1:
return None
ignored_identifiers, ignore_all = _ignored_identifiers(ignore_conditions)
if ignore_all:
return None
repo = self._repo
items: list[ConditionModel] = await repo.all()
items: list[ConditionModel] = await repo.list()
if len(items) < 1:
return None
@ -143,33 +110,18 @@ class Conditions(metaclass=Singleton):
if not item.enabled:
continue
if str(item.id) in ignored_identifiers or item.name in ignored_identifiers:
continue
if not item.filter:
LOG.error(
"Filter is empty for '%s'.", item.name, extra={"condition_id": item.id, "condition_name": item.name}
)
LOG.error(f"Filter is empty for '{item.name}'.")
continue
try:
if not match_str(item.filter, info):
continue
LOG.debug(
"Matched '%s: %s' with filter '%s'.",
item.id,
item.name,
item.filter,
extra={"condition_id": item.id, "condition_name": item.name, "filter": item.filter},
)
LOG.debug(f"Matched '{item.id}: {item.name}' with filter '{item.filter}'.")
return item
except Exception as e:
LOG.exception(
"Failed to evaluate condition '%s'.",
item.name,
extra={"condition_id": item.id, "condition_name": item.name, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to evaluate '{item.id}: {item.name}'. '{e!s}'.")
continue
return None
@ -189,7 +141,7 @@ class Conditions(metaclass=Singleton):
if not info or not isinstance(info, dict) or len(info) < 1:
return None
if not (item := await self.get(str(identifier))) or not item.enabled or not item.filter:
if not (item := await self.get(identifier)) or not item.enabled or not item.filter:
return None
return item if match_str(item.filter, info) else None

View file

@ -4,18 +4,10 @@ from __future__ import annotations
import pytest
import pytest_asyncio
from types import SimpleNamespace
import pytest
from typing import Any
from aiohttp import web
from aiohttp.test_utils import make_mocked_request
from app.features.conditions.router import conditions_test
from app.library.config import Config
from app.library.encoder import Encoder
from app.features.conditions.models import ConditionModel
from app.features.conditions.repository import ConditionsRepository
from app.library.sqlite_store import SqliteStore
from app.tests.helpers import make_in_memory_db_path
@pytest_asyncio.fixture
@ -23,7 +15,7 @@ async def repo():
ConditionsRepository._reset_singleton()
SqliteStore._reset_singleton()
store = SqliteStore(db_path=make_in_memory_db_path("conditions-repository"))
store = SqliteStore(db_path=":memory:")
await store.get_connection()
# Create repository
@ -32,50 +24,38 @@ async def repo():
yield repository
# Cleanup - close connections properly
await store.close()
if store._conn:
await store._conn.close()
if store._engine:
await store._engine.dispose()
# Reset singletons
ConditionsRepository._reset_singleton()
SqliteStore._reset_singleton()
def _json_request(path: str, payload: object) -> web.Request:
mock_request: Any = make_mocked_request("POST", path)
async def _json() -> object:
return payload
mock_request.json = _json
return mock_request
class TestAllowInternalUrlsScope:
def setup_method(self) -> None:
Config._reset_singleton()
@pytest.mark.asyncio
async def test_rejects_internal_url(self) -> None:
config = Config.get_instance()
config.allow_internal_urls = False
encoder = Encoder()
cache = SimpleNamespace(hash=lambda value: value, has=lambda _key: False, set=lambda **_kwargs: None)
request = _json_request(
"/api/conditions/test/",
{"url": "http://127.0.0.1/test", "condition": "title", "preset": config.default_preset},
)
response = await conditions_test(request, encoder, cache, config)
assert response.status == web.HTTPBadRequest.status_code
body = response.body
assert isinstance(body, bytes)
assert b"internal urls" in body.lower()
class TestConditionsRepository:
"""Test suite for ConditionsRepository database operations."""
@pytest.mark.asyncio
async def test_repository_singleton(self, repo):
"""Verify repository follows singleton pattern."""
instance1 = ConditionsRepository.get_instance()
instance2 = ConditionsRepository.get_instance()
assert instance1 is instance2, "Should return same singleton instance"
@pytest.mark.asyncio
async def test_list_empty(self, repo):
"""List returns empty when no conditions exist."""
conditions = await repo.list()
assert conditions == [], "Should return empty list when no conditions"
@pytest.mark.asyncio
async def test_count_empty(self, repo):
"""Count returns 0 when no conditions exist."""
count = await repo.count()
assert count == 0, "Should return 0 when no conditions exist"
@pytest.mark.asyncio
async def test_create_condition(self, repo):
"""Create condition with valid data."""
@ -209,7 +189,7 @@ class TestConditionsRepository:
await repo.create({"name": "A", "filter": "test", "priority": 1})
await repo.create({"name": "C", "filter": "test", "priority": 2})
items = await repo.all()
items = await repo.list()
assert items[0].name == "C", "Highest priority should be first"
assert items[1].name == "A", "Same priority sorted alphabetically"
@ -241,6 +221,6 @@ class TestConditionsRepository:
assert len(result) == 2, "Should create 2 new conditions"
all_items = await repo.all()
all_items = await repo.list()
assert len(all_items) == 2, "Should only have new conditions"
assert all_items[0].name in ["New 1", "New 2"], "Should only have new items"

View file

@ -1,16 +1,15 @@
from __future__ import annotations
import abc
import logging
import time
from pathlib import Path
from typing import TYPE_CHECKING
from app.library.log import get_logger
if TYPE_CHECKING:
from app.library.config import Config
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(abc.ABC):
@ -27,11 +26,7 @@ class Migration(abc.ABC):
try:
await self.migrate()
except Exception as exc:
LOG.exception(
"Feature migration '%s' failed.",
self.name,
extra={"feature": self.name, "exception_type": type(exc).__name__},
)
LOG.exception("Feature migration '%s' failed: %s", self.name, exc)
return False
return True

View file

@ -3,7 +3,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime as SQLADateTime
from sqlalchemy import Dialect, TypeDecorator
from sqlalchemy import TypeDecorator
from sqlalchemy.orm import DeclarativeBase
@ -20,18 +20,16 @@ class UTCDateTime(TypeDecorator):
impl = SQLADateTime
cache_ok = True
def process_bind_param(self, value: datetime | None, dialect: Dialect) -> datetime | None:
def process_bind_param(self, value: datetime | None, _dialect) -> datetime | None:
"""Convert datetime to UTC before storing."""
_ = dialect
if value is not None:
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC).replace(tzinfo=None)
return value
def process_result_value(self, value: datetime | None, dialect: Dialect) -> datetime | None:
def process_result_value(self, value: datetime | None, _dialect) -> datetime | None:
"""Ensure datetime is timezone-aware (UTC) when loading."""
_ = dialect
if value is not None and value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value

View file

@ -1,18 +1,18 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
from app.features.core.migration import Migration as FeatureMigration
from app.features.dl_fields.schemas import DLField
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.dl_fields.repository import DLFieldsRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -36,11 +36,7 @@ class Migration(FeatureMigration):
try:
items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text())
except Exception as exc:
LOG.exception(
"Failed to read download fields migration file '%s'. Ignoring.",
self._source_file,
extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
await self._move_file(self._source_file)
return
@ -58,11 +54,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert download field '%s'.",
normalized["name"],
extra={"field_name": normalized["name"], "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert dl field '%s': %s", normalized["name"], exc)
LOG.info("Migrated %s dl field(s) from %s.", inserted, self._source_file)
await self._move_file(self._source_file)

View file

@ -1,49 +1,30 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any, cast
import logging
from typing import TYPE_CHECKING, Any
from sqlalchemy import delete, func, or_, select
from app.features.core.deps import get_session
from app.features.dl_fields.migration import Migration
from app.features.dl_fields.models import DLFieldModel
from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
from contextlib import AbstractAsyncContextManager
from collections.abc import AsyncGenerator, Iterable
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> DLFieldModel:
model = DLFieldModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for DLFieldModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
if isinstance(payload, DLFieldModel):
return payload
return _model_from_payload(payload)
LOG: logging.Logger = logging.getLogger(__name__)
class DLFieldsRepository(metaclass=Singleton):
def __init__(self, session: SessionFactory | None = None) -> None:
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: SessionFactory = session or get_session
self.session: AsyncGenerator[AsyncSession] = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -56,7 +37,7 @@ class DLFieldsRepository(metaclass=Singleton):
def get_instance() -> DLFieldsRepository:
return DLFieldsRepository()
async def all(self) -> list[DLFieldModel]:
async def list(self) -> list[DLFieldModel]:
async with self.session() as session:
result: Result[tuple[DLFieldModel]] = await session.execute(
select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc())
@ -109,9 +90,9 @@ class DLFieldsRepository(metaclass=Singleton):
result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
async def create(self, payload: DLFieldModel | dict) -> DLFieldModel:
async with self.session() as session:
model = _coerce_model(payload)
model: DLFieldModel = DLFieldModel(**payload) if isinstance(payload, dict) else payload
if await self.get_by_name(name=model.name) is not None:
msg: str = f"DL field with name '{model.name}' already exists."
@ -172,16 +153,13 @@ class DLFieldsRepository(metaclass=Singleton):
await session.commit()
return model
async def replace_all(self, items: Iterable[dict[str, Any] | DLFieldModel]) -> list[DLFieldModel]:
async def replace_all(self, items: Iterable[dict | DLFieldModel]) -> list[DLFieldModel]:
async with self.session() as session:
try:
await session.execute(delete(DLFieldModel))
models: list[DLFieldModel] = []
for item in items:
if isinstance(item, dict):
models.append(DLFieldModel(**cast("dict[str, Any]", item)))
else:
models.append(item)
models: list[DLFieldModel] = [
DLFieldModel(**item) if isinstance(item, dict) else item for item in items
]
session.add_all(models)
await session.commit()
except Exception:

View file

@ -1,3 +1,4 @@
import logging
from typing import Any
from aiohttp import web
@ -10,10 +11,9 @@ from app.features.dl_fields.schemas import DLField, DLFieldList, DLFieldPatch
from app.features.dl_fields.service import DLFields
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.router import route
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def _model(model: Any) -> DLField:

View file

@ -1,18 +1,18 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from app.features.dl_fields.models import DLFieldModel
from app.features.dl_fields.repository import DLFieldsRepository
from app.features.dl_fields.schemas import DLField
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from aiohttp import web
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("feature.dl_fields")
class DLFields(metaclass=Singleton):
@ -33,10 +33,10 @@ class DLFields(metaclass=Singleton):
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations")
async def get_all(self) -> list[DLFieldModel]:
return await self._repo.all()
return await self._repo.list()
async def get_all_serialized(self) -> list[dict[str, Any]]:
items = await self._repo.all()
items = await self._repo.list()
return [DLField.model_validate(item).model_dump() for item in items]
async def save(self, item: DLField | dict) -> DLFieldModel:

View file

@ -8,24 +8,26 @@ import pytest_asyncio
from app.features.dl_fields.repository import DLFieldsRepository
from app.features.dl_fields.service import DLFields
from app.library.sqlite_store import SqliteStore
from app.tests.helpers import make_in_memory_db_path
@pytest_asyncio.fixture
async def repo():
async def repo(tmp_path):
"""Provide a fresh repository instance with initialized database for each test."""
DLFieldsRepository._reset_singleton()
DLFields._reset_singleton()
SqliteStore._reset_singleton()
store = SqliteStore(db_path=make_in_memory_db_path("dl-fields-service"))
store = SqliteStore(db_path=":memory:")
await store.get_connection()
repository = DLFieldsRepository.get_instance()
yield repository
await store.close()
if store._conn:
await store._conn.close()
if store._engine:
await store._engine.dispose()
DLFieldsRepository._reset_singleton()
DLFields._reset_singleton()

View file

@ -5,25 +5,28 @@ from __future__ import annotations
import pytest
import pytest_asyncio
from app.features.dl_fields.models import DLFieldModel
from app.features.dl_fields.repository import DLFieldsRepository
from app.library.sqlite_store import SqliteStore
from app.tests.helpers import make_in_memory_db_path
@pytest_asyncio.fixture
async def repo():
async def repo(tmp_path):
"""Provide a fresh repository instance with initialized database for each test."""
DLFieldsRepository._reset_singleton()
SqliteStore._reset_singleton()
store = SqliteStore(db_path=make_in_memory_db_path("dl-fields-repository"))
store = SqliteStore(db_path=str(":memory:"))
await store.get_connection()
repository = DLFieldsRepository.get_instance()
yield repository
await store.close()
if store._conn:
await store._conn.close()
if store._engine:
await store._engine.dispose()
DLFieldsRepository._reset_singleton()
SqliteStore._reset_singleton()
@ -32,6 +35,25 @@ async def repo():
class TestDLFieldsRepository:
"""Test suite for DLFieldsRepository database operations."""
@pytest.mark.asyncio
async def test_repository_singleton(self, repo):
"""Verify repository follows singleton pattern."""
instance1 = DLFieldsRepository.get_instance()
instance2 = DLFieldsRepository.get_instance()
assert instance1 is instance2, "Should return same singleton instance"
@pytest.mark.asyncio
async def test_list_empty(self, repo):
"""List returns empty when no fields exist."""
fields = await repo.list()
assert fields == [], "Should return empty list when no fields"
@pytest.mark.asyncio
async def test_count_empty(self, repo):
"""Count returns 0 when no fields exist."""
count = await repo.count()
assert count == 0, "Should return 0 when no fields exist"
@pytest.mark.asyncio
async def test_create_field(self, repo):
"""Create field with valid data."""
@ -204,7 +226,7 @@ class TestDLFieldsRepository:
await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1})
await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0})
items = await repo.all()
items = await repo.list()
assert items[0].name == "c", "Lowest order should be first"
assert items[1].name == "a", "Same order sorted alphabetically"
@ -243,6 +265,6 @@ class TestDLFieldsRepository:
assert len(result) == 2, "Should create 2 new fields"
all_items = await repo.all()
all_items = await repo.list()
assert len(all_items) == 2, "Should only have new fields"
assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items"

View file

@ -1,6 +1,7 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
@ -8,12 +9,11 @@ from app.features.core.migration import Migration as FeatureMigration
from app.features.notifications.schemas import NotificationEvents
from app.features.presets.service import Presets
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.notifications.repository import NotificationsRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -37,11 +37,7 @@ class Migration(FeatureMigration):
try:
items: list[dict] | None = json.loads(self._source_file.read_text())
except Exception as exc:
LOG.exception(
"Failed to read notifications migration file '%s'. Ignoring.",
self._source_file,
extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
await self._move_file(self._source_file)
return
@ -60,11 +56,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert notification target '%s'.",
normalized.get("name"),
extra={"target_name": normalized.get("name"), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert notification '%s': %s", normalized.get("name"), exc)
LOG.info("Migrated %s notification target(s) from %s.", inserted, self._source_file)
await self._move_file(self._source_file)

View file

@ -1,5 +1,6 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from sqlalchemy import func, or_, select
@ -7,43 +8,23 @@ from sqlalchemy import func, or_, select
from app.features.core.deps import get_session
from app.features.notifications.migration import Migration
from app.features.notifications.models import NotificationModel
from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from collections.abc import AsyncGenerator
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> NotificationModel:
model = NotificationModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for NotificationModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: NotificationModel | dict[str, Any]) -> NotificationModel:
if isinstance(payload, NotificationModel):
return payload
return _model_from_payload(payload)
LOG: logging.Logger = logging.getLogger(__name__)
class NotificationsRepository(metaclass=Singleton):
def __init__(self, session: SessionFactory | None = None) -> None:
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: SessionFactory = session or get_session
self.session: AsyncGenerator[AsyncSession] = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -56,7 +37,7 @@ class NotificationsRepository(metaclass=Singleton):
def get_instance() -> NotificationsRepository:
return NotificationsRepository()
async def all(self) -> list[NotificationModel]:
async def list(self) -> list[NotificationModel]:
async with self.session() as session:
result: Result[tuple[NotificationModel]] = await session.execute(
select(NotificationModel).order_by(NotificationModel.name.asc())
@ -111,11 +92,11 @@ class NotificationsRepository(metaclass=Singleton):
result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: NotificationModel | dict[str, Any]) -> NotificationModel:
async def create(self, payload: NotificationModel | dict) -> NotificationModel:
async with self.session() as session:
model = _coerce_model(payload)
model: NotificationModel = NotificationModel(**payload) if isinstance(payload, dict) else payload
if model.id is not None:
model.id = None # ty: ignore
model.id = None
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Notification target with name '{model.name}' already exists."

View file

@ -1,3 +1,4 @@
import logging
from typing import Any
from aiohttp import web
@ -10,10 +11,9 @@ from app.features.notifications.schemas import Notification, NotificationEvents,
from app.features.notifications.service import Notifications
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.router import route
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def _model(model: Any) -> Notification:

View file

@ -16,7 +16,6 @@ class NotificationEvents:
ITEM_COMPLETED: str = Events.ITEM_COMPLETED
ITEM_CANCELLED: str = Events.ITEM_CANCELLED
ITEM_DELETED: str = Events.ITEM_DELETED
ITEM_BULK_DELETED: str = Events.ITEM_BULK_DELETED
ITEM_PAUSED: str = Events.ITEM_PAUSED
ITEM_RESUMED: str = Events.ITEM_RESUMED
ITEM_MOVED: str = Events.ITEM_MOVED

View file

@ -1,8 +1,10 @@
from __future__ import annotations
import asyncio
import logging
import traceback
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
from typing import TYPE_CHECKING, Any
from app.features.notifications.models import NotificationModel
from app.features.notifications.repository import NotificationsRepository
@ -21,7 +23,6 @@ from app.library.encoder import Encoder
from app.library.Events import Event, EventBus, Events
from app.library.httpx_client import async_client
from app.library.ItemDTO import Item, ItemDTO
from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
@ -30,7 +31,7 @@ if TYPE_CHECKING:
import httpx
from aiohttp import web
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("feature.notifications")
class Notifications(metaclass=Singleton):
@ -64,10 +65,10 @@ class Notifications(metaclass=Singleton):
await self._repo.run_migrations()
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations")
EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{type(self).__name__}.emit")
EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{__class__.__name__}.emit")
async def all(self) -> list[NotificationModel]:
return await self._repo.all()
async def list(self) -> list[NotificationModel]:
return await self._repo.list()
async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]:
return await self._repo.list_paginated(page, per_page)
@ -92,8 +93,8 @@ class Notifications(metaclass=Singleton):
async def delete(self, identifier: int | str) -> NotificationModel:
return await self._repo.delete(identifier)
async def send(self, ev: Event, wait: bool = True) -> list[dict | Awaitable[dict]]:
targets = await self._repo.all()
async def send(self, ev: Event, wait: bool = True) -> list[dict] | list[Awaitable[dict]]:
targets = await self._repo.list()
if len(targets) < 1:
return []
@ -126,7 +127,7 @@ class Notifications(metaclass=Singleton):
if wait:
return await asyncio.gather(*tasks)
return cast("list[dict | Awaitable[dict]]", tasks)
return tasks
def emit(self, e: Event, _, **__) -> None:
if not NotificationEvents.is_valid(e.event):
@ -271,27 +272,14 @@ class Notifications(metaclass=Singleton):
if not status:
msg = "Apprise failed to send notification."
self._raise_apprise_error(msg)
raise RuntimeError(msg) # noqa: TRY301
except Exception as exc:
LOG.exception(
"Failed to send Apprise notification for event '%s'.",
ev.event,
extra={
"event_id": ev.id,
"event": ev.event,
"target_count": len(targets),
"targets": [t.name for t in targets],
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
LOG.error("Error sending Apprise notification: %s", exc)
return {"error": str(exc), "event": ev.event, "id": ev.id, "targets": [t.name for t in targets]}
return {}
@staticmethod
def _raise_apprise_error(msg: str) -> None:
raise RuntimeError(msg)
async def _send(self, target: Notification, ev: Event) -> dict:
try:
LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name)
@ -347,17 +335,14 @@ class Notifications(metaclass=Singleton):
return resp_data
except Exception as exc:
LOG.exception(
"Failed to send notification event '%s: %s' to '%s'.",
err_msg = str(exc) or type(exc).__name__
tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
LOG.error(
"Error sending notification event '%s: %s' to '%s'. '%s'. %s",
ev.event,
ev.id,
target.name,
extra={
"event_id": ev.id,
"event": ev.event,
"target_name": target.name,
"url": target.request.url,
"exception_type": type(exc).__name__,
},
err_msg,
tb,
)
return {"url": target.request.url, "status": 500, "text": str(ev)}

View file

@ -7,23 +7,25 @@ import pytest_asyncio
from app.features.notifications.repository import NotificationsRepository
from app.library.sqlite_store import SqliteStore
from app.tests.helpers import make_in_memory_db_path
@pytest_asyncio.fixture
async def repo():
async def repo(tmp_path):
"""Provide a fresh repository instance with initialized database for each test."""
NotificationsRepository._reset_singleton()
SqliteStore._reset_singleton()
store = SqliteStore(db_path=make_in_memory_db_path("notifications-repository"))
store = SqliteStore(db_path=":memory:")
await store.get_connection()
repository = NotificationsRepository.get_instance()
yield repository
await store.close()
if store._conn:
await store._conn.close()
if store._engine:
await store._engine.dispose()
NotificationsRepository._reset_singleton()
SqliteStore._reset_singleton()
@ -35,7 +37,7 @@ class TestNotificationsRepository:
@pytest.mark.asyncio
async def test_list_empty(self, repo):
"""List returns empty when no notifications exist."""
notifications = await repo.all()
notifications = await repo.list()
assert notifications == [], "Should return empty list when no notifications exist"
@pytest.mark.asyncio

View file

@ -1,6 +1,7 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any
@ -8,12 +9,11 @@ from app.features.core.migration import Migration as FeatureMigration
from app.features.presets.schemas import Preset
from app.features.presets.utils import preset_name
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.presets.repository import PresetsRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -32,11 +32,7 @@ class Migration(FeatureMigration):
try:
items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text())
except Exception as exc:
LOG.exception(
"Failed to read presets migration file '%s'. Ignoring.",
self._source_file,
extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
await self._move_file(self._source_file)
return
@ -46,7 +42,7 @@ class Migration(FeatureMigration):
return
inserted = 0
seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.all()}
seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.list()}
for index, item in enumerate(items):
if not (normalized := self._normalize(item, index, seen_names)):
@ -55,11 +51,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert preset '%s'.",
normalized["name"],
extra={"preset": normalized["name"], "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert preset '%s': %s", normalized["name"], exc)
LOG.info("Migrated %s preset(s) from %s.", inserted, self._source_file)
await self._move_file(self._source_file)

View file

@ -1,5 +1,6 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from sqlalchemy import func, or_, select
@ -11,13 +12,11 @@ from app.features.presets.models import PresetModel
from app.features.presets.utils import preset_name, seed_defaults
from app.library.config import Config
from app.library.Events import Event, EventBus, Events
from app.library.log import get_logger
from app.library.Services import Services
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from collections.abc import AsyncGenerator
from aiohttp import web
from sqlalchemy.engine.result import Result
@ -25,58 +24,13 @@ if TYPE_CHECKING:
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
def _payload_data(payload: PresetModel | dict[str, Any]) -> dict[str, Any]:
if isinstance(payload, dict):
data: dict[str, Any] = {}
for key, value in payload.items():
if not isinstance(key, str):
msg = "Preset payload keys must be strings."
raise TypeError(msg)
data[key] = value
return data
return {
"name": payload.name,
"description": payload.description,
"folder": payload.folder,
"template": payload.template,
"cookies": payload.cookies,
"cli": payload.cli,
"default": payload.default,
"priority": payload.priority,
"created_at": payload.created_at,
"updated_at": payload.updated_at,
}
LOG: logging.Logger = logging.getLogger(__name__)
class PresetsRepository(metaclass=Singleton):
SORT_FIELDS: dict[str, Any] = {
"id": PresetModel.id,
"name": PresetModel.name,
"priority": PresetModel.priority,
"default": PresetModel.default,
"created_at": PresetModel.created_at,
"updated_at": PresetModel.updated_at,
}
SORT_DIRECTIONS: tuple[str, str] = ("asc", "desc")
DEFAULT_SORT_ORDER: tuple[tuple[str, str], ...] = (("priority", "desc"), ("name", "asc"))
FIELD_DEFAULT_DIRECTIONS: dict[str, str] = {
"id": "asc",
"name": "asc",
"priority": "desc",
"default": "desc",
"created_at": "desc",
"updated_at": "desc",
}
def __init__(self, session: SessionFactory | None = None) -> None:
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: SessionFactory = session or get_session
self.session: AsyncGenerator[AsyncSession] = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -97,15 +51,15 @@ class PresetsRepository(metaclass=Singleton):
LOG.debug("Refreshing presets cache due to configuration update.")
await self._update_cache()
Services.get_instance().add(PresetsRepository.__name__, self)
Services.get_instance().add(__class__.__name__, self)
EventBus.get_instance().subscribe(
Events.STARTED, handle_event, f"{PresetsRepository.__name__}.run_migrations"
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache")
async def _update_cache(self) -> None:
from app.features.presets.service import Presets
await Presets.get_instance().refresh_cache(await self.all())
await Presets.get_instance().refresh_cache(await self.list())
@staticmethod
def get_instance() -> PresetsRepository:
@ -121,115 +75,33 @@ class PresetsRepository(metaclass=Singleton):
LOG.error("Default preset '%s' not found, using 'default' preset.", default_name)
config.default_preset = "default"
async def all(self) -> list[PresetModel]:
async def list(self) -> list[PresetModel]:
async with self.session() as session:
result: Result[tuple[PresetModel]] = await session.execute(
select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc())
)
return list(result.scalars().all())
@classmethod
def parse_sorting(cls, sort: str | None = None, order: str | None = None) -> tuple[tuple[str, str], ...]:
sort_value = (sort or "").strip()
order_value = (order or "").strip()
if not sort_value and not order_value:
return cls.DEFAULT_SORT_ORDER
if order_value and not sort_value:
msg = "sort is required when order is provided."
raise ValueError(msg)
fields: list[str] = []
for raw_field in sort_value.split(","):
field = raw_field.strip().lower()
if not field:
continue
if field not in cls.SORT_FIELDS:
msg = f"sort must use supported fields: {', '.join(cls.SORT_FIELDS)}."
raise ValueError(msg)
if field not in fields:
fields.append(field)
if not fields:
msg = "sort must include at least one field."
raise ValueError(msg)
directions: list[str] = []
if order_value:
for raw_direction in order_value.split(","):
direction = raw_direction.strip().lower()
if not direction:
continue
if direction not in cls.SORT_DIRECTIONS:
msg = "order must be 'asc' or 'desc'."
raise ValueError(msg)
directions.append(direction)
if not directions:
msg = "order must include at least one direction."
raise ValueError(msg)
if len(directions) not in {1, len(fields)}:
msg = "order must provide one direction or match the number of sort fields."
raise ValueError(msg)
if not directions:
return tuple((field, cls.FIELD_DEFAULT_DIRECTIONS.get(field, "asc")) for field in fields)
if len(directions) == 1:
return tuple((field, directions[0]) for field in fields)
return tuple((field, directions[index]) for index, field in enumerate(fields))
@classmethod
def _apply_sort_direction(cls, field: str, direction: str) -> Any:
column = cls.SORT_FIELDS[field]
return column.asc() if direction == "asc" else column.desc()
@classmethod
def _build_order_by(cls, sort: str | None = None, order: str | None = None) -> list[Any]:
sorting = cls.parse_sorting(sort, order)
order_by: list[Any] = [cls._apply_sort_direction(field, direction) for field, direction in sorting]
if all(field != "id" for field, _ in sorting):
order_by.append(PresetModel.id.asc())
return order_by
async def list_paginated(
self,
page: int,
per_page: int,
sort: str | None = None,
order: str | None = None,
exclude_defaults: bool = False,
) -> tuple[list[PresetModel], int, int, int]:
order_by = self._build_order_by(sort, order)
async def list_paginated(self, page: int, per_page: int) -> tuple[list[PresetModel], int, int, int]:
async with self.session() as session:
total: int = await self.count(exclude_defaults=exclude_defaults)
total: int = await self.count()
total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1
if page > total_pages and total > 0:
page = total_pages
query: Select[tuple[PresetModel]] = select(PresetModel)
if exclude_defaults:
query = query.where(PresetModel.default.is_(False))
query = query.order_by(*order_by).limit(per_page).offset((page - 1) * per_page)
query: Select[tuple[PresetModel]] = (
select(PresetModel)
.order_by(PresetModel.priority.desc(), PresetModel.name.asc())
.limit(per_page)
.offset((page - 1) * per_page)
)
result: Result[tuple[PresetModel]] = await session.execute(query)
return list(result.scalars().all()), total, page, total_pages
async def count(self, exclude_defaults: bool = False) -> int:
async def count(self) -> int:
async with self.session() as session:
query = select(func.count()).select_from(PresetModel)
if exclude_defaults:
query = query.where(PresetModel.default.is_(False))
result: Result[tuple[int]] = await session.execute(query)
result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(PresetModel))
return int(result.scalar_one())
async def get(self, identifier: int | str) -> PresetModel | None:
@ -262,12 +134,11 @@ class PresetsRepository(metaclass=Singleton):
result: Result[tuple[PresetModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: PresetModel | dict[str, Any]) -> PresetModel:
async def create(self, payload: PresetModel | dict) -> PresetModel:
async with self.session() as session:
data = _payload_data(payload)
data.pop("id", None)
model = PresetModel(**data)
model: PresetModel = PresetModel(**payload) if isinstance(payload, dict) else payload
if model.id is not None:
model.id = None
model.name = preset_name(model.name)
@ -292,10 +163,12 @@ class PresetsRepository(metaclass=Singleton):
result: Result[tuple[PresetModel]] = await session.execute(select(PresetModel).where(clause).limit(1))
model: PresetModel | None = result.scalar_one_or_none()
if model is None:
if None is model:
msg: str = f"Preset '{identifier}' not found."
raise KeyError(msg)
assert None is not model
payload.pop("id", None)
payload.pop("created_at", None)
payload.pop("updated_at", None)
@ -304,10 +177,9 @@ class PresetsRepository(metaclass=Singleton):
if hasattr(model, key):
setattr(model, key, value)
normalized_name = preset_name(model.name)
model.name = normalized_name
if await self.get_by_name(name=normalized_name, exclude_id=model.id) is not None:
msg = f"Preset with name '{normalized_name}' already exists."
model.name = preset_name(model.name)
if await self.get_by_name(name=model.name, exclude_id=model.id) is not None:
msg = f"Preset with name '{model.name}' already exists."
raise ValueError(msg)
await session.commit()
@ -324,11 +196,12 @@ class PresetsRepository(metaclass=Singleton):
clause = PresetModel.name == preset_name(identifier)
result: Result[tuple[PresetModel]] = await session.execute(select(PresetModel).where(clause).limit(1))
model: PresetModel | None = result.scalar_one_or_none()
if model is None:
if not (model := result.scalar_one_or_none()):
msg: str = f"Preset '{identifier}' not found."
raise KeyError(msg)
assert None is not model
await session.delete(model)
await session.commit()
return model

View file

@ -23,18 +23,8 @@ def _serialize(model: Any) -> dict:
@route("GET", "api/presets/", "presets")
async def presets_list(request: Request, encoder: Encoder, repo: PresetsRepository) -> Response:
try:
page, per_page = normalize_pagination(request)
items, total, current_page, total_pages = await repo.list_paginated(
page=page,
per_page=per_page,
sort=request.query.get("sort"),
order=request.query.get("order"),
exclude_defaults=bool(request.query.get("exclude_defaults", False)),
)
except ValueError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
page, per_page = normalize_pagination(request)
items, total, current_page, total_pages = await repo.list_paginated(page, per_page)
return web.json_response(
data=PresetList(
items=[_model(model) for model in items],

View file

@ -12,7 +12,7 @@ class Presets(metaclass=Singleton):
def __init__(self, repo: PresetsRepository | None = None) -> None:
self._repo: PresetsRepository = repo or PresetsRepository.get_instance()
self._cache: list[tuple[int, str, Preset]] = []
Services.get_instance().add(type(self).__name__, self)
Services.get_instance().add(__class__.__name__, self)
@staticmethod
def get_instance() -> Presets:

View file

@ -1,18 +1,10 @@
from __future__ import annotations
import json
from unittest.mock import MagicMock
import pytest
import pytest_asyncio
from aiohttp import web
from aiohttp.web import Request
from app.features.presets.repository import PresetsRepository
from app.features.presets.router import presets_list
from app.library.encoder import Encoder
from app.library.sqlite_store import SqliteStore
from app.tests.helpers import make_in_memory_db_path
@pytest_asyncio.fixture
@ -20,13 +12,16 @@ async def repo():
PresetsRepository._reset_singleton()
SqliteStore._reset_singleton()
store = SqliteStore(db_path=make_in_memory_db_path("presets-repository"))
store = SqliteStore(db_path=":memory:")
await store.get_connection()
repository = PresetsRepository.get_instance()
yield repository
await store.close()
if store._conn:
await store._conn.close()
if store._engine:
await store._engine.dispose()
PresetsRepository._reset_singleton()
SqliteStore._reset_singleton()
@ -57,7 +52,7 @@ class TestPresetsRepository:
await repo.create({"name": "A", "priority": 1})
await repo.create({"name": "C", "priority": 2})
items = await repo.all()
items = await repo.list()
assert items[0].name == "c", "Highest priority should be first"
assert items[1].name == "a", "Same priority should sort by name"
@ -74,106 +69,3 @@ class TestPresetsRepository:
assert total == 5, "Should report total count"
assert page == 1, "Should be on page 1"
assert total_pages == 3, "Should have 3 pages total"
assert [item.priority for item in items] == [4, 3], "Should keep default priority-desc order"
@pytest.mark.asyncio
async def test_list_paginated_sorts_by_name_desc(self, repo):
await repo.create({"name": "Alpha", "priority": 1})
await repo.create({"name": "Gamma", "priority": 3})
await repo.create({"name": "Beta", "priority": 2})
items, _, _, _ = await repo.list_paginated(page=1, per_page=10, sort="name", order="desc")
assert [item.name for item in items] == ["gamma", "beta", "alpha"], "Should sort by requested field"
@pytest.mark.asyncio
async def test_list_paginated_excludes_defaults(self, repo):
await repo.create({"name": "System Default", "default": True, "priority": 10})
await repo.create({"name": "Custom Preset", "priority": 1})
items, total, page, total_pages = await repo.list_paginated(page=1, per_page=10, exclude_defaults=True)
assert [item.name for item in items] == ["custom_preset"], "Should exclude default presets"
assert total == 1, "Should count only custom presets"
assert page == 1, "Should keep current page when filtered results exist"
assert total_pages == 1, "Should compute pages from the filtered total"
@pytest.mark.asyncio
async def test_list_paginated_multi_sort(self, repo):
await repo.create({"name": "Charlie", "priority": 2})
await repo.create({"name": "Alpha", "priority": 1})
await repo.create({"name": "Bravo", "priority": 1})
items, _, _, _ = await repo.list_paginated(page=1, per_page=10, sort="priority,name", order="asc,desc")
assert [(item.priority, item.name) for item in items] == [
(1, "bravo"),
(1, "alpha"),
(2, "charlie"),
], "Should support multiple sort fields and directions"
@pytest.mark.asyncio
async def test_list_paginated_bad_sort(self, repo):
with pytest.raises(ValueError, match="sort must use supported fields"):
await repo.list_paginated(page=1, per_page=10, sort="cli", order="asc")
@pytest.mark.asyncio
async def test_list_paginated_bad_order(self, repo):
with pytest.raises(ValueError, match="order must be 'asc' or 'desc'"):
await repo.list_paginated(page=1, per_page=10, sort="name", order="sideways")
@pytest.mark.asyncio
async def test_list_paginated_mismatched_order(self, repo):
with pytest.raises(ValueError, match="order must provide one direction or match the number of sort fields"):
await repo.list_paginated(page=1, per_page=10, sort="priority,name", order="asc,desc,asc")
@pytest.mark.asyncio
class TestPresetRoutes:
async def test_list_route_sort(self, repo):
await repo.create({"name": "Alpha", "priority": 1})
await repo.create({"name": "Bravo", "priority": 1})
await repo.create({"name": "Charlie", "priority": 2})
request = MagicMock(spec=Request)
request.query = {"page": "1", "per_page": "10", "sort": "priority,name", "order": "asc,desc"}
response = await presets_list(request, Encoder(), repo)
payload = json.loads(response.text)
assert response.status == web.HTTPOk.status_code, "Should return 200 for valid sorting"
assert [item["name"] for item in payload["items"]] == ["bravo", "alpha", "charlie"], "Should sort response"
async def test_list_route_bad_sort(self, repo):
request = MagicMock(spec=Request)
request.query = {"sort": "cli", "order": "asc"}
response = await presets_list(request, Encoder(), repo)
payload = json.loads(response.text)
assert response.status == web.HTTPBadRequest.status_code, "Should reject unsupported sort field"
assert "sort" in payload["error"], "Should explain invalid sort field"
async def test_list_route_bad_order(self, repo):
request = MagicMock(spec=Request)
request.query = {"sort": "name", "order": "sideways"}
response = await presets_list(request, Encoder(), repo)
payload = json.loads(response.text)
assert response.status == web.HTTPBadRequest.status_code, "Should reject unsupported sort direction"
assert "order" in payload["error"], "Should explain invalid sort direction"
async def test_list_route_exclude_defaults(self, repo):
await repo.create({"name": "System Default", "default": True, "priority": 10})
await repo.create({"name": "Custom Preset", "priority": 1})
request = MagicMock(spec=Request)
request.query = {"page": "1", "per_page": "10", "exclude_defaults": "true"}
response = await presets_list(request, Encoder(), repo)
payload = json.loads(response.text)
assert response.status == web.HTTPOk.status_code, "Should return 200 for valid default exclusion"
assert [item["name"] for item in payload["items"]] == ["custom_preset"], "Should exclude default presets"
assert payload["pagination"]["total"] == 1, "Should report filtered total"

View file

@ -1,11 +1,10 @@
import logging
import re
from datetime import UTC, datetime
from app.library.log import get_logger
NAME_WHITESPACE_PATTERN = re.compile(r"\s+")
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
async def seed_defaults(repo) -> None:
@ -49,11 +48,7 @@ async def seed_defaults(repo) -> None:
await repo.update(existing.id, payload)
except Exception as exc:
LOG.exception(
"Failed to seed default preset '%s'.",
preset.get("name"),
extra={"preset": preset.get("name"), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to seed default preset '%s': %s", preset.get("name"), exc)
def preset_name(value: str) -> str:

View file

@ -5,16 +5,18 @@ Python wrapper for ffprobe command line tool. ffprobe must exist in the path.
import asyncio
import functools
import json
import logging
import operator
import os
import subprocess # qa: ignore
from pathlib import Path
import anyio
from app.features.streaming.types import FFProbeError
from app.library.log import get_logger
from app.library.Utils import timed_lru_cache
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("streaming.ffprobe")
class FFStream:
@ -37,26 +39,21 @@ class FFStream:
def __repr__(self):
if "codec_long_name" not in self.__dict__:
self.__dict__["codec_long_name"] = self.__dict__.get("codec_name", "")
index = self.__dict__.get("index", "?")
codec_type = self.__dict__.get("codec_type", "unknown")
codec_long_name = self.__dict__.get("codec_long_name", "")
self.codec_long_name = self.__dict__.get("codec_name", "")
if self.is_video():
return f"<Stream: #{index} [{codec_type}] {codec_long_name}, {self.__dict__.get('framerate')}, ({self.__dict__.get('width')}x{self.__dict__.get('height')})>"
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, {self.framerate}, ({self.width}x{self.height})>"
if self.is_audio():
return (
f"<Stream: #{index} [{codec_type}] {codec_long_name}, "
f"channels: {self.__dict__.get('channels')} ({self.__dict__.get('channel_layout')}), "
f"{self.__dict__.get('sample_rate')}Hz> "
f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, channels: {self.channels} ({self.channel_layout}), "
"{sample_rate}Hz> "
)
if self.is_subtitle() or self.is_attachment():
return f"<Stream: #{index} [{codec_type}] {codec_long_name}>"
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}>"
return f"<Stream: #{index} [{codec_type}]>"
return f"<Stream: #{self.index} [{self.codec_type}]>"
def is_audio(self):
"""
@ -255,14 +252,14 @@ async def ffprobe(file: Path | str) -> FFProbeResult:
raise OSError(msg)
try:
proc = await asyncio.create_subprocess_exec(
"ffprobe",
"-h",
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
)
await proc.wait()
async with await anyio.open_file(os.devnull, "w") as tempf:
await asyncio.create_subprocess_exec(
"ffprobe",
"-h",
stdout=tempf,
stderr=tempf,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
)
except FileNotFoundError as e:
msg = "ffprobe not found."
raise OSError(msg) from e

View file

@ -39,7 +39,7 @@ class M3u8:
m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
segmentSize: str = f"{self.duration:.6f}"
segmentSize: float = f"{self.duration:.6f}"
splits: int = math.ceil(duration / self.duration)
segmentParams: dict = {}

View file

@ -14,7 +14,7 @@ class Playlist:
"The path where files are downloaded."
async def make(self, file: Path) -> str:
ref: Path = Path(str(file.relative_to(self.download_path)).strip("/"))
ref: str = Path(str(file.relative_to(self.download_path)).strip("/"))
try:
ff = await ffprobe(file)

View file

@ -1,7 +1,7 @@
# flake8: noqa: ARG002
from __future__ import annotations
import os
import shutil
import subprocess
import sys
from functools import lru_cache
@ -16,13 +16,9 @@ def detect_qsv_capabilities() -> dict[str, dict[str, bool]]:
Returns a dict where keys are codec names (e.g. "h264", "hevc", "vp9") and
values are dicts with keys "full" and "lp" booleans.
"""
vainfo = shutil.which("vainfo")
if not vainfo:
return {}
try:
result: subprocess.CompletedProcess[str] = subprocess.run(
[vainfo],
["vainfo"], # noqa: S607
capture_output=True,
text=True,
check=False,
@ -91,13 +87,9 @@ def ffmpeg_encoders() -> set[str]:
"""
from app.library.config import SUPPORTED_CODECS
ffmpeg = shutil.which("ffmpeg")
if not ffmpeg:
return set()
try:
result: subprocess.CompletedProcess[str] = subprocess.run(
[ffmpeg, "-hide_banner", "-loglevel", "error", "-encoders"],
["ffmpeg", "-hide_banner", "-loglevel", "error", "-encoders"], # noqa: S607
capture_output=True,
text=True,
check=False,
@ -157,11 +149,9 @@ class _BaseBuilder:
codec_name: str
def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
_ = ctx
return []
def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
_ = ctx
return [*args, "-codec:v", self.codec_name]
@ -322,4 +312,4 @@ def encoder_fallback_chain(codec: str) -> tuple[str, ...]:
"libx264": [],
}
return tuple(chains.get(codec, chains["libx264"]))
return chains.get(codec, chains["libx264"])

View file

@ -1,10 +1,11 @@
import asyncio
import hashlib
import logging
import os
import subprocess
import subprocess # type: ignore
import sys
import tempfile
from pathlib import Path
from secrets import token_hex
from typing import TYPE_CHECKING, ClassVar
from aiohttp import web
@ -18,7 +19,6 @@ from app.features.streaming.library.segment_encoders import (
select_encoder,
)
from app.library.config import SUPPORTED_CODECS, Config
from app.library.log import get_logger
if TYPE_CHECKING:
from asyncio.subprocess import Process
@ -26,7 +26,7 @@ if TYPE_CHECKING:
from .ffprobe import FFProbeResult
from .segment_encoders import EncoderBuilder
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("player.segments")
class Segments:
@ -68,21 +68,21 @@ class Segments:
self.attempted: set[str] = set()
"The set of attempted codecs."
def _make_stream_input(self, file: Path) -> Path:
while True:
stream_input = Path(tempfile.gettempdir()).joinpath(f"ytptube_stream.{token_hex(8)}")
try:
stream_input.symlink_to(file, target_is_directory=False)
return stream_input
except FileExistsError:
continue
async def build_ffmpeg_args(self, file: Path, s_codec: str, *, stream_input: Path | None = None) -> list[str]:
async def build_ffmpeg_args(self, file: Path, s_codec: str) -> list[str]:
try:
ff: FFProbeResult = await ffprobe(file)
except UnicodeDecodeError:
pass
input_path = stream_input or file
tmpFile: Path = Path(tempfile.gettempdir()).joinpath(
f"ytptube_stream.{hashlib.sha256(str(file).encode()).hexdigest()}"
)
if not tmpFile.exists():
try:
tmpFile.symlink_to(file, target_is_directory=False)
except FileExistsError:
pass
startTime: str = f"{0:.6f}" if self.index == 0 else f"{self.duration * self.index:.6f}"
@ -123,7 +123,7 @@ class Segments:
# hardware/global input options must come before -i
*input_args,
"-i",
f"file:{input_path}",
f"file:{tmpFile}",
"-map_metadata",
"-1",
]
@ -170,15 +170,9 @@ class Segments:
stderr_task: asyncio.Task[None] = asyncio.create_task(_drain_stderr())
LOG.debug(
"Streaming segment %s for '%s'.",
self.index,
file,
extra={"file": str(file), "segment_index": self.index, "ffmpeg_args": args},
)
LOG.debug(f"Streaming '{file}' segment '{self.index}'. ffmpeg: {' '.join(args)}")
try:
assert proc.stdout is not None
while True:
chunk: bytes = await proc.stdout.read(1024 * 64)
if not chunk:
@ -197,10 +191,7 @@ class Segments:
try:
await asyncio.wait_for(proc.wait(), timeout=5)
except TimeoutError:
LOG.warning(
"Segment stream for '%s' did not stop ffmpeg in time after the client disconnected; killing it.",
file,
)
LOG.error("ffmpeg process did not terminate in time. Killing it.")
proc.kill()
raise
except ConnectionResetError:
@ -213,10 +204,7 @@ class Segments:
try:
rc: int = await asyncio.wait_for(proc.wait(), timeout=5)
except TimeoutError:
LOG.warning(
"Segment stream for '%s' did not stop ffmpeg in time after the client disconnected; killing it.",
file,
)
LOG.error("ffmpeg process did not terminate in time after disconnect. Killing it.")
proc.kill()
try:
rc = await asyncio.wait_for(proc.wait(), timeout=5)
@ -227,7 +215,7 @@ class Segments:
try:
rc = await asyncio.wait_for(proc.wait(), timeout=30)
except TimeoutError:
LOG.warning("Segment stream for '%s' did not stop ffmpeg in time; killing the process.", file)
LOG.error("ffmpeg process wait timed out. Killing it.")
proc.kill()
try:
rc = await asyncio.wait_for(proc.wait(), timeout=5)
@ -241,47 +229,33 @@ class Segments:
return (wrote_any, rc, client_disconnected_local, stderr_buf.decode("utf-8", errors="ignore").strip())
async def stream(self, file: Path, resp: web.StreamResponse) -> None:
codec: str = self.vcodec
if Segments._cache_initialized and Segments._cached_vcodec:
codec = Segments._cached_vcodec
codec: str = Segments._cached_vcodec if Segments._cache_initialized else self.vcodec
if not codec or codec not in SUPPORTED_CODECS:
codec: str = select_encoder(self.vcodec or "")
LOG.debug("Selected video codec '%s' for segment streaming.", codec, extra={"codec": codec})
LOG.debug(f"Selected video codec '{codec}' for segment streaming.")
if Segments._cached_vcodec and Segments._cache_initialized:
codecs: list[str] = [Segments._cached_vcodec]
else:
codecs: list[str] = [codec, *list(encoder_fallback_chain(codec))]
stream_input = self._make_stream_input(file)
try:
for s_codec in codecs:
if s_codec in self.attempted:
continue
for s_codec in codecs:
if s_codec in self.attempted:
continue
ffmpeg_args: list[str] = await self.build_ffmpeg_args(file, s_codec, stream_input=stream_input)
_, rc, client_disconnected, stderr_text = await self._run(resp, file, ffmpeg_args)
ffmpeg_args: list[str] = await self.build_ffmpeg_args(file, s_codec)
_, rc, client_disconnected, stderr_text = await self._run(resp, file, ffmpeg_args)
if 0 == rc:
Segments._cached_vcodec = s_codec
Segments._cache_initialized = True
return
if 0 == rc:
Segments._cached_vcodec = s_codec
Segments._cache_initialized = True
return
if client_disconnected:
return
if client_disconnected:
return
if 0 != rc:
err: str = stderr_text[:500] if stderr_text else "no error output"
LOG.warning(
"Retrying segment %s for '%s' because hardware encoder '%s' failed: %s.",
self.index,
file,
s_codec,
err,
extra={"ffmpeg_args": ffmpeg_args, "returncode": rc, "stderr": err, "codec": s_codec},
)
self.attempted.add(s_codec)
finally:
if stream_input.is_symlink():
stream_input.unlink()
if 0 != rc:
err: str = stderr_text[:500] if stderr_text else "no error output"
LOG.warning(f"transcoding has failed (cmd={ffmpeg_args}) (rc={rc}): {err}. Trying fallbacks.")
self.attempted.add(s_codec)

View file

@ -1,45 +1,14 @@
from __future__ import annotations
from dataclasses import dataclass
import logging
from pathlib import Path
from typing import Any
import anyio
import pysubs2
from pysubs2.formats.substation import SubstationFormat
from pysubs2.time import ms_to_times
from app.library.log import get_logger
from app.library.Utils import ALLOWED_SUBS_EXTENSIONS, get_file_sidecar
from app.library.Utils import ALLOWED_SUBS_EXTENSIONS
LOG = get_logger()
SOURCE_FORMATS: tuple[str, ...] = ("vtt", "srt", "ass")
DELIVERY_FORMATS: dict[str, str] = {
"vtt": "vtt",
"srt": "vtt",
"ass": "ass",
}
RENDERERS: dict[str, str] = {
"vtt": "native",
"srt": "native",
"ass": "assjs",
}
MEDIA_TYPES: dict[str, str] = {
"vtt": "text/vtt; charset=UTF-8",
"ass": "text/x-ssa; charset=UTF-8",
}
TEXT_ENCODINGS: tuple[str, ...] = ("utf-8-sig", "utf-16", "utf-16-le", "utf-16-be", "cp1252")
@dataclass(frozen=True, slots=True)
class SubtitleTrack:
file: Path
lang: str
name: str
source_format: str
delivery_format: str
renderer: str
LOG: logging.Logger = logging.getLogger("player.subtitle")
def ms_to_timestamp(ms: int) -> str:
@ -49,51 +18,18 @@ def ms_to_timestamp(ms: int) -> str:
return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}"
_substation_format: Any = SubstationFormat
_substation_format.ms_to_timestamp = ms_to_timestamp
SubstationFormat.ms_to_timestamp = ms_to_timestamp
class Subtitle:
@staticmethod
def normalize_format(source_format: str) -> str | None:
fmt = source_format.strip().lower().removeprefix(".")
if fmt not in SOURCE_FORMATS:
return None
return fmt
async def read_text(self, file: Path) -> str:
async with await anyio.open_file(file, "rb") as f:
subtitle_bytes = await f.read()
return self.decode_bytes(subtitle_bytes)
@staticmethod
def decode_bytes(subtitle_bytes: bytes) -> str:
for encoding in TEXT_ENCODINGS:
try:
return subtitle_bytes.decode(encoding)
except UnicodeDecodeError:
continue
return subtitle_bytes.decode("utf-8", errors="replace")
def media_type(self, file: Path) -> str:
fmt = self.normalize_format(file.suffix)
if fmt is None:
msg = f"File '{file}' subtitle type is not supported."
raise Exception(msg)
return MEDIA_TYPES[DELIVERY_FORMATS[fmt]]
async def make(self, file: Path) -> str:
fmt = self.normalize_format(file.suffix)
if fmt is None or file.suffix not in ALLOWED_SUBS_EXTENSIONS:
if file.suffix not in ALLOWED_SUBS_EXTENSIONS:
msg: str = f"File '{file}' subtitle type is not supported."
raise Exception(msg)
if fmt == "vtt":
return await self.read_text(file)
if file.suffix == ".vtt":
async with await anyio.open_file(file) as f:
return await f.read()
subs: pysubs2.SSAFile = pysubs2.load(path=str(file))
@ -111,45 +47,3 @@ class Subtitle:
pass
return subs.to_string("vtt")
async def make_delivery(self, file: Path) -> tuple[str, str]:
fmt = self.normalize_format(file.suffix)
if fmt is None or file.suffix not in ALLOWED_SUBS_EXTENSIONS:
msg: str = f"File '{file}' subtitle type is not supported."
raise Exception(msg)
if fmt == "ass":
return await self.read_text(file), self.media_type(file)
return await self.make(file), self.media_type(file)
def get_subtitle_tracks(file: Path) -> list[SubtitleTrack]:
sidecars = get_file_sidecar(file).get("subtitle", [])
indexed_tracks: list[tuple[int, SubtitleTrack]] = []
for index, item in enumerate(sidecars):
track_file = item.get("file")
if not isinstance(track_file, Path):
continue
fmt = Subtitle.normalize_format(track_file.suffix)
if fmt is None:
continue
indexed_tracks.append(
(
index,
SubtitleTrack(
file=track_file,
lang=str(item.get("lang") or "und"),
name=str(item.get("name") or track_file.name),
source_format=fmt,
delivery_format=DELIVERY_FORMATS[fmt],
renderer=RENDERERS[fmt],
),
)
)
indexed_tracks.sort(key=lambda item: (SOURCE_FORMATS.index(item[1].source_format), item[0]))
return [track for _, track in indexed_tracks]

View file

@ -1,284 +0,0 @@
from __future__ import annotations
import asyncio
import os
import subprocess
from pathlib import Path
from app.features.streaming.library.ffprobe import ffprobe
from app.library.cache import Cache
from app.library.config import Config
from app.library.log import get_logger
from app.library.Utils import FILES_TYPE, get_file_sidecar
LOG = get_logger()
IMAGE_TYPES: tuple[str, ...] = (".jpg", ".jpeg", ".png", ".webp")
FOLDER_IMAGE_ORDER: tuple[str, ...] = ("thumbnail", "poster", "artwork", "cover", "fanart")
THUMBNAIL_SAMPLE_FRAMES = 200
THUMBNAIL_DEFAULT_SEEK_SECONDS = 3.0
THUMBNAIL_MIN_SEEK_SECONDS = 1.0
THUMBNAIL_MAX_SEEK_SECONDS = 15.0
THUMBNAIL_END_MARGIN_SECONDS = 0.1
THUMBNAIL_MISS_TTL = 3600.0
_LOCK = asyncio.Lock()
_IN_PROCESS: dict[str, asyncio.Task[Path | None]] = {}
_SEM: asyncio.Semaphore | None = None
_SEM_LIMIT: int | None = None
def _get_semaphore() -> asyncio.Semaphore:
global _SEM, _SEM_LIMIT # noqa: PLW0603
limit: int = max(1, int(Config.get_instance().thumb_concurrency))
if _SEM is None or _SEM_LIMIT != limit:
_SEM = asyncio.Semaphore(limit)
_SEM_LIMIT = limit
LOG.info("Configured thumbnail generation to run %s job(s) at a time.", limit, extra={"limit": limit})
return _SEM
def _is_same_stem_image(media_file: Path, image_file: Path) -> bool:
if image_file.parent != media_file.parent:
return False
for image_type in FILES_TYPE:
if "image" != image_type.get("type"):
continue
rx = image_type.get("rx")
if rx and rx.search(image_file.name):
break
else:
return False
return image_file.stem == media_file.stem
def pick_local_thumb(media_file: Path) -> Path | None:
sidecar: dict[str, list[dict[str, str]]] = get_file_sidecar(media_file)
images: list[dict[str, str]] = sidecar.get("image", [])
local_images: list[Path] = [Path(item["file"]) for item in images if isinstance(item.get("file"), Path)]
for image_file in local_images:
if image_file.suffix.lower() in IMAGE_TYPES and _is_same_stem_image(media_file, image_file):
return image_file
by_name: dict[str, Path] = {
image_file.stem.lower(): image_file for image_file in local_images if image_file.suffix.lower() in IMAGE_TYPES
}
for name in FOLDER_IMAGE_ORDER:
if image_file := by_name.get(name):
return image_file
return None
def _seek_seconds(ff_info) -> float | None:
duration: float = 0.0
try:
duration = float(ff_info.metadata.get("duration") or 0.0)
except (TypeError, ValueError):
duration = 0.0
if duration <= 0:
return THUMBNAIL_DEFAULT_SEEK_SECONDS
if duration <= THUMBNAIL_END_MARGIN_SECONDS:
return None
return min(
max(duration * 0.10, THUMBNAIL_MIN_SEEK_SECONDS),
THUMBNAIL_MAX_SEEK_SECONDS,
duration - THUMBNAIL_END_MARGIN_SECONDS,
)
def _build_ffmpeg_args(media_file: Path, output_file: Path, *, seek_seconds: float | None) -> list[str]:
args: list[str] = [
"ffmpeg",
"-nostdin",
"-hide_banner",
"-loglevel",
"error",
"-y",
]
if seek_seconds is not None and seek_seconds > 0:
args.extend(["-ss", f"{seek_seconds:.3f}"])
args.extend(
[
"-i",
str(media_file),
"-vf",
f"thumbnail={THUMBNAIL_SAMPLE_FRAMES},scale=1280:-1:force_original_aspect_ratio=decrease",
"-frames:v",
"1",
"-q:v",
"3",
"-an",
str(output_file),
]
)
return args
async def _run_ffmpeg(media_file: Path, output_file: Path) -> Path | None:
ff_info = await ffprobe(media_file)
if not ff_info.has_video():
LOG.debug(
"Skipping thumbnail generation for '%s' because no video stream exists.",
media_file,
extra={"media_file": str(media_file)},
)
return None
output_file.parent.mkdir(parents=True, exist_ok=True)
temp_file = output_file.with_suffix(".tmp.jpg")
if temp_file.exists():
temp_file.unlink(missing_ok=True)
seek_seconds: float | None = _seek_seconds(ff_info)
attempts: list[float | None] = [seek_seconds]
if seek_seconds is not None and seek_seconds > 0:
attempts.append(None)
sem = _get_semaphore()
if sem.locked():
limit = _SEM_LIMIT or 1
LOG.debug(
"Waiting for a thumbnail generation slot for '%s' (limit=%s).",
media_file,
limit,
extra={"media_file": str(media_file), "limit": limit},
)
async with sem:
last_error: str = "ffmpeg produced an empty thumbnail file"
for idx, attempt_seek in enumerate(attempts, start=1):
temp_file.unlink(missing_ok=True)
args: list[str] = _build_ffmpeg_args(media_file, temp_file, seek_seconds=attempt_seek)
LOG.debug(
"Generating thumbnail for '%s'. attempt=%s/%s seek=%s",
media_file,
idx,
len(attempts),
"none" if attempt_seek is None else f"{attempt_seek:.3f}s",
extra={
"media_file": str(media_file),
"attempt": idx,
"attempt_count": len(attempts),
"seek": "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
},
)
try:
proc = await asyncio.create_subprocess_exec(
*args,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
)
except FileNotFoundError as exc:
msg = "ffmpeg not found."
raise OSError(msg) from exc
stdout, stderr = await proc.communicate()
if 0 == proc.returncode and temp_file.exists() and temp_file.stat().st_size > 0:
temp_file.replace(output_file)
LOG.info(
"Generated thumbnail '%s' for '%s'.",
output_file,
media_file,
extra={"media_file": str(media_file), "output_file": str(output_file)},
)
return output_file
if 0 != proc.returncode:
last_error: str = (
f"ffmpeg thumbnail generation failed (rc={proc.returncode}).\n"
f"stdout:\n{stdout.decode('utf-8', errors='replace')}\n"
f"stderr:\n{stderr.decode('utf-8', errors='replace')}"
)
else:
last_error: str = "ffmpeg produced an empty thumbnail file"
LOG.debug(
"Thumbnail generation attempt failed for '%s'. seek=%s",
media_file,
"none" if attempt_seek is None else f"{attempt_seek:.3f}s",
extra={
"media_file": str(media_file),
"seek": "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
},
)
temp_file.unlink(missing_ok=True)
raise OSError(last_error)
async def ensure_thumb(media_file: Path, cache_root: Path, item_id: str | None = None) -> Path | None:
config: Config = Config.get_instance()
cache: Cache = Cache.get_instance()
if bool(config.thumb_generate) is not True:
return None
sidecar: bool = bool(config.thumb_sidecar)
if not sidecar and not item_id:
msg = "item_id is required when thumbnail sidecar is disabled."
raise ValueError(msg)
thumb_id: str = str(media_file.resolve()) if sidecar else str(item_id)
cache_file: Path = media_file.with_name(f"{media_file.name}.jpg") if sidecar else cache_root / f"{item_id}.jpg"
miss_key: str = f"thumbnail-miss:{thumb_id}"
if cache_file.exists() and cache_file.stat().st_size > 0:
cache.delete(miss_key)
return cache_file
if cache.has(miss_key):
return None
async with _LOCK:
if cache_file.exists() and cache_file.stat().st_size > 0:
cache.delete(miss_key)
return cache_file
if cache.has(miss_key):
return None
task = _IN_PROCESS.get(thumb_id)
if task is not None and not task.done():
LOG.debug("Waiting for thumbnail generation for '%s'.", media_file, extra={"media_file": str(media_file)})
else:
task = asyncio.create_task(_run_ffmpeg(media_file, cache_file), name=f"thumb-{item_id or media_file.stem}")
_IN_PROCESS[thumb_id] = task
LOG.debug(
"Starting thumbnail generation for '%s' -> '%s'.",
media_file,
cache_file,
extra={"media_file": str(media_file), "cache_file": str(cache_file)},
)
try:
result: Path | None = await task
if result is None:
cache.set(miss_key, value=True, ttl=THUMBNAIL_MISS_TTL)
else:
cache.delete(miss_key)
return result
except OSError:
cache.set(miss_key, value=True, ttl=THUMBNAIL_MISS_TTL)
raise
finally:
async with _LOCK:
current = _IN_PROCESS.get(thumb_id)
if current is task and task.done():
_IN_PROCESS.pop(thumb_id, None)

View file

@ -1,22 +1,21 @@
import logging
import time
from datetime import UTC, datetime
from pathlib import Path
from aiohttp import web
from aiohttp.web import Request, Response
from aiohttp.web_response import StreamResponse
from app.features.streaming.library.m3u8 import M3u8
from app.features.streaming.library.playlist import Playlist
from app.features.streaming.library.segments import Segments
from app.features.streaming.library.subtitle import Subtitle, get_subtitle_tracks
from app.features.streaming.library.subtitle import Subtitle
from app.features.streaming.types import StreamingError
from app.library.config import Config
from app.library.log import get_logger
from app.library.router import route
from app.library.Utils import get_file
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("streaming")
@route("GET", "api/player/playlist/{file:.*}.m3u8", "playlist_create")
@ -33,7 +32,7 @@ async def playlist_create(request: Request, config: Config, app: web.Application
Response: The response object.
"""
file: str | None = request.match_info.get("file")
file: str = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@ -84,8 +83,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
Response: The response object.
"""
file: str | None = request.match_info.get("file")
mode: str | None = request.match_info.get("mode")
file: str = request.match_info.get("file")
mode: str = request.match_info.get("mode")
if mode not in ["video", "subtitle"]:
return web.json_response(
@ -95,14 +94,13 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
duration: float | None = None
duration_arg = request.query.get("duration", None)
duration = request.query.get("duration", None)
if "subtitle" in mode:
if not duration_arg:
if not duration:
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
duration = float(duration_arg)
duration = float(duration)
base_path: str = config.base_path.rstrip("/")
@ -126,19 +124,11 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
if "subtitle" in mode:
if duration is None:
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
text = await cls.make_subtitle(file=realFile, duration=duration)
else:
text = await cls.make_stream(file=realFile)
except StreamingError as e:
LOG.exception(
"Failed to create %s streaming playlist for '%s': %s.",
mode,
file,
e,
extra={"route": "streaming.playlist", "file_path": file, "mode": mode, "exception_type": type(e).__name__},
)
LOG.exception(e)
return web.json_response(data={"error": str(e)}, status=web.HTTPNotFound.status_code)
return web.Response(
@ -153,7 +143,7 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
@route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream")
async def segments_stream(request: Request, config: Config, app: web.Application) -> StreamResponse:
async def segments_stream(request: Request, config: Config, app: web.Application) -> Response:
"""
Get the segments.
@ -166,9 +156,9 @@ async def segments_stream(request: Request, config: Config, app: web.Application
Response: The response object.
"""
file: str | None = request.match_info.get("file")
segment: str | None = request.match_info.get("segment")
sd: str | None = request.query.get("sd")
file: str = request.match_info.get("file")
segment: int = request.match_info.get("segment")
sd: int = request.query.get("sd")
vc: int = int(request.query.get("vc", 0))
ac: int = int(request.query.get("ac", 0))
@ -245,7 +235,7 @@ async def subtitles_get(request: Request, config: Config, app: web.Application)
Response: The response object.
"""
file: str | None = request.match_info.get("file")
file: str = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@ -287,134 +277,3 @@ async def subtitles_get(request: Request, config: Config, app: web.Application)
},
status=web.HTTPOk.status_code,
)
@route("GET", "api/player/subtitles/manifest/{file:.*}", "subtitles_manifest_get")
async def subtitles_manifest_get(request: Request, config: Config, app: web.Application) -> Response:
"""
Get subtitle track metadata for a media file.
Args:
request (Request): The request object.
config (Config): The configuration instance.
app (web.Application): The aiohttp application instance.
Returns:
Response: The response object.
"""
file: str | None = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
realFile, status = get_file(download_path=config.download_path, file=file)
if web.HTTPFound.status_code == status:
return Response(
status=status,
headers={
"Location": str(
app.router["subtitles_manifest_get"].url_for(
file=str(realFile).replace(config.download_path, "").strip("/")
)
),
},
)
if web.HTTPNotFound.status_code == status:
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
tracks = [
{
"lang": track.lang,
"name": track.name,
"source_format": track.source_format,
"delivery_format": track.delivery_format,
"renderer": track.renderer,
"url": str(
app.router["subtitles_track_get"].url_for(
source_format=track.source_format,
file=str(track.file).replace(config.download_path, "").strip("/"),
)
),
}
for track in get_subtitle_tracks(realFile)
]
return web.json_response(data={"subtitles": tracks}, status=web.HTTPOk.status_code)
@route("GET", "api/player/subtitles/{source_format}/{file:.*}", "subtitles_track_get")
async def subtitles_track_get(request: Request, config: Config, app: web.Application) -> Response:
"""
Get a subtitle file using its preferred delivery format.
Args:
request (Request): The request object.
config (Config): The configuration instance.
app (web.Application): The aiohttp application instance.
Returns:
Response: The response object.
"""
file: str | None = request.match_info.get("file")
source_format: str | None = request.match_info.get("source_format")
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
fmt = Subtitle.normalize_format(source_format or "")
if fmt is None:
return web.json_response(
data={"error": "Only vtt, srt, and ass subtitle formats are supported."},
status=web.HTTPBadRequest.status_code,
)
realFile, status = get_file(download_path=config.download_path, file=file)
if web.HTTPFound.status_code == status:
return Response(
status=status,
headers={
"Location": str(
app.router["subtitles_track_get"].url_for(
source_format=fmt,
file=str(realFile).replace(config.download_path, "").strip("/"),
)
),
},
)
if web.HTTPNotFound.status_code == status:
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
if Subtitle.normalize_format(realFile.suffix) != fmt:
return web.json_response(
data={"error": f"Subtitle file '{file}' does not match requested source format '{fmt}'."},
status=web.HTTPBadRequest.status_code,
)
mtime = realFile.stat().st_mtime
if request.if_modified_since and request.if_modified_since.timestamp() == mtime:
lastMod = time.strftime("%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple())
return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod})
body, content_type = await Subtitle().make_delivery(file=realFile)
return web.Response(
body=body,
headers={
"Content-Type": content_type,
"X-Accel-Buffering": "no",
"Access-Control-Allow-Origin": "*",
"Pragma": "public",
"Cache-Control": f"public, max-age={time.time() + 31536000}",
"Last-Modified": time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple()
),
"Expires": time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple()
),
},
status=web.HTTPOk.status_code,
)

View file

@ -1,17 +1,16 @@
import tempfile
from pathlib import Path
from unittest.mock import AsyncMock, patch
import pytest
from app.tests.helpers import make_test_temp_dir
class TestFFProbe:
"""Test the ffprobe module functionality."""
def setup_method(self):
"""Set up test files."""
self.temp_dir = str(make_test_temp_dir("ffprobe"))
self.temp_dir = tempfile.mkdtemp()
self.test_file = Path(self.temp_dir) / "test_video.mp4"
self.test_file.touch()
@ -84,11 +83,13 @@ class TestFFProbe:
# First call should execute the function
result1 = await ffprobe(str(self.test_file))
assert result1 is not None
assert isinstance(result1.metadata, dict)
first_call_count = call_count
# Second call with same argument should use cached result
result2 = await ffprobe(str(self.test_file))
assert result2 is not None
assert isinstance(result2.metadata, dict)
assert call_count == first_call_count, (
@ -117,7 +118,7 @@ class TestFFProbe:
# Test with Path object
result = await ffprobe(self.test_file)
assert result.metadata == {"duration": "10.0"}
assert result is not None
def test_ffprobe_result_properties(self):
"""Test FFProbeResult object properties."""

View file

@ -64,7 +64,7 @@ async def test_make_stream_basic_ok_codecs(tmp_path: Path, monkeypatch: pytest.M
@pytest.mark.asyncio
async def test_make_stream_transcode_flags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
async def test_make_stream_transcode_flags_and_remainder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
base = tmp_path / "dl"
base.mkdir()
media = base / "v.mp4"
@ -107,7 +107,7 @@ async def test_make_stream_transcode_flags(tmp_path: Path, monkeypatch: pytest.M
@pytest.mark.asyncio
async def test_make_stream_no_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
async def test_make_stream_raises_without_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
base = tmp_path / "dl"
base.mkdir()
media = base / "v.mp4"

View file

@ -99,7 +99,7 @@ async def test_make_playlist_with_subtitles(tmp_path: Path, monkeypatch: pytest.
@pytest.mark.asyncio
async def test_make_playlist_no_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
async def test_make_playlist_raises_without_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
base = tmp_path / "downloads"
base.mkdir()
media = base / "file.mp4"

View file

@ -1,13 +1,13 @@
import asyncio
import hashlib
import logging
import tempfile
from pathlib import Path
from typing import Any
import pytest
from aiohttp import web
from app.features.streaming.library.segments import Segments
from app.tests.helpers import get_test_system_temp_root
class DummyFF:
@ -22,10 +22,6 @@ class DummyFF:
return self._a
def _ffmpeg_input_path(args: list[str]) -> Path:
return Path(args[args.index("-i") + 1].removeprefix("file:"))
@pytest.mark.asyncio
async def test_build_ffmpeg_args_video_and_audio(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
# Create a dummy media file
@ -57,7 +53,8 @@ async def test_build_ffmpeg_args_video_and_audio(tmp_path: Path, monkeypatch: py
assert captured_args, "ffmpeg was not invoked"
args = captured_args[0]
tmp_file = _ffmpeg_input_path(args)
# Compute expected symlink path used by Segments
tmpFile = Path(tempfile.gettempdir()).joinpath(f"ytptube_stream.{hashlib.sha256(str(media).encode()).hexdigest()}")
# Start time is duration * index with 6 decimals for non-zero index
assert "-ss" in args
@ -67,9 +64,7 @@ async def test_build_ffmpeg_args_video_and_audio(tmp_path: Path, monkeypatch: py
assert args[args.index("-t") + 1] == f"{5.5:.6f}"
# Input uses file:<symlink>
assert "-i" in args
assert tmp_file.parent == get_test_system_temp_root()
assert tmp_file.name.startswith("ytptube_stream.")
assert not tmp_file.exists()
assert args[args.index("-i") + 1] == f"file:{tmpFile}"
# Includes video and audio mapping and codecs
assert "-map" in args
assert "0:v:0" in args
@ -149,18 +144,18 @@ class _FakeProc:
self.killed = True
class _FakeResp(web.StreamResponse):
def __init__(self, fail_with: BaseException | None = None) -> None:
class _FakeResp:
def __init__(self, fail_with: Exception | None = None) -> None:
self.data: bytearray = bytearray()
self.eof = False
self._exc = fail_with
async def write(self, data: bytes | bytearray | memoryview, *_args: Any, **_kwargs: Any) -> None:
async def write(self, data: bytes) -> None:
if self._exc:
raise self._exc
self.data.extend(data)
async def write_eof(self, *_args: Any, **_kwargs: Any) -> None:
async def write_eof(self) -> None:
self.eof = True
@ -173,7 +168,7 @@ class _FakeProcFail(_FakeProc):
@pytest.mark.asyncio
async def test_build_ffmpeg_args_no_dri(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
async def test_build_ffmpeg_args_no_dri_falls_back_to_software(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
media = tmp_path / "file.mp4"
media.write_bytes(b"data")
@ -220,7 +215,7 @@ async def test_build_ffmpeg_args_no_dri(tmp_path: Path, monkeypatch: pytest.Monk
@pytest.mark.asyncio
async def test_stream_gpu_fallback(
async def test_stream_gpu_failure_falls_back_to_software(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
async def fake_ffprobe(_file: Path):
@ -256,14 +251,14 @@ async def test_stream_gpu_fallback(
# Encourage GPU preference
seg.vcodec = "" # empty -> try GPUs first
resp = _FakeResp()
with caplog.at_level(logging.WARNING, logger="ytptube"):
with caplog.at_level(logging.WARNING, logger="player.segments"):
await seg.stream(tmp_path / "file.mp4", resp)
# Ensure fallback path streamed data
assert len(resp.data) > 0
# Ensure we logged the reason for GPU failure (message text may vary)
assert any(
("hardware encoder" in r.message.lower()) or ("transcoding has failed" in r.message) for r in caplog.records
("Hardware encoder failed" in r.message) or ("transcoding has failed" in r.message) for r in caplog.records
)
assert any("nvenc failure" in r.message for r in caplog.records)
# Verify second invocation switched codec to a safe fallback (software)
@ -343,9 +338,6 @@ async def test_stream_gpu_fallback_switches_codec(monkeypatch: pytest.MonkeyPatc
@pytest.mark.asyncio
async def test_stream_normal_flow(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
media = tmp_path / "file.mp4"
media.write_bytes(b"data")
async def fake_ffprobe(_file: Path):
return DummyFF(v=True, a=True)
@ -354,21 +346,16 @@ async def test_stream_normal_flow(monkeypatch: pytest.MonkeyPatch, tmp_path: Pat
# Process that yields two chunks and then EOF
proc = _FakeProc([b"abc", b"def", b""])
captured_args: list[list[str]] = []
async def fake_create_subprocess_exec(*args: Any, **_kwargs: Any):
captured_args.append(list(args[1:]))
async def fake_create_subprocess_exec(*_args: Any, **_kwargs: Any):
return proc
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec)
seg = Segments(download_path=str(tmp_path), index=0, duration=1.0, vconvert=True, aconvert=True)
resp = _FakeResp()
await seg.stream(media, resp)
await seg.stream(tmp_path / "file.mp4", resp)
assert bytes(resp.data) == b"abcdef"
assert not _ffmpeg_input_path(captured_args[0]).exists()
assert media.exists()
# EOF behavior may differ; don't require True
@ -381,10 +368,7 @@ async def test_stream_client_reset(monkeypatch: pytest.MonkeyPatch, tmp_path: Pa
proc = _FakeProc([b"abc", b"def"]) # will attempt to write and fail
captured_args: list[list[str]] = []
async def fake_create_subprocess_exec(*args: Any, **_kwargs: Any):
captured_args.append(list(args[1:]))
async def fake_create_subprocess_exec(*_args: Any, **_kwargs: Any):
return proc
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec)
@ -396,7 +380,6 @@ async def test_stream_client_reset(monkeypatch: pytest.MonkeyPatch, tmp_path: Pa
# Should not write EOF due to client disconnect
assert resp.eof is False
assert not _ffmpeg_input_path(captured_args[0]).exists()
@pytest.mark.asyncio

View file

@ -4,7 +4,7 @@ from unittest.mock import patch
import pytest
from app.features.streaming.library.subtitle import Subtitle, get_subtitle_tracks, ms_to_timestamp
from app.features.streaming.library.subtitle import Subtitle, ms_to_timestamp
class TestMsToTimestamp:
@ -24,12 +24,12 @@ class TestMsToTimestamp:
@pytest.mark.asyncio
async def test_make_unsupported_extension(tmp_path: Path) -> None:
file = tmp_path / "sub.txt"
file.write_text("not a subtitle")
srt = tmp_path / "sub.txt"
srt.write_text("not a subtitle")
subtitle = Subtitle()
sub = Subtitle()
with pytest.raises(Exception, match="subtitle type is not supported"):
await subtitle.make(file)
await sub.make(srt)
@pytest.mark.asyncio
@ -38,51 +38,11 @@ async def test_make_vtt_reads_file(tmp_path: Path) -> None:
content = "WEBVTT\n\n00:00:00.00 --> 00:00:01.00\nHello"
vtt.write_text(content)
subtitle = Subtitle()
out = await subtitle.make(vtt)
sub = Subtitle()
out = await sub.make(vtt)
assert out == content
@pytest.mark.asyncio
async def test_make_delivery_ass(tmp_path: Path) -> None:
ass = tmp_path / "file.ass"
content = "[Script Info]\nTitle: Demo\n"
ass.write_text(content, encoding="utf-8")
subtitle = Subtitle()
out, media_type = await subtitle.make_delivery(ass)
assert out == content
assert media_type == "text/x-ssa; charset=UTF-8"
def test_tracks_order(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
media = tmp_path / "video.mkv"
media.write_text("x", encoding="utf-8")
ass_file = tmp_path / "video.ass"
ass_file.write_text("ass", encoding="utf-8")
vtt_file = tmp_path / "video.vtt"
vtt_file.write_text("WEBVTT\n\n", encoding="utf-8")
srt_file = tmp_path / "video.en.srt"
srt_file.write_text("1\n00:00:00,000 --> 00:00:01,000\nHello\n", encoding="utf-8")
monkeypatch.setattr(
"app.features.streaming.library.subtitle.get_file_sidecar",
lambda _file: {
"subtitle": [
{"file": ass_file, "lang": "en", "name": "ASS"},
{"file": srt_file, "lang": "en", "name": "SRT"},
{"file": vtt_file, "lang": "en", "name": "VTT"},
]
},
)
tracks = get_subtitle_tracks(media)
assert [track.source_format for track in tracks] == ["vtt", "srt", "ass"]
assert [track.delivery_format for track in tracks] == ["vtt", "vtt", "ass"]
assert [track.renderer for track in tracks] == ["native", "native", "assjs"]
class _DummySubs:
def __init__(self, events):
self.events = events
@ -101,13 +61,13 @@ async def test_make_no_events_raises(tmp_path: Path) -> None:
with patch("app.features.streaming.library.subtitle.pysubs2.load") as mock_load:
mock_load.return_value = _DummySubs(events=[])
subtitle = Subtitle()
sub = Subtitle()
with pytest.raises(Exception, match="No subtitle events were found"):
await subtitle.make(srt)
await sub.make(srt)
@pytest.mark.asyncio
async def test_make_single_vtt(tmp_path: Path) -> None:
async def test_make_single_event_returns_vtt(tmp_path: Path) -> None:
srt = tmp_path / "sub.ass"
srt.write_text("dummy")
@ -115,14 +75,14 @@ async def test_make_single_vtt(tmp_path: Path) -> None:
d = _DummySubs(events=[single])
with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
subtitle = Subtitle()
out = await subtitle.make(srt)
sub = Subtitle()
out = await sub.make(srt)
assert out == "OUT"
assert d.snapshot == [1000], "Snapshot should contain the single event"
@pytest.mark.asyncio
async def test_make_two_events_same_end(tmp_path: Path) -> None:
async def test_make_two_events_pop_first_when_ends_equal(tmp_path: Path) -> None:
srt = tmp_path / "sub.srt"
srt.write_text("dummy")
@ -131,14 +91,14 @@ async def test_make_two_events_same_end(tmp_path: Path) -> None:
d = _DummySubs(events=[e1, e2])
with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
subtitle = Subtitle()
out = await subtitle.make(srt)
sub = Subtitle()
out = await sub.make(srt)
assert out == "OUT"
assert d.snapshot == [5000], "Since ends are equal, first should be popped => only last remains"
@pytest.mark.asyncio
async def test_make_two_events_keep_both(tmp_path: Path) -> None:
async def test_make_two_events_no_pop_when_different(tmp_path: Path) -> None:
srt = tmp_path / "sub.srt"
srt.write_text("dummy")
@ -147,7 +107,7 @@ async def test_make_two_events_keep_both(tmp_path: Path) -> None:
d = _DummySubs(events=[e1, e2])
with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
subtitle = Subtitle()
out = await subtitle.make(srt)
sub = Subtitle()
out = await sub.make(srt)
assert out == "OUT"
assert d.snapshot == [5000, 6000], "Both remain since ends differ"

View file

@ -1,132 +0,0 @@
from pathlib import Path
from types import SimpleNamespace
import pytest
from aiohttp import web
from aiohttp.test_utils import make_mocked_request
from app.features.streaming import router
from app.library.config import Config
class _DummyRoute:
def __init__(self, pattern: str) -> None:
self.pattern = pattern
def url_for(self, **params: str) -> str:
path = self.pattern
for key, value in params.items():
path = path.replace(f"{{{key}:.*}}", value)
path = path.replace(f"{{{key}}}", value)
return path
class _DummyRouter(dict[str, _DummyRoute]):
def __init__(self) -> None:
super().__init__(
{
"subtitles_manifest_get": _DummyRoute("/api/player/subtitles/manifest/{file:.*}"),
"subtitles_track_get": _DummyRoute("/api/player/subtitles/{source_format}/{file:.*}"),
"subtitles_get": _DummyRoute("/api/player/subtitle/{file:.*}.vtt"),
}
)
def _make_request(path: str, *, match_info: dict[str, str]) -> web.Request:
return make_mocked_request("GET", path, app=SimpleNamespace(router=_DummyRouter()), match_info=match_info)
@pytest.mark.asyncio
async def test_subtitles_manifest_order(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
config = Config.get_instance()
config.download_path = str(tmp_path)
media = tmp_path / "video.mp4"
media.write_text("x", encoding="utf-8")
monkeypatch.setattr(
"app.features.streaming.router.get_file",
lambda **_kwargs: (media, web.HTTPOk.status_code),
)
monkeypatch.setattr(
"app.features.streaming.router.get_subtitle_tracks",
lambda _file: [
SimpleNamespace(
lang="en",
name="English VTT",
source_format="vtt",
delivery_format="vtt",
renderer="native",
file=tmp_path / "video.vtt",
),
SimpleNamespace(
lang="en",
name="English ASS",
source_format="ass",
delivery_format="ass",
renderer="assjs",
file=tmp_path / "video.ass",
),
],
)
req = _make_request(
"/api/player/subtitles/manifest/video.mp4",
match_info={"file": "video.mp4"},
)
response = await router.subtitles_manifest_get(req, config, req.app)
assert response.status == web.HTTPOk.status_code
body = response.text
assert '"source_format": "vtt"' in body
assert '"renderer": "native"' in body
assert '"source_format": "ass"' in body
assert '"renderer": "assjs"' in body
assert body.index('"source_format": "vtt"') < body.index('"source_format": "ass"')
@pytest.mark.asyncio
async def test_subtitles_track_get_ass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
config = Config.get_instance()
config.download_path = str(tmp_path)
subtitle = tmp_path / "video.ass"
subtitle.write_text("[Script Info]\nTitle: Demo\n", encoding="utf-8")
monkeypatch.setattr(
"app.features.streaming.router.get_file",
lambda **_kwargs: (subtitle, web.HTTPOk.status_code),
)
req = _make_request(
"/api/player/subtitles/ass/video.ass",
match_info={"source_format": "ass", "file": "video.ass"},
)
response = await router.subtitles_track_get(req, config, req.app)
assert response.status == web.HTTPOk.status_code
assert response.text == "[Script Info]\nTitle: Demo\n"
assert response.headers["Content-Type"] == "text/x-ssa; charset=UTF-8"
@pytest.mark.asyncio
async def test_subtitles_track_bad_format(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
config = Config.get_instance()
config.download_path = str(tmp_path)
subtitle = tmp_path / "video.ass"
subtitle.write_text("[Script Info]\nTitle: Demo\n", encoding="utf-8")
monkeypatch.setattr(
"app.features.streaming.router.get_file",
lambda **_kwargs: (subtitle, web.HTTPOk.status_code),
)
req = _make_request(
"/api/player/subtitles/vtt/video.ass",
match_info={"source_format": "vtt", "file": "video.ass"},
)
response = await router.subtitles_track_get(req, config, req.app)
assert response.status == web.HTTPBadRequest.status_code
assert b"does not match requested source format" in response.body

View file

@ -1,358 +0,0 @@
from __future__ import annotations
import asyncio
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock
import pytest
from app.tests.helpers import temporary_test_dir
@pytest.mark.asyncio
async def test_pick_same_stem() -> None:
from app.features.streaming.library.thumbnail import pick_local_thumb
with temporary_test_dir("thumb-local") as temp_dir:
media = temp_dir / "video.mp4"
poster = temp_dir / "poster.jpg"
thumb = temp_dir / "video.jpg"
media.write_text("video")
poster.write_text("poster")
thumb.write_text("thumb")
assert pick_local_thumb(media) == thumb
@pytest.mark.asyncio
async def test_singleflight(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-singleflight") as temp_dir:
media = temp_dir / "video.mp4"
cache_root = temp_dir / "cache"
media.write_text("video")
item_id = "item-1"
calls = {"count": 0}
async def fake_run_ffmpeg(_file: Path, output_file: Path) -> Path:
calls["count"] += 1
await asyncio.sleep(0.01)
output_file.parent.mkdir(parents=True, exist_ok=True)
output_file.write_text("image")
return output_file
monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
first, second = await asyncio.gather(
thumbnail.ensure_thumb(media, cache_root, item_id=item_id),
thumbnail.ensure_thumb(media, cache_root, item_id=item_id),
)
assert first == second
assert first is not None
assert calls["count"] == 1
assert thumbnail._IN_PROCESS == {}
@pytest.mark.asyncio
async def test_cache_hit(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-cache") as temp_dir:
media = temp_dir / "video.mp4"
cache_root = temp_dir / "cache"
media.write_text("video")
item_id = "item-1"
cache_file = cache_root / f"{item_id}.jpg"
cache_file.parent.mkdir(parents=True, exist_ok=True)
cache_file.write_text("image")
async def fake_run_ffmpeg(_file: Path, _output_file: Path) -> Path:
raise AssertionError("ffmpeg should not run when cache exists")
monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
result = await thumbnail.ensure_thumb(media, cache_root, item_id=item_id)
assert result == cache_file
@pytest.mark.asyncio
async def test_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-disabled") as temp_dir:
media = temp_dir / "video.mp4"
media.write_text("video")
monkeypatch.setattr(
thumbnail.Config,
"get_instance",
staticmethod(lambda: type("Cfg", (), {"thumb_generate": False, "thumb_sidecar": False})()),
)
result = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id="item-1")
assert result is None
@pytest.mark.asyncio
async def test_sidecar_mode(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail._SEM = None
thumbnail._SEM_LIMIT = None
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-sidecar") as temp_dir:
media = temp_dir / "video.mp4"
media.write_text("video")
sidecar = media.with_name(f"{media.name}.jpg")
monkeypatch.setattr(
thumbnail.Config,
"get_instance",
staticmethod(
lambda: type(
"Cfg",
(),
{"thumb_generate": True, "thumb_sidecar": True, "thumb_concurrency": 1},
)()
),
)
async def fake_run_ffmpeg(_file: Path, out_file: Path) -> Path:
out_file.write_text("image")
return out_file
monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
result = await thumbnail.ensure_thumb(media, temp_dir / "cache")
assert result == sidecar
assert sidecar.exists()
@pytest.mark.asyncio
async def test_miss_cache(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-miss") as temp_dir:
media = temp_dir / "video.mp4"
media.write_text("video")
item_id = "item-1"
monkeypatch.setattr(
thumbnail.Config,
"get_instance",
staticmethod(
lambda: type(
"Cfg",
(),
{"thumb_generate": True, "thumb_sidecar": False, "thumb_concurrency": 1},
)()
),
)
calls = {"count": 0}
async def fake_run_ffmpeg(_file: Path, _out_file: Path) -> Path | None:
calls["count"] += 1
return None
monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
first = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id=item_id)
second = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id=item_id)
assert first is None
assert second is None
assert calls["count"] == 1
def test_sem_limit(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._SEM = None
thumbnail._SEM_LIMIT = None
monkeypatch.setattr(
thumbnail.Config,
"get_instance",
staticmethod(
lambda: type("Cfg", (), {"thumb_concurrency": 3, "thumb_generate": True, "thumb_sidecar": False})()
),
)
sem = thumbnail._get_semaphore()
assert sem._value == 3
assert thumbnail._SEM_LIMIT == 3
def test_seek_bounds() -> None:
from app.features.streaming.library.ffprobe import FFProbeResult
from app.features.streaming.library.thumbnail import _seek_seconds
ff_info = FFProbeResult()
ff_info.metadata = {"duration": "120.0"}
assert _seek_seconds(ff_info) == 12.0
ff_info.metadata = {"duration": "2.0"}
assert _seek_seconds(ff_info) == 1.0
ff_info.metadata = {"duration": "0.05"}
assert _seek_seconds(ff_info) is None
ff_info.metadata = {}
assert _seek_seconds(ff_info) == 3.0
@pytest.mark.asyncio
async def test_cache_name_uses_item_id(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-id-name") as temp_dir:
media = temp_dir / "video.mp4"
cache_root = temp_dir / "cache"
media.write_text("video")
async def fake_run_ffmpeg(_file: Path, output_file: Path) -> Path:
output_file.parent.mkdir(parents=True, exist_ok=True)
output_file.write_text("image")
return output_file
monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
result = await thumbnail.ensure_thumb(media, cache_root, item_id="item-1")
assert result == cache_root / "item-1.jpg"
@pytest.mark.asyncio
async def test_retry_no_seek(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library.ffprobe import FFProbeResult
from app.features.streaming.library import thumbnail
with temporary_test_dir("thumb-retry") as temp_dir:
media = temp_dir / "video.mp4"
output = temp_dir / ".jpg"
media.write_text("video")
ff_info: Any = FFProbeResult()
ff_info.metadata = {"duration": "120.0"}
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))
calls: list[list[str]] = []
class DummyProc:
def __init__(self, attempt: int, out_path: Path) -> None:
self.returncode = 1 if attempt == 1 else 0
self._out_path = out_path
async def communicate(self) -> tuple[bytes, bytes]:
if self.returncode == 0:
self._out_path.write_text("image")
return b"", b""
return b"", b"seek failed"
async def fake_create_subprocess_exec(*args, **kwargs):
del kwargs
calls.append([str(arg) for arg in args])
return DummyProc(len(calls), Path(str(args[-1])))
monkeypatch.setattr(thumbnail.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
result = await thumbnail._run_ffmpeg(media, output)
assert result == output
assert len(calls) == 2
assert "-ss" in calls[0]
assert calls[0][calls[0].index("-ss") + 1] == "12.000"
assert "-ss" not in calls[1]
assert calls[0][calls[0].index("-vf") + 1] == "thumbnail=200,scale=1280:-1:force_original_aspect_ratio=decrease"
assert calls[0][-1].endswith(".tmp.jpg")
@pytest.mark.asyncio
async def test_limit_wait(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
from app.features.streaming.library.ffprobe import FFProbeResult
thumbnail._SEM = None
thumbnail._SEM_LIMIT = None
thumbnail._IN_PROCESS.clear()
monkeypatch.setattr(
thumbnail.Config,
"get_instance",
staticmethod(
lambda: type("Cfg", (), {"thumb_concurrency": 1, "thumb_generate": True, "thumb_sidecar": False})()
),
)
with temporary_test_dir("thumb-limit") as temp_dir:
media1 = temp_dir / "video1.mp4"
media2 = temp_dir / "video2.mp4"
out1 = temp_dir / "out1.jpg"
out2 = temp_dir / "out2.jpg"
media1.write_text("video")
media2.write_text("video")
ff_info: Any = FFProbeResult()
ff_info.metadata = {"duration": "60.0"}
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))
active = {"count": 0, "max": 0}
class DummyProc:
def __init__(self, out_path: Path) -> None:
self.returncode = 0
self._out_path = out_path
async def communicate(self) -> tuple[bytes, bytes]:
active["count"] += 1
active["max"] = max(active["max"], active["count"])
await asyncio.sleep(0.01)
self._out_path.write_text("image")
active["count"] -= 1
return b"", b""
async def fake_create_subprocess_exec(*args, **kwargs):
del kwargs
return DummyProc(Path(str(args[-1])))
monkeypatch.setattr(thumbnail.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
first, second = await asyncio.gather(
thumbnail._run_ffmpeg(media1, out1),
thumbnail._run_ffmpeg(media2, out2),
)
assert first == out1
assert second == out2
assert active["max"] == 1

View file

@ -1,3 +1,4 @@
# flake8: noqa: ARG004
from typing import TYPE_CHECKING, Any
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult
@ -11,7 +12,6 @@ if TYPE_CHECKING:
class BaseHandler:
@staticmethod
async def can_handle(task: HandleTask) -> bool:
_ = task
return False
@staticmethod
@ -24,7 +24,6 @@ class BaseHandler:
@staticmethod
def parse(url: str) -> Any | None:
_ = url
return None
@staticmethod

View file

@ -6,12 +6,12 @@ import asyncio
import fnmatch
import hashlib
import json
import logging
import re
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin
import httpx
import jmespath
from parsel import Selector
@ -25,16 +25,16 @@ from app.features.ytdlp.utils import get_archive_id
from app.library.cache import Cache
from app.library.config import Config
from app.library.httpx_client import Globals, build_request_headers, get_async_client, resolve_curl_transport
from app.library.log import get_logger
from ._base_handler import BaseHandler
if TYPE_CHECKING:
from pathlib import Path
import httpx
from parsel.selector import SelectorList
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("handlers.generic")
CACHE: Cache = Cache()
@ -64,15 +64,12 @@ class GenericTaskHandler(BaseHandler):
from app.features.tasks.definitions.utils import model_to_schema
repo = TaskDefinitionsRepository.get_instance()
models = await repo.all()
models = await repo.list()
cls._definitions = [model_to_schema(model) for model in models]
return cls._definitions
except Exception as exc:
LOG.exception(
"Failed to load generic task definitions.",
extra={"error": str(exc), "exception_type": type(exc).__name__},
)
LOG.error(f"Failed to load task definitions from database: {exc}")
return []
@classmethod
@ -105,15 +102,7 @@ class GenericTaskHandler(BaseHandler):
if pattern_str and re.match(pattern_str, url):
return definition
except Exception as exc:
LOG.exception(
"Failed to match a generic task definition.",
extra={
"definition": definition.name,
"url": url,
"error": str(exc),
"exception_type": type(exc).__name__,
},
)
LOG.error(f"Error while matching definition '{definition.name}': {exc}")
return None
@ -131,18 +120,13 @@ class GenericTaskHandler(BaseHandler):
"""
definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
if definition:
LOG.debug(
"Task '%s' matched a generic task definition.",
task.name,
extra={"task_name": task.name, "url": task.url, "definition": definition.name},
)
LOG.debug(f"'{task.name}': Matched generic task definition '{definition.name}'.")
return True
return False
@staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004
definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
if not definition:
return TaskFailure(message="No generic task definition matched the provided URL.")
@ -150,34 +134,14 @@ class GenericTaskHandler(BaseHandler):
ytdlp_opts: dict[str, Any] = task.get_ytdlp_opts().get_all()
target_url: str = definition.definition.request.url or task.url
LOG.debug(
"Fetching content for task '%s'.",
task.name,
extra={
"task_name": task.name,
"definition": definition.name,
"url": target_url,
"engine": definition.definition.engine.type,
},
)
LOG.debug(f"{task.name!r}: Fetching '{target_url}' using engine '{definition.definition.engine.type}'.")
try:
body_text, json_data = await GenericTaskHandler._fetch_content(
url=target_url, definition=definition, ytdlp_opts=ytdlp_opts
)
except httpx.HTTPError as exc:
return TaskFailure(message="Failed to fetch target URL.", error=str(exc))
except Exception as exc:
LOG.exception(
"Failed to fetch content for task '%s'.",
task.name,
extra={
"task_name": task.name,
"definition": definition.name,
"url": target_url,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return TaskFailure(message="Failed to fetch target URL.", error=str(exc))
if "json" == definition.definition.response.type and json_data is None:
@ -215,9 +179,7 @@ class GenericTaskHandler(BaseHandler):
continue
else:
LOG.warning(
"Task '%s' could not generate a static archive ID. Fetching it with yt-dlp.",
task.name,
extra={"definition": definition.name, "task_name": task.name, "url": url},
f"[{definition.name}]: '{task.name}': Unable to generate static archive id for '{url}' in feed. Doing real request to fetch yt-dlp archive id."
)
(info, _) = await fetch_info(
@ -225,23 +187,18 @@ class GenericTaskHandler(BaseHandler):
url=url,
no_archive=True,
no_log=True,
budget_sleep=True,
)
if not info:
LOG.error(
"Task '%s' failed to extract info to generate an archive ID. Skipping item.",
task.name,
extra={"definition": definition.name, "task_name": task.name, "url": url},
f"[{definition.name}]: '{task.name}': Failed to extract info for URL '{url}' to generate archive ID. Skipping."
)
CACHE.set(cache_key, None)
continue
if not info.get("id") or not info.get("extractor_key"):
LOG.error(
"Task '%s' returned incomplete info while generating an archive ID. Skipping item.",
task.name,
extra={"definition": definition.name, "task_name": task.name, "url": url},
f"[{definition.name}]: '{task.name}': Incomplete info extracted for URL '{url}' to generate archive ID. Skipping."
)
CACHE.set(cache_key, None)
continue
@ -337,17 +294,7 @@ class GenericTaskHandler(BaseHandler):
try:
json_data: dict[str, Any] = response.json()
except Exception as exc:
LOG.exception(
"Task definition '%s' returned invalid JSON for '%s'.",
definition.name,
url,
extra={
"definition": definition.name,
"url": url,
"error": str(exc),
"exception_type": type(exc).__name__,
},
)
LOG.error(f"Failed to decode JSON response from '{url}': {exc}")
return response.text, None
return response.text, json_data
@ -377,11 +324,7 @@ class GenericTaskHandler(BaseHandler):
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
except ImportError as exc:
LOG.exception(
"Task definition '%s' requested Selenium, but Selenium is not installed.",
definition.name,
extra={"definition": definition.name, "error": str(exc), "exception_type": type(exc).__name__},
)
LOG.error(f"Selenium engine requested but selenium is not installed: {exc!s}.")
return (None, None)
options_map: dict[str, Any] = definition.definition.engine.options
@ -394,12 +337,7 @@ class GenericTaskHandler(BaseHandler):
browser: str = str(options_map.get("browser", "chrome")).lower()
if "chrome" != browser:
LOG.error(
"Task definition '%s' requested unsupported Selenium browser '%s'.",
definition.name,
browser,
extra={"definition": definition.name, "browser": browser},
)
LOG.error(f"Unsupported selenium browser '{browser}'. Only 'chrome' is supported.")
return (None, None)
arguments: list[str] | str = options_map.get("arguments", ["--headless", "--disable-gpu"])
@ -481,9 +419,7 @@ class GenericTaskHandler(BaseHandler):
link_values: list[str] = extracted.get("link", [])
if not link_values:
LOG.debug(
"Definition '%s' produced no link values.", definition.name, extra={"definition": definition.name}
)
LOG.debug(f"Definition '{definition.name}' produced no link values.")
return []
total_items: int = len(link_values)
@ -518,11 +454,7 @@ class GenericTaskHandler(BaseHandler):
base_url: str,
) -> list[dict[str, str]]:
if json_data is None:
LOG.debug(
"Definition '%s' expects JSON but no data was parsed.",
definition.name,
extra={"definition": definition.name},
)
LOG.debug(f"Definition '{definition.name}' expects JSON but no data was parsed.")
return []
if definition.definition.parse.get("items"):
@ -561,11 +493,7 @@ class GenericTaskHandler(BaseHandler):
container_type = container.get("type", "css")
container_selector = container.get("selector") or container.get("expression") or ""
if not container_selector:
LOG.error(
"Task definition '%s' is missing an item container selector.",
definition.name,
extra={"definition": definition.name},
)
LOG.error(f"Container missing selector/expression. Definition '{definition.name}'.")
return []
container_fields = container.get("fields", {})
@ -619,11 +547,7 @@ class GenericTaskHandler(BaseHandler):
container_fields = container.get("fields", {})
if "jsonpath" != container_type:
LOG.error(
"JSON response requires container selector type 'jsonpath'. Definition '%s'.",
definition.name,
extra={"definition": definition.name, "container_type": container_type},
)
LOG.error(f"JSON response requires container selector type 'jsonpath'. Definition '{definition.name}'.")
return []
nodes: Any = GenericTaskHandler._json_search(json_data, container_selector)
@ -679,16 +603,7 @@ class GenericTaskHandler(BaseHandler):
try:
pattern: re.Pattern[str] = re.compile(rule.expression, re.MULTILINE | re.DOTALL)
except re.error as exc:
LOG.exception(
"Invalid regex expression '%s'.",
rule.expression,
extra={
"field": field,
"expression": rule.expression,
"error": str(exc),
"exception_type": type(exc).__name__,
},
)
LOG.error(f"Invalid regex expression '{rule.expression}': {exc}")
return values
for match in pattern.finditer(target):
@ -699,12 +614,7 @@ class GenericTaskHandler(BaseHandler):
return values
LOG.error(
"Unsupported extraction type '%s' for JSON data in field '%s'.",
rule.type,
field,
extra={"field": field, "rule_type": rule.type},
)
LOG.error(f"Unsupported extraction type '{rule.type}' for JSON data in field '{field}'.")
return values
@staticmethod
@ -712,11 +622,7 @@ class GenericTaskHandler(BaseHandler):
try:
return jmespath.search(expression, data)
except Exception as exc:
LOG.exception(
"JSONPath search failed for expression '%s'.",
expression,
extra={"expression": expression, "error": str(exc), "exception_type": type(exc).__name__},
)
LOG.error(f"JSONPath search failed for expression '{expression}': {exc}")
return None
@staticmethod
@ -751,16 +657,7 @@ class GenericTaskHandler(BaseHandler):
try:
pattern: re.Pattern[str] = re.compile(rule.expression, re.MULTILINE | re.DOTALL)
except re.error as exc:
LOG.exception(
"Invalid regex expression '%s'.",
rule.expression,
extra={
"field": field,
"expression": rule.expression,
"error": str(exc),
"exception_type": type(exc).__name__,
},
)
LOG.error(f"Invalid regex expression '{rule.expression}': {exc}")
return values
for match in pattern.finditer(html):
@ -772,7 +669,7 @@ class GenericTaskHandler(BaseHandler):
return values
if "jsonpath" == rule.type:
LOG.error("Field '%s' uses 'jsonpath' on a non-JSON response.", field, extra={"field": field})
LOG.error("Extraction type 'jsonpath' is only valid for JSON responses.")
return values
selection: SelectorList[Selector] = (
@ -804,12 +701,7 @@ class GenericTaskHandler(BaseHandler):
try:
return match.group(attribute)
except (IndexError, KeyError):
LOG.debug(
"Regex group '%s' not found in pattern '%s'.",
attribute,
match.re.pattern,
extra={"attribute": attribute, "pattern": match.re.pattern},
)
LOG.debug(f"Regex group '{attribute}' not found in pattern '{match.re.pattern}'.")
return None
if match.groupdict():

View file

@ -1,23 +1,20 @@
import hashlib
import logging
import re
from typing import TYPE_CHECKING, Any
from xml.etree.ElementTree import Element
import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.extractor import fetch_info
from app.features.ytdlp.utils import get_archive_id
from app.library.cache import Cache
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
if TYPE_CHECKING:
from xml.etree.ElementTree import Element
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("handlers.rss")
CACHE: Cache = Cache()
@ -29,11 +26,7 @@ class RssGenericHandler(BaseHandler):
@staticmethod
async def can_handle(task: HandleTask) -> bool:
LOG.debug(
"Checking if task '%s' uses a parsable RSS feed.",
task.name,
extra={"task_name": task.name, "url": task.url},
)
LOG.debug(f"'{task.name}': Checking if task URL is parsable RSS feed: {task.url}")
return RssGenericHandler.parse(task.url) is not None
@staticmethod
@ -57,11 +50,7 @@ class RssGenericHandler(BaseHandler):
from defusedxml.ElementTree import fromstring
feed_url: str = parsed["url"]
LOG.debug(
"Fetching RSS/Atom feed for task '%s'.",
task.name,
extra={"task_name": task.name, "feed_url": feed_url},
)
LOG.debug(f"'{task.name}': Fetching RSS/Atom feed from {feed_url}")
response = await RssGenericHandler.request(url=feed_url, ytdlp_opts=params)
response.raise_for_status()
@ -82,12 +71,7 @@ class RssGenericHandler(BaseHandler):
# Try to parse as Atom feed first
entries = root.findall("atom:entry", ns)
if entries:
LOG.debug(
"'%s': Detected Atom feed format with %s entries",
task.name,
len(entries),
extra={"task_name": task.name, "feed_url": feed_url, "entry_count": len(entries)},
)
LOG.debug(f"'{task.name}': Detected Atom feed format with {len(entries)} entries")
for entry in entries:
link_elem: Element | None = entry.find("atom:link[@rel='alternate']", ns)
if link_elem is None:
@ -98,11 +82,7 @@ class RssGenericHandler(BaseHandler):
url = link_elem.get("href", "")
if not url:
LOG.warning(
"'%s': Atom entry missing URL. Skipping.",
task.name,
extra={"task_name": task.name, "feed_url": feed_url},
)
LOG.warning(f"'{task.name}': Atom entry missing URL. Skipping.")
continue
title_elem: Element | None = entry.find("atom:title", ns)
@ -116,12 +96,7 @@ class RssGenericHandler(BaseHandler):
else:
# Try to parse as RSS feed
rss_items = root.findall(".//item")
LOG.debug(
"'%s': Detected RSS feed format with %s items",
task.name,
len(rss_items),
extra={"task_name": task.name, "feed_url": feed_url, "entry_count": len(rss_items)},
)
LOG.debug(f"'{task.name}': Detected RSS feed format with {len(rss_items)} items")
for item in rss_items:
# Try different link element names (link, url, media:content)
@ -142,11 +117,7 @@ class RssGenericHandler(BaseHandler):
url = enclosure_elem.get("url", "")
if not url:
LOG.warning(
"'%s': RSS item missing URL. Skipping.",
task.name,
extra={"task_name": task.name, "feed_url": feed_url},
)
LOG.warning(f"'{task.name}': RSS item missing URL. Skipping.")
continue
title_elem = item.find("title")
@ -161,14 +132,12 @@ class RssGenericHandler(BaseHandler):
return feed_url, items, real_count
@staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
"""
Extract items from an RSS/Atom feed.
Args:
task (Task): The task containing the feed URL.
config (Config | None): Optional handler configuration.
Returns:
TaskResult | TaskFailure: Extraction result with parsed items or failure information.
@ -182,19 +151,8 @@ class RssGenericHandler(BaseHandler):
try:
feed_url, items, real_count = await RssGenericHandler._get(task, params, parsed)
except httpx.HTTPError as exc:
return TaskFailure(message="Failed to fetch RSS/Atom feed.", error=str(exc))
except Exception as exc:
LOG.exception(
"Failed to fetch RSS/Atom feed for task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return TaskFailure(message="Failed to fetch RSS/Atom feed.", error=str(exc))
task_items: list[TaskItem] = []
@ -214,17 +172,12 @@ class RssGenericHandler(BaseHandler):
if CACHE.has(cache_key):
archive_id = CACHE.get(cache_key)
if not archive_id:
LOG.debug(
"Task '%s' has a cached archive ID lookup failure. Skipping item.",
task.name,
extra={"task_name": task.name, "url": url},
)
LOG.debug(f"'{task.name}': Cached failure for URL '{url}'. Skipping.")
continue
else:
LOG.warning(
"Task '%s' could not generate a static archive ID. Fetching it with yt-dlp.",
task.name,
extra={"task_name": task.name, "url": url},
f"'{task.name}': Unable to generate static archive ID for '{url}' in feed. "
"Doing real request to fetch yt-dlp archive ID."
)
(info, _) = await fetch_info(
@ -232,23 +185,18 @@ class RssGenericHandler(BaseHandler):
url=url,
no_archive=True,
no_log=True,
budget_sleep=True,
)
if not info:
LOG.error(
"Task '%s' failed to extract info to generate an archive ID. Skipping item.",
task.name,
extra={"task_name": task.name, "url": url},
f"'{task.name}': Failed to extract info for URL '{url}' to generate archive ID. Skipping."
)
CACHE.set(cache_key, None)
continue
if not info.get("id") or not info.get("extractor_key"):
LOG.error(
"Task '%s' returned incomplete info while generating an archive ID. Skipping item.",
task.name,
extra={"task_name": task.name, "url": url},
f"'{task.name}': Incomplete info extracted for URL '{url}' to generate archive ID. Skipping."
)
CACHE.set(cache_key, None)
continue

View file

@ -1,15 +1,12 @@
import logging
import re
import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("handlers.tver")
class TverHandler(BaseHandler):
@ -25,11 +22,7 @@ class TverHandler(BaseHandler):
@staticmethod
async def can_handle(task: HandleTask) -> bool:
LOG.debug(
"Checking if task '%s' uses a parsable Tver series URL.",
task.name,
extra={"task_name": task.name, "url": task.url},
)
LOG.debug(f"Checking if task '{task.name}' is using parsable Tver series URL: {task.url}")
return TverHandler.parse(task.url) is not None
@staticmethod
@ -60,11 +53,7 @@ class TverHandler(BaseHandler):
return {"platform_uid": platform_uid, "platform_token": platform_token}
except Exception as exc:
LOG.warning(
"Failed to create Tver session.",
extra={"error": str(exc), "exception_type": type(exc).__name__},
exc_info=True,
)
LOG.warning(f"Failed to create tver session: {exc}")
return None
@staticmethod
@ -92,11 +81,7 @@ class TverHandler(BaseHandler):
feed_url = TverHandler.SERIES_API.format(id=series_id)
LOG.debug(
"Fetching Tver episodes for task '%s'.",
task.name,
extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
)
LOG.debug(f"Fetching '{task.name}' episodes from tver series {series_id}.")
response = await TverHandler.request(
url=feed_url,
@ -118,11 +103,7 @@ class TverHandler(BaseHandler):
try:
contents = data.get("result", {}).get("contents", [])
if not contents:
LOG.warning(
"No contents found in Tver series response for '%s'.",
task.name,
extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
)
LOG.warning(f"No contents found in tver series response for '{task.name}'.")
return feed_url, items, has_items
season_block = contents[0] if contents else {}
@ -133,13 +114,9 @@ class TverHandler(BaseHandler):
continue
content = episode_data.get("content", {})
episode_id = content.pop("id", None)
episode_id = content.pop("id")
if not episode_id:
LOG.warning(
"Episode missing ID in '%s' feed. Skipping.",
task.name,
extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
)
LOG.warning(f"Episode missing ID in '{task.name}' feed. Skipping.")
continue
url = f"https://tver.jp/episodes/{episode_id}"
@ -150,9 +127,7 @@ class TverHandler(BaseHandler):
archive_id = id_dict.get("archive_id")
if not archive_id:
LOG.warning(
"Task '%s' could not compute an archive ID for an episode. Skipping item.",
task.name,
extra={"task_name": task.name, "series_id": series_id, "episode_id": episode_id, "url": url},
f"Could not compute archive ID for episode '{episode_id}' in '{task.name}' feed. Skipping."
)
continue
@ -162,24 +137,12 @@ class TverHandler(BaseHandler):
)
except Exception as exc:
LOG.warning(
"Failed to parse Tver episodes for task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"series_id": series_id,
"error": str(exc),
"exception_type": type(exc).__name__,
},
exc_info=True,
)
LOG.warning(f"Error parsing tver episodes for '{task.name}': {exc}")
return feed_url, items, has_items
@staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
series_id: str | None = TverHandler.parse(task.url)
if not series_id:
return TaskFailure(message="Unrecognized Tver series URL.")
@ -188,20 +151,8 @@ class TverHandler(BaseHandler):
try:
feed_url, items, has_items = await TverHandler._collect_feed(task, params, series_id)
except httpx.HTTPError as exc:
return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc))
except Exception as exc:
LOG.exception(
"Failed to fetch Tver feed for task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"series_id": series_id,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc))
task_items: list[TaskItem] = []
@ -210,11 +161,10 @@ class TverHandler(BaseHandler):
if not (url := entry.get("url")):
continue
archive_id: str | None = entry.get("archive_id")
metadata = entry.get("metadata", {})
if not isinstance(metadata, dict):
metadata = {}
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
archive_id: str = entry.get("archive_id")
task_items.append(
TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=entry.get("metadata", {}))
)
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})

View file

@ -1,20 +1,17 @@
import logging
import re
from typing import TYPE_CHECKING
from xml.etree.ElementTree import Element
import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
if TYPE_CHECKING:
from xml.etree.ElementTree import Element
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("handlers.twitch")
class TwitchHandler(BaseHandler):
@ -24,11 +21,7 @@ class TwitchHandler(BaseHandler):
@staticmethod
async def can_handle(task: HandleTask) -> bool:
LOG.debug(
"Checking if task '%s' uses a parsable Twitch URL.",
task.name,
extra={"task_name": task.name, "url": task.url},
)
LOG.debug(f"Checking if task '{task.name}' is using parsable Twitch URL: {task.url}")
return TwitchHandler.parse(task.url) is not None
@staticmethod
@ -41,7 +34,7 @@ class TwitchHandler(BaseHandler):
feed_url: str = TwitchHandler.FEED.format(handle=handle_name)
LOG.debug("Fetching '%s' feed.", task.name, extra={"task_name": task.name, "feed_url": feed_url})
LOG.debug(f"Fetching '{task.name}' feed.")
response = await TwitchHandler.request(url=feed_url, ytdlp_opts=params)
response.raise_for_status()
@ -53,22 +46,14 @@ class TwitchHandler(BaseHandler):
link_elem: Element[str] | None = entry.find("link")
url: str = link_elem.text.strip() if link_elem is not None and link_elem.text else ""
if not url:
LOG.warning(
"Entry in '%s' feed is missing URL. Skipping entry.",
task.name,
extra={"task_name": task.name, "feed_url": feed_url},
)
LOG.warning(f"Entry in '{task.name}' feed is missing URL. Skipping entry.")
continue
match: re.Match[str] | None = re.search(
r"^https?://(?:www\.)?twitch\.tv/videos/(?P<id>\d+)(?:[/?].*)?$", url
)
if not match:
LOG.warning(
"Task '%s' produced a feed entry that does not look like a Twitch VOD link. Skipping entry.",
task.name,
extra={"task_name": task.name, "url": url},
)
LOG.warning(f"URL in '{task.name}' feed does not look like a VOD link: {url}")
continue
vid: str = match.group("id")
@ -81,11 +66,7 @@ class TwitchHandler(BaseHandler):
id_dict = get_archive_id(url)
archive_id: str | None = id_dict.get("archive_id")
if not archive_id:
LOG.warning(
"Task '%s' could not compute an archive ID for a Twitch video. Skipping entry.",
task.name,
extra={"task_name": task.name, "video_id": vid, "url": url},
)
LOG.warning(f"Could not compute archive ID for video '{vid}' in '{task.name}' feed. Skipping entry.")
continue
items.append({"id": vid, "url": url, "title": title, "archive_id": archive_id})
@ -93,8 +74,7 @@ class TwitchHandler(BaseHandler):
return feed_url, items, has_items
@staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
handle_name: str | None = TwitchHandler.parse(task.url)
if not handle_name:
return TaskFailure(message="Unrecognized Twitch channel URL.")
@ -103,19 +83,8 @@ class TwitchHandler(BaseHandler):
try:
feed_url, items, has_items = await TwitchHandler._collect_feed(task, params, handle_name)
except httpx.HTTPError as exc:
return TaskFailure(message="Failed to fetch Twitch feed.", error=str(exc))
except Exception as exc:
LOG.exception(
"Failed to fetch Twitch feed for task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return TaskFailure(message="Failed to fetch Twitch feed.", error=str(exc))
task_items: list[TaskItem] = []
@ -124,7 +93,7 @@ class TwitchHandler(BaseHandler):
if not (url := entry.get("url")):
continue
archive_id: str | None = entry.get("archive_id")
archive_id: str = entry.get("archive_id")
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id))
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})

View file

@ -1,20 +1,17 @@
import logging
import re
from typing import TYPE_CHECKING, Any
from xml.etree.ElementTree import Element
import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
if TYPE_CHECKING:
from xml.etree.ElementTree import Element
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("handlers.youtube")
class YoutubeHandler(BaseHandler):
@ -30,11 +27,7 @@ class YoutubeHandler(BaseHandler):
@staticmethod
async def can_handle(task: HandleTask) -> bool:
LOG.debug(
"Checking if task '%s' uses a parsable YouTube URL.",
task.name,
extra={"task_name": task.name, "url": task.url},
)
LOG.debug(f"'{task.name}': Checking if task URL is parsable YouTube URL: {task.url}")
return YoutubeHandler.parse(task.url) is not None
@staticmethod
@ -54,7 +47,7 @@ class YoutubeHandler(BaseHandler):
from defusedxml.ElementTree import fromstring
feed_url: str = YoutubeHandler.FEED.format(type=parsed["type"], id=parsed["id"])
LOG.debug("'%s': Fetching feed.", task.name, extra={"task_name": task.name, "feed_url": feed_url})
LOG.debug(f"'{task.name}': Fetching feed.")
response = await YoutubeHandler.request(url=feed_url, ytdlp_opts=params)
response.raise_for_status()
@ -72,11 +65,7 @@ class YoutubeHandler(BaseHandler):
vid_elem: Element[str] | None = entry.find("yt:videoId", ns)
vid: str = vid_elem.text if vid_elem is not None and vid_elem.text else ""
if not vid:
LOG.warning(
"'%s': Entry in the feed is missing a video ID. Skipping.",
task.name,
extra={"task_name": task.name, "feed_url": feed_url},
)
LOG.warning(f"'{task.name}': Entry in the feed is missing a video ID. Skipping.")
continue
url: str = f"https://www.youtube.com/watch?v={vid}"
@ -84,11 +73,7 @@ class YoutubeHandler(BaseHandler):
id_dict: dict[str, str | None] = get_archive_id(url)
archive_id: str | None = id_dict.get("archive_id")
if not archive_id:
LOG.warning(
"Task '%s' could not compute an archive ID for a YouTube video. Skipping item.",
task.name,
extra={"task_name": task.name, "video_id": vid, "url": url},
)
LOG.warning(f"'{task.name}': Could not compute archive ID for video '{vid}' in feed. Skipping.")
continue
title_elem: Element[str] | None = entry.find("atom:title", ns)
@ -104,8 +89,7 @@ class YoutubeHandler(BaseHandler):
return feed_url, items, real_count
@staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
if not parsed:
return TaskFailure(message="Unrecognized YouTube channel or playlist URL.")
@ -114,19 +98,8 @@ class YoutubeHandler(BaseHandler):
try:
feed_url, items, real_count = await YoutubeHandler._get(task, params, parsed)
except httpx.HTTPError as exc:
return TaskFailure(message="Failed to fetch YouTube feed.", error=str(exc))
except Exception as exc:
LOG.exception(
"Failed to fetch YouTube feed for task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return TaskFailure(message="Failed to fetch YouTube feed.", error=str(exc))
task_items: list[TaskItem] = []
@ -135,7 +108,7 @@ class YoutubeHandler(BaseHandler):
if not (url := entry.get("url")):
continue
archive_id: str | None = entry.get("archive_id")
archive_id: str = entry.get("archive_id")
metadata: dict[str, Any] = {"published": entry.get("published")}
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))

View file

@ -1,17 +1,17 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any
from app.features.core.migration import Migration as FeatureMigration
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.tasks.definitions.repository import TaskDefinitionsRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -48,11 +48,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert task definition '%s'.",
normalized.get("name"),
extra={"definition": normalized.get("name"), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert task definition '%s': %s", normalized.get("name"), exc)
finally:
await self._move_file(path)
@ -66,21 +62,13 @@ class Migration(FeatureMigration):
try:
content = path.read_text(encoding="utf-8")
except Exception as exc:
LOG.exception(
"Failed to read task definition '%s'.",
path,
extra={"path": str(path), "exception_type": type(exc).__name__},
)
LOG.error("Failed to read task definition '%s': %s", path, exc)
return None
try:
payload = json.loads(content)
except Exception as exc:
LOG.exception(
"Failed to parse JSON for task definition '%s'.",
path,
extra={"path": str(path), "exception_type": type(exc).__name__},
)
LOG.error("Failed to parse JSON for '%s': %s", path, exc)
return None
if not isinstance(payload, dict):

View file

@ -1,5 +1,6 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from sqlalchemy import func, or_, select
@ -9,28 +10,24 @@ from app.features.core.schemas import CEFeature, ConfigEvent
from app.features.tasks.definitions.migration import Migration
from app.features.tasks.definitions.models import TaskDefinitionModel
from app.library.Events import Event, EventBus, Events
from app.library.log import get_logger
from app.library.Services import Services
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from collections.abc import AsyncGenerator
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class TaskDefinitionsRepository(metaclass=Singleton):
def __init__(self, session: SessionFactory | None = None) -> None:
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: SessionFactory = session or get_session
self.session: AsyncGenerator[AsyncSession] = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -54,12 +51,12 @@ class TaskDefinitionsRepository(metaclass=Singleton):
LOG.debug("Refreshing task definitions due to configuration update.")
await GenericTaskHandler.refresh_definitions(force=True)
Services.get_instance().add(TaskDefinitionsRepository.__name__, self)
Services.get_instance().add(__class__.__name__, self)
EventBus.get_instance().subscribe(
Events.STARTED, handle_event, f"{TaskDefinitionsRepository.__name__}.run_migrations"
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions")
async def all(self) -> list[TaskDefinitionModel]:
async def list(self) -> list[TaskDefinitionModel]:
async with self.session() as session:
result: Result[tuple[TaskDefinitionModel]] = await session.execute(
select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc())
@ -118,9 +115,9 @@ class TaskDefinitionsRepository(metaclass=Singleton):
async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel:
async with self.session() as session:
model: TaskDefinitionModel = TaskDefinitionModel(**payload)
model: TaskDefinitionModel = TaskDefinitionModel(**payload) if isinstance(payload, dict) else payload
if model.id is not None:
model.id = None # ty: ignore
model.id = None
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Task definition with name '{model.name}' already exists."

View file

@ -48,13 +48,10 @@ class HandleTask(TaskSchema):
if isinstance(ret, tuple):
return ret
archive_file = ret.get("file")
items = ret.get("items", set())
if not isinstance(archive_file, Path) or not isinstance(items, set):
return (False, "Failed to get archive information.")
archive_items = [item for item in items if isinstance(item, str)]
archive_file: Path = ret.get("file")
items: set[str] = ret.get("items", set())
if len(archive_items) < 1 or not archive_add(archive_file, archive_items):
if len(items) < 1 or not archive_add(archive_file, list(items)):
return (True, "No new items to mark as downloaded.")
return (True, f"Task '{self.name}' items marked as downloaded.")
@ -73,13 +70,10 @@ class HandleTask(TaskSchema):
if isinstance(ret, tuple):
return ret
archive_file = ret.get("file")
items = ret.get("items", set())
if not isinstance(archive_file, Path) or not isinstance(items, set):
return (False, "Failed to get archive information.")
archive_items = [item for item in items if isinstance(item, str)]
archive_file: Path = ret.get("file")
items: set[str] = ret.get("items", set())
if len(archive_items) < 1 or not archive_delete(archive_file, archive_items):
if len(items) < 1 or not archive_delete(archive_file, list(items)):
return (True, "No items to remove from archive file.")
return (True, f"Removed '{self.name}' items from archive file.")
@ -113,7 +107,6 @@ class HandleTask(TaskSchema):
no_archive=True,
follow_redirect=False,
sanitize_info=True,
budget_sleep=True,
)
if not ie_info or not isinstance(ie_info, dict):
@ -140,7 +133,7 @@ class HandleTask(TaskSchema):
archive_file: Path = Path(archive_file)
(ie_info, _) = await fetch_info(params, self.url, no_archive=True, follow_redirect=True, budget_sleep=True)
(ie_info, _) = await fetch_info(params, self.url, no_archive=True, follow_redirect=True)
if not ie_info or not isinstance(ie_info, dict):
return (False, "Failed to extract information from URL.")

View file

@ -1,3 +1,4 @@
import logging
from typing import Any
from aiohttp import web
@ -15,10 +16,9 @@ from app.features.tasks.definitions.schemas import (
from app.features.tasks.definitions.utils import model_to_schema, schema_to_payload
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.router import route
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
@route("GET", "api/tasks/definitions/", "task_definitions")
@ -95,11 +95,7 @@ async def task_definitions_create(request: Request, encoder: Encoder, notify: Ev
except ValueError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
except Exception as exc:
LOG.exception(
"Failed to create task definition '%s'.",
getattr(definition_input, "name", None),
extra={"definition": getattr(definition_input, "name", None), "exception_type": type(exc).__name__},
)
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to create task definition."},
status=web.HTTPInternalServerError.status_code,
@ -148,15 +144,7 @@ async def task_definitions_update(request: Request, encoder: Encoder, notify: Ev
except ValueError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
except Exception as exc:
LOG.exception(
"Failed to update task definition '%s'.",
definition_input.name,
extra={
"definition_id": identifier,
"definition": definition_input.name,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to update task definition."},
status=web.HTTPInternalServerError.status_code,
@ -214,11 +202,7 @@ async def task_definitions_patch(request: Request, encoder: Encoder, notify: Eve
except ValueError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
except Exception as exc:
LOG.exception(
"Failed to patch task definition '%s'.",
identifier,
extra={"definition_id": identifier, "exception_type": type(exc).__name__},
)
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to patch task definition."},
status=web.HTTPInternalServerError.status_code,
@ -244,11 +228,7 @@ async def task_definitions_delete(request: Request, encoder: Encoder, notify: Ev
except KeyError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code)
except Exception as exc:
LOG.exception(
"Failed to delete task definition '%s'.",
identifier,
extra={"definition_id": identifier, "exception_type": type(exc).__name__},
)
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to delete task definition."},
status=web.HTTPInternalServerError.status_code,

View file

@ -3,19 +3,18 @@ from __future__ import annotations
import asyncio
import importlib
import inspect
import logging
import pkgutil
import random
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from app.features.tasks.definitions.handlers._base_handler import BaseHandler
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.tasks.models import TaskModel
from app.features.ytdlp.utils import archive_read
from app.library.downloads.queue_manager import DownloadQueue
from app.library.Events import EventBus, Events
from app.library.ItemDTO import Item, ItemDTO
from app.library.log import get_logger
from app.library.Services import Services
if TYPE_CHECKING:
@ -23,12 +22,12 @@ if TYPE_CHECKING:
from app.library.config import Config
from app.library.Scheduler import Scheduler
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("definitions.service")
class TaskHandle:
def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None:
self._handlers: list[type[BaseHandler]] = []
self._handlers: list[type] = []
"The available handlers."
self._repo: TasksRepository = tasks
"The tasks manager."
@ -36,7 +35,7 @@ class TaskHandle:
"The scheduler."
self._config: Config = config
"The configuration."
self._task_name: str = f"{TaskHandle.__name__}._dispatcher"
self._task_name: str = f"{__class__.__name__}._dispatcher"
"The task name for the scheduler."
self._queued: dict[str, set[str]] = {}
"Queued archive IDs per handler."
@ -46,11 +45,11 @@ class TaskHandle:
EventBus.get_instance().subscribe(
Events.ITEM_ERROR,
self._handle_item_error,
f"{TaskHandle.__name__}.item_error",
f"{__class__.__name__}.item_error",
)
def load(self) -> None:
self._handlers: list[type[BaseHandler]] = self._discover()
self._handlers: list[type] = self._discover()
timer: str = self._config.tasks_handler_timer
try:
@ -59,30 +58,20 @@ class TaskHandle:
CronSim(timer, datetime.now(UTC))
except Exception as e:
timer = "15 */1 * * *"
LOG.error(
"Invalid task handler timer '%s'; using default '%s'.",
self._config.tasks_handler_timer,
timer,
extra={
"timer": self._config.tasks_handler_timer,
"default_timer": timer,
"exception_type": type(e).__name__,
},
)
LOG.error(f"Invalid timer format. '{e!s}'. Defaulting to '{timer}'.")
self._scheduler.add(
timer=timer,
func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"),
id=f"{TaskHandle.__name__}._dispatcher",
id=f"{__class__.__name__}._dispatcher",
)
async def _dispatcher(self):
s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []}
handler_groups: dict[str, list[tuple[HandleTask, type[BaseHandler]]]] = {}
dispatches: list[tuple[HandleTask, type[BaseHandler], asyncio.Task[TaskResult | TaskFailure | None]]] = []
handler_groups: dict[str, list[tuple[HandleTask, type]]] = {}
tasks: list[TaskModel] = await self._repo.all()
tasks: list[TaskModel] = await self._repo.list()
for task_model in tasks:
task: HandleTask = HandleTask.model_validate(task_model)
@ -92,16 +81,12 @@ class TaskHandle:
continue
if not task.get_ytdlp_opts().get_all().get("download_archive"):
LOG.debug(
"Task '%s' does not have an archive file configured.",
task.name,
extra={"task_id": task.id, "task_name": task.name},
)
LOG.debug(f"Task '{task.name}' does not have an archive file configured.")
s["f"].append(task.name)
continue
try:
handler: type[BaseHandler] | None = await self._find_handler(task)
handler: type | None = await self._find_handler(task)
if handler is None:
s["u"].append(task.name)
continue
@ -110,12 +95,9 @@ class TaskHandle:
if handler_name not in handler_groups:
handler_groups[handler_name] = []
handler_groups[handler_name].append((task, handler))
s["h"].append(task.name)
except Exception as e:
LOG.exception(
"Failed to find handler for task '%s'.",
task.name,
extra={"task_id": task.id, "task_name": task.name, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to handle task '{task.name}'. '{e!s}'.")
s["f"].append(task.name)
for tasks_with_handlers in handler_groups.values():
@ -130,61 +112,15 @@ class TaskHandle:
name=f"taskHandler-{task.id}",
)
t.add_done_callback(lambda fut, t=task: self._handle_exception(fut, t))
dispatches.append((task, handler, t))
except Exception as e:
LOG.exception(
"Failed to schedule handler '%s' for task '%s'.",
handler.__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"exception_type": type(e).__name__,
},
)
s["f"].append(task.name)
if dispatches:
results = await asyncio.gather(*(t for _, _, t in dispatches), return_exceptions=True)
for (task, handler, _), result in zip(dispatches, results, strict=True):
if isinstance(result, TaskResult):
s["h"].append(task.name)
continue
if isinstance(result, TaskFailure):
s["f"].append(task.name)
continue
if result is None:
LOG.error(
"Handler '%s' returned no result for task '%s'.",
handler.__name__,
task.name,
extra={"task_id": task.id, "task_name": task.name, "handler_name": handler.__name__},
)
s["f"].append(task.name)
LOG.error(f"Failed to dispatch task '{task.name}'. '{e!s}'.")
if len(tasks) > 0:
LOG.info(
"Task dispatch finished: %s handled, %s unhandled, %s disabled, %s failed.",
len(s["h"]),
len(s["u"]),
len(s["d"]),
len(s["f"]),
extra={
"handled_count": len(s["h"]),
"unhandled_count": len(s["u"]),
"disabled_count": len(s["d"]),
"failed_count": len(s["f"]),
},
f"Tasks handler summary: Handled: {len(s['h'])}, Unhandled: {len(s['u'])}, Disabled: {len(s['d'])}, Failed: {len(s['f'])}."
)
async def _dispatch(
self, task: HandleTask, handler: type[BaseHandler], delay: float
) -> TaskResult | TaskFailure | None:
async def _dispatch(self, task: HandleTask, handler: type, delay: float) -> TaskResult | TaskFailure | None:
"""
Dispatch a task after a random delay to avoid rate limiting.
@ -198,12 +134,7 @@ class TaskHandle:
"""
if delay > 0:
LOG.debug(
"Delaying dispatch of task '%s' by %.1f seconds.",
task.name,
delay,
extra={"task_id": task.id, "task_name": task.name, "delay_s": round(delay, 1)},
)
LOG.debug(f"Delaying dispatch of task '{task.name}' by {delay:.1f} seconds.")
await asyncio.sleep(delay)
return await self.dispatch(task, handler=handler)
@ -212,29 +143,15 @@ class TaskHandle:
return
if exc := fut.exception():
LOG.exception(
"Task handler raised after dispatch.",
extra={"task_id": task.id, "task_name": task.name, "exception_type": type(exc).__name__},
exc_info=(type(exc), exc, exc.__traceback__),
)
LOG.error(f"Exception while handling task '{task.name}': {exc}")
async def _find_handler(self, task: HandleTask) -> type[BaseHandler] | None:
async def _find_handler(self, task: HandleTask) -> type | None:
for cls in self._handlers:
try:
if await Services.get_instance().handle_async(handler=cls.can_handle, task=task):
return cls
except Exception as e:
LOG.exception(
"Handler '%s' capability check failed for task '%s'.",
cls.__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": cls.__name__,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
continue
return None
@ -242,10 +159,9 @@ class TaskHandle:
async def dispatch(
self,
task: HandleTask,
handler: type[BaseHandler] | None = None,
**kwargs,
handler: type | None = None,
**kwargs, # noqa: ARG002
) -> TaskResult | TaskFailure | None:
_ = kwargs
"""
Dispatch a task to the appropriate handler.
@ -269,59 +185,20 @@ class TaskHandle:
extraction: TaskResult | TaskFailure = await services.handle_async(
handler=handler.extract, task=task, config=self._config
)
except NotImplementedError as exc:
LOG.exception(
"Task handler '%s' does not implement extraction for task '%s'.",
handler.__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"exception_type": type(exc).__name__,
},
)
except NotImplementedError:
LOG.error(f"Handler '{handler.__name__}' does not implement extract().")
return TaskFailure(message="Handler does not support extraction.")
except Exception as exc:
LOG.exception(
"Handler '%s' extraction failed for task '%s'.",
handler.__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
raise
if isinstance(extraction, TaskFailure):
msg: str = extraction.message
if extraction.error and extraction.error != extraction.message:
msg = f"{msg} {extraction.error}"
LOG.error(
"Handler '%s' failed to extract items for task '%s' because %s.",
handler.__name__,
task.name,
msg,
extra={"task_id": task.id, "task_name": task.name, "handler_name": handler.__name__, "error": msg},
)
LOG.error(f"Handler '{handler.__name__}' failed to extract items: {extraction.message}")
return extraction
if not isinstance(extraction, TaskResult):
LOG.error(
"Handler '%s' returned unexpected result type '%s' for task '%s'.",
handler.__name__,
type(extraction).__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"result_type": type(extraction).__name__,
},
f"Handler '{handler.__name__}' returned unexpected result type '{type(extraction).__name__}'.",
)
return TaskFailure(
message="Handler returned invalid result type.", metadata={"type": type(extraction).__name__}
@ -349,18 +226,7 @@ class TaskHandle:
for item in raw_items:
if not isinstance(item, TaskItem):
LOG.warning(
"Handler '%s' returned unexpected item type '%s' for task '%s'.",
handler.__name__,
type(item).__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"item_type": type(item).__name__,
},
)
LOG.warning("Handler '{handler.__name__}' produced non-TaskItem entry: {item!r}")
continue
url: str = item.url
@ -369,13 +235,7 @@ class TaskHandle:
archive_id: str | None = item.archive_id
if not archive_id:
LOG.warning(
"Handler '%s' skipped '%s' for task '%s' because it has no archive ID.",
handler.__name__,
url,
task.name,
extra={"task_id": task.id, "task_name": task.name, "handler_name": handler.__name__, "url": url},
)
LOG.warning(f"'{task.name}': Item with URL '{url}' is missing an archive ID. Skipping.")
continue
if archive_id in queued:
@ -404,31 +264,12 @@ class TaskHandle:
if not filtered:
if raw_items:
LOG.debug(
"Handler '%s' found %s item(s) for task '%s', but none were queued.",
handler.__name__,
len(raw_items),
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"raw_count": len(raw_items),
},
f"Handler '{handler.__name__}' produced '{len(raw_items)}' for '{task.name}' items, none queued after filtering."
)
return TaskResult(items=[], metadata=metadata)
LOG.info(
"Handler '%s' found %s new item(s) for task '%s'.",
handler.__name__,
len(filtered),
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"item_count": len(filtered),
"raw_count": len(raw_items),
},
f"Handler '{handler.__name__}' Found '{len(filtered)}' new items for '{task.name}' (raw={len(raw_items)})."
)
base_item = Item.format(
@ -503,12 +344,7 @@ class TaskHandle:
try:
matched = await services.handle_async(handler=handler_cls.can_handle, task=task)
except Exception as exc: # pragma: no cover - defensive
LOG.exception(
"Handler '%s' inspection capability check failed for '%s'.",
handler_cls.__name__,
url,
extra={"handler_name": handler_cls.__name__, "url": url, "exception_type": type(exc).__name__},
)
LOG.exception(exc)
message = str(exc)
return TaskFailure(
message=message,
@ -546,12 +382,7 @@ class TaskHandle:
metadata={**base_metadata, "supported": False},
)
except Exception as exc:
LOG.exception(
"Handler '%s' manual inspection failed for '%s'.",
handler_cls.__name__,
url,
extra={"handler_name": handler_cls.__name__, "url": url, "exception_type": type(exc).__name__},
)
LOG.exception(exc)
message = str(exc)
return TaskFailure(
message=message,
@ -572,11 +403,7 @@ class TaskHandle:
if not isinstance(extraction, TaskResult):
LOG.error(
"Handler '%s' returned unexpected result type '%s' while inspecting '%s'.",
handler_cls.__name__,
type(extraction).__name__,
url,
extra={"handler_name": handler_cls.__name__, "url": url, "result_type": type(extraction).__name__},
f"Handler '{handler_cls.__name__}' returned unexpected result type '{type(extraction).__name__}' during inspection.",
)
extraction = TaskResult()
@ -586,11 +413,11 @@ class TaskHandle:
return TaskResult(items=list(extraction.items), metadata=combined_metadata)
def _discover(self) -> list[type[BaseHandler]]:
def _discover(self) -> list[type]:
"""Discover all available task handlers."""
import app.features.tasks.definitions.handlers as handlers_pkg
handlers: list[type[BaseHandler]] = []
handlers: list[type] = []
for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__):
if module_name.startswith("_"):

View file

@ -8,7 +8,6 @@ from app.features.tasks.definitions.results import TaskFailure, TaskResult
from app.features.tasks.definitions.schemas import (
Definition,
EngineConfig,
Parse,
RequestConfig,
ResponseConfig,
TaskDefinition,
@ -22,7 +21,7 @@ def reset_generic_handler(monkeypatch):
monkeypatch.setattr(GenericTaskHandler, "_sources_mtime", {})
def test_build_def_payload():
def test_build_task_definition_parses_valid_payload():
definition = TaskDefinition(
id=1,
name="example",
@ -31,24 +30,26 @@ def test_build_def_payload():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
}
),
parse={
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
),
)
assert definition.name == "example"
assert definition.match_url == ["https://example.com/articles/*"]
assert definition.definition.parse["link"]["expression"] == ".article a.link::attr(href)"
assert definition is not None, "TaskDefinition should be created"
assert "example" == definition.name, "Name should match"
assert "https://example.com/articles/*" in definition.match_url, "Match URL should be in list"
assert "link" in definition.definition.parse, "Parse should contain link field"
assert ".article a.link::attr(href)" == definition.definition.parse["link"]["expression"], (
"Link expression should match"
)
def test_build_def_container():
def test_build_task_definition_handles_container():
definition = TaskDefinition(
id=2,
name="container",
@ -57,28 +58,28 @@ def test_build_def_container():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"selector": ".cards .card",
"fields": {
"link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
},
}
parse={
"items": {
"selector": ".cards .card",
"fields": {
"link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
),
)
assert definition.definition.parse["items"]["selector"] == ".cards .card"
assert definition.definition.parse["items"]["fields"]["link"]["attribute"] == "href"
assert definition is not None, "TaskDefinition should be created"
assert "items" in definition.definition.parse, "Parse should contain items container"
assert ".cards .card" == definition.definition.parse["items"]["selector"], "Items selector should match"
assert "link" in definition.definition.parse["items"]["fields"], "Items fields should contain link"
def test_build_def_json():
def test_build_task_definition_handles_json():
definition = TaskDefinition(
id=3,
name="json-def",
@ -87,30 +88,32 @@ def test_build_def_json():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
parse={
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
),
)
assert definition.definition.response.type == "json"
assert definition.definition.parse["items"]["type"] == "jsonpath"
assert definition.definition.parse["items"]["fields"]["link"]["type"] == "jsonpath"
assert definition is not None, "TaskDefinition should be created"
assert "json" == definition.definition.response.type, "Response type should be json"
assert "items" in definition.definition.parse, "Parse should contain items container"
assert "jsonpath" == definition.definition.parse["items"]["type"], "Items type should be jsonpath"
assert "jsonpath" == definition.definition.parse["items"]["fields"]["link"]["type"], (
"Link field type should be jsonpath"
)
def test_parse_items_basic():
def test_parse_items_extracts_values():
definition = TaskDefinition(
id=4,
name="example",
@ -119,13 +122,11 @@ def test_parse_items_basic():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
"id": {"type": "css", "expression": ".article", "attribute": "data-id"},
}
),
parse={
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
"id": {"type": "css", "expression": ".article", "attribute": "data-id"},
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
@ -145,16 +146,14 @@ def test_parse_items_basic():
items = GenericTaskHandler._parse_items(definition, html, "https://example.com/base/")
assert len(items) == 2
assert items[0] == {
"link": "https://example.com/article-101",
"title": "First Title",
"id": "101",
}
assert items[1]["link"] == "https://example.com/article-102"
assert 2 == len(items), "Should extract 2 items"
assert "https://example.com/article-101" == items[0]["link"], "First item link should be absolute URL"
assert "First Title" == items[0]["title"], "First item title should match"
assert "101" == items[0]["id"], "First item id should match"
assert "https://example.com/article-102" == items[1]["link"], "Second item link should match"
def test_parse_items_cards():
def test_parse_items_handles_nested_card_layout():
definition = TaskDefinition(
id=5,
name="nested",
@ -163,36 +162,34 @@ def test_parse_items_cards():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"type": "css",
"selector": ".columns .card",
"fields": {
"link": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "href",
},
"title": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "text",
},
"poet": {
"type": "css",
"expression": "footer .card-footer-item:first-child a",
"attribute": "text",
},
"category": {
"type": "css",
"expression": "footer .card-footer-item:nth-child(2) a",
"attribute": "text",
},
parse={
"items": {
"type": "css",
"selector": ".columns .card",
"fields": {
"link": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "href",
},
}
"title": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "text",
},
"poet": {
"type": "css",
"expression": "footer .card-footer-item:first-child a",
"attribute": "text",
},
"category": {
"type": "css",
"expression": "footer .card-footer-item:nth-child(2) a",
"attribute": "text",
},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
@ -237,21 +234,19 @@ def test_parse_items_cards():
items = GenericTaskHandler._parse_items(definition, html, "https://example.com")
assert len(items) == 2
assert items[0] == {
"link": "https://example.com/poems/view/111",
"title": "First Poem",
"poet": "Poet Alpha",
"category": "Category One",
}
assert items[1] == {
"link": "https://example.com/poems/view/222",
"title": "Second Poem",
"poet": "Poet Beta",
}
assert 2 == len(items), "Should extract 2 items"
assert "https://example.com/poems/view/111" == items[0]["link"], "First item link should match"
assert "First Poem" == items[0]["title"], "First item title should match"
assert "Poet Alpha" == items[0]["poet"], "First item poet should match"
assert "Category One" == items[0]["category"], "First item category should match"
assert "https://example.com/poems/view/222" == items[1]["link"], "Second item link should match"
assert "Second Poem" == items[1]["title"], "Second item title should match"
assert "Poet Beta" == items[1]["poet"], "Second item poet should match"
assert "category" not in items[1], "Second item should not have category"
def test_parse_items_json():
def test_parse_items_handles_json_container():
definition = TaskDefinition(
id=6,
name="json",
@ -260,19 +255,17 @@ def test_parse_items_json():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "entries",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
"id": {"type": "jsonpath", "expression": "id"},
},
}
parse={
"items": {
"type": "jsonpath",
"selector": "entries",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
"id": {"type": "jsonpath", "expression": "id"},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
@ -294,10 +287,14 @@ def test_parse_items_json():
json_data=payload,
)
assert items == [
{"link": "https://example.com/video/1", "title": "First", "id": "1"},
{"link": "https://example.com/video/2", "title": "Second", "id": "2"},
]
assert 2 == len(items), "Should extract 2 items (third missing link)"
assert "https://example.com/video/1" == items[0]["link"], "First item link should be absolute"
assert "First" == items[0]["title"], "First item title should match"
assert "1" == items[0]["id"], "First item id should match"
assert "https://example.com/video/2" == items[1]["link"], "Second item link should match"
assert "Second" == items[1]["title"], "Second item title should match"
assert "2" == items[1]["id"], "Second item id should match"
@pytest.mark.asyncio
@ -310,18 +307,16 @@ async def test_generic_task_handler_inspect(monkeypatch):
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
parse={
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
@ -350,14 +345,14 @@ async def test_generic_task_handler_inspect(monkeypatch):
task = HandleTask(id=1, name="Inspect", url="https://example.com/api")
result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task)
assert isinstance(result, TaskResult)
assert len(result.items) == 1
assert isinstance(result, TaskResult), "Result should be TaskResult"
assert 1 == len(result.items), "Should have 1 item"
item = result.items[0]
assert item.url == "https://example.com/video/1"
assert item.title == "First"
assert "https://example.com/video/1" == item.url, "Item URL should match"
assert "First" == item.title, "Item title should match"
def test_parse_items_json_list():
def test_parse_items_handles_json_top_level_list():
definition = TaskDefinition(
id=8,
name="json-list",
@ -366,18 +361,16 @@ def test_parse_items_json_list():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "[]",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
parse={
"items": {
"type": "jsonpath",
"selector": "[]",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
@ -396,7 +389,8 @@ def test_parse_items_json_list():
json_data=payload,
)
assert items == [
{"link": "https://example.com/video/1", "title": "First"},
{"link": "https://example.com/video/2", "title": "Second"},
]
assert 2 == len(items), "Should extract 2 items"
assert "https://example.com/video/1" == items[0]["link"], "First item link should match"
assert "First" == items[0]["title"], "First item title should match"
assert "https://example.com/video/2" == items[1]["link"], "Second item link should match"
assert "Second" == items[1]["title"], "Second item title should match"

View file

@ -1,5 +1,3 @@
from pathlib import Path
import pytest
from app.features.tasks.definitions.handlers.rss import RssGenericHandler
@ -23,10 +21,6 @@ class DummyOpts:
return self._data
def _opts(tmp_path: Path) -> DummyOpts:
return DummyOpts({"download_archive": str(tmp_path / "archive.txt")})
class TestRssHandlerParsing:
"""Test URL parsing for RSS/Atom feeds using the tests() method."""
@ -53,7 +47,7 @@ class TestRssHandlerExtraction:
"""Test RSS feed extraction and parsing."""
@pytest.mark.asyncio
async def test_rss_atom_feed_extraction(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_rss_atom_feed_extraction(self, monkeypatch):
"""Test extraction from Atom feed."""
atom_feed = """<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
@ -75,7 +69,7 @@ class TestRssHandlerExtraction:
return DummyResponse(atom_feed)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,
@ -95,7 +89,7 @@ class TestRssHandlerExtraction:
assert result.metadata["entry_count"] == 2
@pytest.mark.asyncio
async def test_rss_feed_extraction(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_rss_feed_extraction(self, monkeypatch):
"""Test extraction from RSS feed."""
rss_feed = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
@ -119,7 +113,7 @@ class TestRssHandlerExtraction:
return DummyResponse(rss_feed)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,
@ -139,7 +133,7 @@ class TestRssHandlerExtraction:
assert result.metadata["entry_count"] == 2
@pytest.mark.asyncio
async def test_can_handle(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_can_handle(self, monkeypatch):
"""Test can_handle method."""
atom_feed = """<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
@ -153,7 +147,7 @@ class TestRssHandlerExtraction:
return DummyResponse(atom_feed)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,
@ -178,7 +172,7 @@ class TestRssHandlerEdgeCases:
"""Test edge cases in RSS handling."""
@pytest.mark.asyncio
async def test_empty_feed(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_empty_feed(self, monkeypatch):
"""Test handling of empty feed."""
empty_feed = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
@ -192,7 +186,7 @@ class TestRssHandlerEdgeCases:
return DummyResponse(empty_feed)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,
@ -208,7 +202,7 @@ class TestRssHandlerEdgeCases:
assert result.metadata["entry_count"] == 0
@pytest.mark.asyncio
async def test_invalid_feed_url(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_invalid_feed_url(self, monkeypatch):
"""Test handling of invalid feed URL."""
from app.features.tasks.definitions.results import TaskFailure
@ -217,7 +211,7 @@ class TestRssHandlerEdgeCases:
raise Exception(msg)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,
@ -231,7 +225,7 @@ class TestRssHandlerEdgeCases:
assert isinstance(result, TaskFailure)
@pytest.mark.asyncio
async def test_missing_urls_in_feed(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_missing_urls_in_feed(self, monkeypatch):
"""Test handling of entries missing URLs."""
feed = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
@ -251,7 +245,7 @@ class TestRssHandlerEdgeCases:
return DummyResponse(feed)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,

View file

@ -21,7 +21,6 @@ from app.features.tasks.definitions.router import (
from app.library.encoder import Encoder
from app.library.sqlite_store import SqliteStore
from app.main import EventBus
from app.tests.helpers import make_in_memory_db_path
def _sample_definition(name: str = "example", *, priority: int = 0) -> dict:
@ -50,14 +49,17 @@ async def repo() -> AsyncGenerator[TaskDefinitionsRepository, None]:
TaskDefinitionsRepository._reset_singleton()
SqliteStore._reset_singleton()
store = SqliteStore(db_path=make_in_memory_db_path("task-definitions-repository"))
store = SqliteStore(db_path=":memory:")
await store.get_connection()
repository = TaskDefinitionsRepository.get_instance()
yield repository
await store.close()
if store._conn:
await store._conn.close()
if store._engine:
await store._engine.dispose()
TaskDefinitionsRepository._reset_singleton()
SqliteStore._reset_singleton()
@ -69,7 +71,7 @@ class TestTaskDefinitionsRepository:
await repo.create(_sample_definition("Alpha", priority=2))
await repo.create(_sample_definition("Beta", priority=1))
items = await repo.all()
items = await repo.list()
assert len(items) == 2, "Should return two task definitions"
assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name"

View file

@ -1,5 +1,3 @@
from pathlib import Path
import pytest
from app.features.tasks.definitions.handlers.tver import TverHandler
@ -27,7 +25,7 @@ class DummyOpts:
@pytest.mark.asyncio
async def test_tver_handler_extract(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_tver_handler_extract(monkeypatch):
"""Test tver handler extraction of episodes from series."""
session_response = {
"result": {
@ -120,10 +118,8 @@ async def test_tver_handler_extract(monkeypatch: pytest.MonkeyPatch, tmp_path: P
msg = f"Unexpected URL: {url}"
raise RuntimeError(msg)
archive_path = tmp_path / "archive.txt"
monkeypatch.setattr(TverHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": str(archive_path)}))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": "/tmp/archive"}))
task = HandleTask(id=1, name="Test Tver Series", url="https://tver.jp/series/sr8sb9pnhc", preset="default")
@ -146,7 +142,7 @@ async def test_tver_handler_extract(monkeypatch: pytest.MonkeyPatch, tmp_path: P
@pytest.mark.parametrize(("url", "should_match"), TverHandler.tests())
def test_parse(url: str, should_match: bool) -> None:
def test_tver_handler_parse(url: str, should_match: bool):
"""Test tver URL parsing."""
result = TverHandler.parse(url)
if should_match:
@ -157,7 +153,7 @@ def test_parse(url: str, should_match: bool) -> None:
@pytest.mark.asyncio
async def test_can_handle() -> None:
async def test_tver_handler_can_handle():
"""Test tver handler can_handle method."""
task_valid = HandleTask(id=1, name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default")
task_invalid = HandleTask(id=2, name="Test", url="https://youtube.com/watch?v=123", preset="default")

View file

@ -1,5 +1,3 @@
from pathlib import Path
import pytest
from app.features.tasks.definitions.handlers.twitch import TwitchHandler
@ -24,7 +22,7 @@ class DummyOpts:
@pytest.mark.asyncio
async def test_twitch_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_twitch_handler_inspect(monkeypatch):
feed = """
<rss><channel>
<item>
@ -41,10 +39,8 @@ async def test_twitch_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path:
async def fake_request(**kwargs): # noqa: ARG001
return DummyResponse(feed)
archive_path = tmp_path / "archive.txt"
monkeypatch.setattr(TwitchHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": str(archive_path)})) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,

View file

@ -1,5 +1,3 @@
from pathlib import Path
import pytest
from app.features.tasks.definitions.handlers.youtube import YoutubeHandler
@ -24,7 +22,7 @@ class DummyOpts:
@pytest.mark.asyncio
async def test_youtube_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_youtube_handler_inspect(monkeypatch):
feed = """
<feed xmlns='http://www.w3.org/2005/Atom' xmlns:yt='http://www.youtube.com/xml/schemas/2015'>
<entry>
@ -43,10 +41,8 @@ async def test_youtube_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path
async def fake_request(**kwargs): # noqa: ARG001
return DummyResponse(feed)
archive_path = tmp_path / "archive.txt"
monkeypatch.setattr(YoutubeHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": str(archive_path)})) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,

View file

@ -1,17 +1,9 @@
from typing import Any, Literal, overload
from typing import Any
from app.features.tasks.definitions.models import TaskDefinitionModel
from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary
@overload
def model_to_schema(model: TaskDefinitionModel, summary: Literal[False] = False) -> TaskDefinition: ...
@overload
def model_to_schema(model: TaskDefinitionModel, summary: Literal[True]) -> TaskDefinitionSummary: ...
def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary:
"""
Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema.

View file

@ -1,18 +1,18 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any
from app.features.core.migration import Migration as FeatureMigration
from app.features.tasks.schemas import Task
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.tasks.repository import TasksRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -36,11 +36,7 @@ class Migration(FeatureMigration):
try:
items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text())
except Exception as exc:
LOG.exception(
"Failed to read tasks migration file '%s'. Ignoring.",
self._source_file,
extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
await self._move_file(self._source_file)
return
@ -58,11 +54,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert task '%s'.",
normalized["name"],
extra={"task_name": normalized["name"], "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert task '%s': %s", normalized["name"], exc)
LOG.info("Migrated %s task(s) from %s.", inserted, self._source_file)
await self._move_file(self._source_file)

View file

@ -1,48 +1,29 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from sqlalchemy import func, or_, select
from app.features.core.deps import get_session
from app.features.tasks.models import TaskModel
from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from collections.abc import AsyncGenerator
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> TaskModel:
model = TaskModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for TaskModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: TaskModel | dict[str, Any]) -> TaskModel:
if isinstance(payload, TaskModel):
return payload
return _model_from_payload(payload)
LOG: logging.Logger = logging.getLogger(__name__)
class TasksRepository(metaclass=Singleton):
def __init__(self, session: SessionFactory | None = None) -> None:
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: SessionFactory = session or get_session
self.session = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -57,7 +38,7 @@ class TasksRepository(metaclass=Singleton):
def get_instance() -> TasksRepository:
return TasksRepository()
async def all(self) -> list[TaskModel]:
async def list(self) -> list[TaskModel]:
async with self.session() as session:
result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc()))
return list(result.scalars().all())
@ -109,7 +90,7 @@ class TasksRepository(metaclass=Singleton):
"""Get all enabled tasks."""
async with self.session() as session:
result: Result[tuple[TaskModel]] = await session.execute(
select(TaskModel).where(TaskModel.enabled).order_by(TaskModel.name.asc())
select(TaskModel).where(TaskModel.enabled == True).order_by(TaskModel.name.asc()) # noqa: E712
)
return list(result.scalars().all())
@ -121,11 +102,11 @@ class TasksRepository(metaclass=Singleton):
)
return list(result.scalars().all())
async def create(self, payload: TaskModel | dict[str, Any]) -> TaskModel:
async def create(self, payload: TaskModel | dict) -> TaskModel:
async with self.session() as session:
model = _coerce_model(payload)
model: TaskModel = TaskModel(**payload) if isinstance(payload, dict) else payload
if model.id is not None:
model.id = None # ty: ignore
model.id = None
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Task with name '{model.name}' already exists."

View file

@ -1,4 +1,4 @@
import asyncio
import logging
from typing import TYPE_CHECKING, Any
from aiohttp import web
@ -17,7 +17,6 @@ from app.library.ag_utils import ag
from app.library.config import Config
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.router import route
from app.library.Utils import get_channel_images, get_file, validate_url
@ -25,7 +24,7 @@ if TYPE_CHECKING:
from pathlib import Path
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
TIMER_SLOTS_PER_HOUR: int = 12
@ -94,11 +93,7 @@ def _offset_timer(timer: str, index: int) -> str:
return f"{minute} {hour} {dom} {month} {dow}"
except Exception as e:
LOG.warning(
"Failed to offset task timer.",
extra={"timer": timer, "index": index, "error": str(e), "exception_type": type(e).__name__},
exc_info=True,
)
LOG.warning(f"Failed to offset timer '{timer}': {e}")
return timer
@ -141,15 +136,10 @@ async def _get_info(url: str, preset: str) -> tuple[str | None, str | None]:
return (name, converted_url)
except TimeoutError:
LOG.debug("Timeout while inferring name from '%s'.", url, extra={"url": url, "preset": preset})
LOG.debug(f"Timeout while inferring name from '{url}'")
return (None, None)
except Exception as e:
LOG.debug(
"Failed to infer a task name from '%s' because %s.",
url,
e,
extra={"url": url, "preset": preset, "error": str(e)},
)
LOG.debug(f"Failed to infer name from '{url}': {e}")
return (None, None)
@ -161,22 +151,6 @@ def _serialize(model: Any) -> dict:
return _model(model).model_dump()
async def _require_timer_or_handler(task: Task, handler: TaskHandle) -> None:
if task.timer:
return
if task.handler_enabled is False:
msg = "Task requires a timer when the handler is disabled."
raise ValueError(msg)
result = await handler.inspect(url=task.url, preset=task.preset, static_only=True)
if isinstance(result, TaskResult) and result.metadata.get("matched") is True:
return
msg = "Task requires a timer when no supported handler matches the URL."
raise ValueError(msg)
@route("GET", "api/tasks/", "tasks_list")
async def tasks_list(request: Request, repo: TasksRepository, encoder: Encoder) -> Response:
page, per_page = normalize_pagination(request)
@ -192,13 +166,7 @@ async def tasks_list(request: Request, repo: TasksRepository, encoder: Encoder)
@route("POST", "api/tasks/", "tasks_add")
async def tasks_add(
request: Request,
repo: TasksRepository,
encoder: Encoder,
notify: EventBus,
handler: TaskHandle,
) -> Response:
async def tasks_add(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response:
data = await request.json()
if isinstance(data, dict):
@ -237,18 +205,17 @@ async def tasks_add(
status=web.HTTPConflict.status_code,
)
pending_tasks: list[dict[str, Any]] = []
created_tasks: list[dict[str, Any]] = []
base_settings = first_task.model_dump(exclude={"id", "name", "url", "created_at", "updated_at"})
for idx, item in enumerate(data):
if not isinstance(item, dict):
LOG.warning("Skipping item %s: not a dict.", idx, extra={"index": idx})
LOG.warning(f"Skipping item {idx}: not a dict")
continue
url = str(item.get("url", "")).strip()
if not url:
LOG.debug("Skipping item %s: empty URL.", idx, extra={"index": idx})
LOG.debug(f"Skipping item {idx}: empty URL")
continue
inferred_name, converted_url = await _get_info(url, item.get("preset", first_task.preset))
@ -276,23 +243,11 @@ async def tasks_add(
try:
validated = Task.model_validate(task_dict)
task_data = validated.model_dump()
except ValidationError as exc:
return web.json_response(
data={"error": "Failed to validate task.", "detail": format_validation_errors(exc)},
status=web.HTTPBadRequest.status_code,
)
LOG.warning(f"Skipping task {idx}: validation failed - {exc}")
continue
try:
await _require_timer_or_handler(validated, handler)
except ValueError as exc:
return web.json_response(
data={"error": str(exc)},
status=web.HTTPBadRequest.status_code,
)
pending_tasks.append(validated.model_dump())
for idx, task_data in enumerate(pending_tasks):
try:
created = await repo.create(task_data)
saved = _serialize(created)
@ -302,11 +257,7 @@ async def tasks_add(
data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.CREATE, data=saved),
)
except ValueError as exc:
LOG.warning(
"Failed to create task from request item %s.",
idx,
extra={"index": idx, "error": str(exc), "exception_type": type(exc).__name__},
)
LOG.warning(f"Failed to create task {idx}: {exc}")
continue
if len(created_tasks) == 0:
@ -353,13 +304,7 @@ async def tasks_delete(request: Request, repo: TasksRepository, encoder: Encoder
@route("PATCH", r"api/tasks/{id:\d+}", "tasks_patch")
async def tasks_patch(
request: Request,
repo: TasksRepository,
encoder: Encoder,
notify: EventBus,
handler: TaskHandle,
) -> Response:
async def tasks_patch(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
@ -389,16 +334,6 @@ async def tasks_patch(
status=web.HTTPConflict.status_code,
)
merged = _model(model).model_copy(update=validated.model_dump(exclude_unset=True))
try:
await _require_timer_or_handler(merged, handler)
except ValueError as exc:
return web.json_response(
data={"error": str(exc)},
status=web.HTTPBadRequest.status_code,
)
updated = _serialize(await repo.update(model.id, validated.model_dump(exclude_unset=True)))
notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.UPDATE, data=updated))
return web.json_response(
@ -409,13 +344,7 @@ async def tasks_patch(
@route("PUT", r"api/tasks/{id:\d+}", "tasks_update")
async def tasks_update(
request: Request,
repo: TasksRepository,
encoder: Encoder,
notify: EventBus,
handler: TaskHandle,
) -> Response:
async def tasks_update(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
@ -445,14 +374,6 @@ async def tasks_update(
status=web.HTTPConflict.status_code,
)
try:
await _require_timer_or_handler(validated, handler)
except ValueError as exc:
return web.json_response(
data={"error": str(exc)},
status=web.HTTPBadRequest.status_code,
)
updated = _serialize(await repo.update(model.id, validated.model_dump(exclude_unset=True)))
notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.UPDATE, data=updated))
@ -483,7 +404,7 @@ async def task_handler_inspect(request: Request, handler: TaskHandle, encoder: E
static_only: bool = data.get("static_only", False) if isinstance(data, dict) else False
if not static_only:
try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
validate_url(url, allow_internal=config.allow_internal_urls)
except ValueError as e:
return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code)
@ -495,16 +416,7 @@ async def task_handler_inspect(request: Request, handler: TaskHandle, encoder: E
url=url, preset=preset, handler_name=handler_name, static_only=static_only
)
except Exception as e:
LOG.exception(
"Failed to inspect task handler for '%s'.",
url,
extra={
"handler_name": handler_name,
"url": url,
"static_only": static_only,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
return web.json_response(
{"error": "Failed to inspect handler.", "message": str(e)},
status=web.HTTPInternalServerError.status_code,
@ -653,12 +565,7 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
if not save_path.exists():
save_path.mkdir(parents=True, exist_ok=True)
except Exception as e:
LOG.warning(
"Failed to resolve the metadata output path for task '%s'.",
task.name,
extra={"task_id": task.id, "task_name": task.name, "error": str(e)},
exc_info=True,
)
LOG.warning(f"Failed to resolve final path from outtmpl. '{e!s}'")
info = {
"id": ag(metadata, ["id", "channel_id"]),
@ -676,12 +583,7 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
data={"error": "Failed to get title from metadata."}, status=web.HTTPBadRequest.status_code
)
LOG.info(
"Generating metadata for task '%s' in '%s'.",
task.name,
save_path,
extra={"task_id": task.id, "task_name": task.name, "save_path": str(save_path)},
)
LOG.info(f"Generating metadata for task '{task.name}' in '{save_path!s}'")
from yt_dlp.utils import sanitize_filename
@ -705,9 +607,8 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
xml_content += f' <uniqueid type="{NFOMakerPP._escape_text(info.get("id_type"))}" default="true">{NFOMakerPP._escape_text(info.get("id"))}</uniqueid>\n'
if info.get("uploader"):
xml_content += f" <studio>{NFOMakerPP._escape_text(info.get('uploader'))}</studio>\n"
tags = info.get("tags", [])
if isinstance(tags, list):
for tag in tags:
if info.get("tags", []):
for tag in info.get("tags", []):
xml_content += f" <tag>{NFOMakerPP._escape_text(tag)}</tag>\n"
if info.get("year"):
xml_content += f" <year>{info.get('year')}</year>\n"
@ -739,25 +640,14 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
url: str | None = None
try:
url = thumbnails.get(key)
LOG.info(
"Fetching '%s' thumbnail for task '%s'.",
key,
task.name,
extra={"task_id": task.id, "task_name": task.name, "thumbnail": key, "url": url},
)
LOG.info(f"Fetching thumbnail '{key}' from '{url}'")
if not url:
continue
try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
except ValueError as exc:
LOG.warning(
"Task '%s' has an invalid '%s' thumbnail URL because %s.",
task.name,
key,
exc,
extra={"task_id": task.id, "task_name": task.name, "thumbnail": key, "url": url},
)
validate_url(url, allow_internal=config.allow_internal_urls)
except ValueError:
LOG.warning(f"Invalid thumbnail url '{url}'")
continue
resp = await client.request(
@ -771,27 +661,11 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
img_file.write_bytes(resp.content)
thumbnails[key] = str(img_file.relative_to(config.download_path))
except Exception as e:
LOG.warning(
"Failed to fetch '%s' thumbnail for task '%s'.",
key,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"thumbnail": key,
"url": url,
"error": str(e),
},
exc_info=True,
)
url_log = url or "unknown"
LOG.warning(f"Failed to fetch thumbnail '{key}' from '{url_log}'. '{e!s}'")
continue
except Exception as e:
LOG.warning(
"Failed to fetch metadata thumbnails for task '%s'.",
task.name,
extra={"task_id": task.id, "task_name": task.name, "error": str(e)},
exc_info=True,
)
LOG.warning(f"Failed to fetch thumbnails. '{e!s}'")
return web.json_response(data=info, status=web.HTTPOk.status_code, dumps=encoder.encode)
except ValueError as e:

View file

@ -1,12 +1,12 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from app.features.core.schemas import CEAction, CEFeature, ConfigEvent
from app.features.tasks.models import TaskModel
from app.features.tasks.utils import cron_time
from app.library.Events import Event, EventBus, Events
from app.library.log import get_logger
from app.library.Scheduler import Scheduler
from app.library.Services import Services
from app.library.Singleton import Singleton
@ -14,7 +14,7 @@ from app.library.Singleton import Singleton
if TYPE_CHECKING:
from aiohttp import web
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("tasks.service")
class Tasks(metaclass=Singleton):
@ -51,7 +51,7 @@ class Tasks(metaclass=Singleton):
pass
async def _load_tasks(self) -> None:
tasks = await self._repo.all()
tasks = await self._repo.list()
for task in tasks:
if not task.timer or not task.enabled:
@ -59,23 +59,10 @@ class Tasks(metaclass=Singleton):
try:
self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=f"task-cronjob-{task.id}")
LOG.info(
"Queued task '%s' to run at '%s'.",
task.name,
cron_time(task.timer),
extra={"task_id": task.id, "task_name": task.name, "timer": task.timer},
)
LOG.info(f"Task '{task.id}: {task.name}' queued to be executed '{cron_time(task.timer)}'.")
except Exception as e:
LOG.exception(
"Failed to queue task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"timer": task.timer,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
LOG.error(f"Failed to queue task '{task.name}'. '{e!s}'.")
async def _init_handlers_service(self, scheduler) -> None:
"""Initialize the handlers service after migrations."""
@ -108,12 +95,7 @@ class Tasks(metaclass=Singleton):
if task.timer and task.enabled:
self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=task_id)
LOG.info(
"Queued task '%s' to run at '%s'.",
task.name,
cron_time(task.timer),
extra={"task_id": task.id, "task_name": task.name, "timer": task.timer},
)
LOG.info(f"Task '{task.id}: {task.name}' queued to be executed '{cron_time(task.timer)}'.")
async def _runner(self, task: TaskModel) -> None:
"""
@ -131,29 +113,17 @@ class Tasks(metaclass=Singleton):
from app.library.ItemDTO import Item
timeNow: str = datetime.now(UTC).isoformat()
task_id = task.id
task_name = task.name
try:
current_task = await self._repo.get(task_id)
if not current_task:
LOG.info("Task '%s' no longer exists.", task_name, extra={"task_id": task_id, "task_name": task_name})
if not (task := await self._repo.get(task.id)):
LOG.info(f"Task '{task.name}' no longer exists.")
return
task = current_task
if not task.enabled:
LOG.debug(
"Task '%s' is disabled. Skipping execution.",
task.name,
extra={"task_id": task.id, "task_name": task.name},
)
LOG.debug(f"Task '{task.name}' is disabled. Skipping execution.")
return
if not task.url:
LOG.error(
"Failed to dispatch task '%s' because it has no URL.",
task.name,
extra={"task_id": task.id, "task_name": task.name},
)
LOG.error(f"Failed to dispatch '{task.name}'. No URL found.")
return
started: float = time.time()
@ -186,19 +156,7 @@ class Tasks(metaclass=Singleton):
timeNow = datetime.now(UTC).isoformat()
ended: float = time.time()
LOG.info(
"Task '%s' completed in %.2f seconds.",
task.name,
ended - started,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"preset": preset,
"elapsed_s": round(ended - started, 2),
"status": status.get("status") if isinstance(status, dict) else None,
},
)
LOG.info(f"Task '{task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.")
notify.emit(
Events.TASK_DISPATCHED,
@ -213,17 +171,7 @@ class Tasks(metaclass=Singleton):
message=f"Task '{task.name}' completed in '{ended - started:.2f}'.",
)
except Exception as e:
LOG.exception(
"Failed to execute scheduled task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"time": timeNow,
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to execute '{task.name}' at '{timeNow}'. '{e!s}'.")
EventBus.get_instance().emit(
Events.LOG_ERROR,
data={"preset": task.preset},

View file

@ -5,25 +5,28 @@ from __future__ import annotations
import pytest
import pytest_asyncio
from app.features.tasks.models import TaskModel
from app.features.tasks.repository import TasksRepository
from app.library.sqlite_store import SqliteStore
from app.tests.helpers import make_in_memory_db_path
@pytest_asyncio.fixture
async def repo():
async def repo(tmp_path):
"""Provide a fresh repository instance with initialized database for each test."""
TasksRepository._reset_singleton()
SqliteStore._reset_singleton()
store = SqliteStore(db_path=make_in_memory_db_path("tasks-repository"))
store = SqliteStore(db_path=str(":memory:"))
await store.get_connection()
repository = TasksRepository.get_instance()
yield repository
await store.close()
if store._conn:
await store._conn.close()
if store._engine:
await store._engine.dispose()
TasksRepository._reset_singleton()
SqliteStore._reset_singleton()
@ -32,6 +35,25 @@ async def repo():
class TestTasksRepository:
"""Test suite for TasksRepository database operations."""
@pytest.mark.asyncio
async def test_repository_singleton(self, repo):
"""Verify repository follows singleton pattern."""
instance1 = TasksRepository.get_instance()
instance2 = TasksRepository.get_instance()
assert instance1 is instance2, "Should return same singleton instance"
@pytest.mark.asyncio
async def test_list_empty(self, repo):
"""List returns empty when no tasks exist."""
tasks = await repo.list()
assert tasks == [], "Should return empty list when no tasks"
@pytest.mark.asyncio
async def test_count_empty(self, repo):
"""Count returns 0 when no tasks exist."""
count = await repo.count()
assert count == 0, "Should return 0 when no tasks exist"
@pytest.mark.asyncio
async def test_create_task(self, repo):
"""Create task with valid data."""
@ -260,7 +282,7 @@ class TestTasksRepository:
await repo.create({"name": "Alice", "url": "https://example.com"})
await repo.create({"name": "Bob", "url": "https://example.com"})
items = await repo.all()
items = await repo.list()
assert items[0].name == "Alice", "Should be alphabetically first"
assert items[1].name == "Bob", "Should be alphabetically second"

View file

@ -1,184 +0,0 @@
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
import pytest
import pytest_asyncio
from aiohttp import web
from aiohttp.test_utils import make_mocked_request
from app.features.tasks import router
from app.features.tasks.definitions.results import TaskFailure, TaskResult
from app.features.tasks.repository import TasksRepository
from app.library.encoder import Encoder
from app.library.sqlite_store import SqliteStore
from app.tests.helpers import make_in_memory_db_path
@pytest_asyncio.fixture
async def repo():
TasksRepository._reset_singleton()
SqliteStore._reset_singleton()
store = SqliteStore(db_path=make_in_memory_db_path("tasks-router"))
await store.get_connection()
repository = TasksRepository.get_instance()
yield repository
await store.close()
TasksRepository._reset_singleton()
SqliteStore._reset_singleton()
@pytest.fixture(autouse=True)
def patch_get_info(monkeypatch: pytest.MonkeyPatch) -> None:
async def _fake_get_info(_url: str, _preset: str) -> tuple[None, None]:
return (None, None)
monkeypatch.setattr(router, "_get_info", _fake_get_info)
def _json_request(method: str, path: str, payload: object, *, match_info: dict[str, str] | None = None) -> web.Request:
mock_request: Any = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace())
async def _json() -> object:
return payload
mock_request.json = _json
return mock_request
class _Notify:
def emit(self, *_args, **_kwargs) -> None:
return None
class _Handler:
def __init__(self, matched: bool | dict[str, bool]) -> None:
self._matched = matched
async def inspect(self, *, url: str, preset: str | None = None, static_only: bool = False, **_kwargs):
del preset, static_only
if isinstance(self._matched, dict):
matched = self._matched.get(url, False)
else:
matched = self._matched
if matched:
return TaskResult(metadata={"matched": True, "handler": "TestHandler"})
return TaskFailure(message="No handler", metadata={"matched": False, "handler": None})
@pytest.mark.asyncio
async def test_add_requires_timer_without_handler(repo) -> None:
request = _json_request(
"POST",
"/api/tasks/",
{"name": "No Timer", "url": "https://example.com/channel"},
)
response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=False))
assert response.status == web.HTTPBadRequest.status_code
assert b"requires a timer" in response.body
assert await repo.all() == []
@pytest.mark.asyncio
async def test_add_all_or_nothing(repo) -> None:
request = _json_request(
"POST",
"/api/tasks/",
[
{"name": "First", "url": "https://example.com/first"},
{"url": "https://example.com/second"},
],
)
response = await router.tasks_add(
request,
repo,
Encoder(),
_Notify(),
_Handler({"https://example.com/first": True, "https://example.com/second": False}),
)
assert response.status == web.HTTPBadRequest.status_code
assert b"requires a timer" in response.body
assert await repo.all() == []
@pytest.mark.asyncio
async def test_add_allows_handler_only(repo) -> None:
request = _json_request(
"POST",
"/api/tasks/",
{"name": "Handler Only", "url": "https://example.com/feed"},
)
response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=True))
assert response.status == web.HTTPOk.status_code
items = await repo.all()
assert len(items) == 1
assert items[0].name == "Handler Only"
@pytest.mark.asyncio
async def test_update_requires_timer_without_handler(repo) -> None:
item = await repo.create({"name": "Needs Timer", "url": "https://example.com/a", "timer": "0 0 * * *"})
request = _json_request(
"PUT",
f"/api/tasks/{item.id}",
{"name": item.name, "url": item.url, "timer": "", "preset": "", "folder": "", "template": "", "cli": ""},
match_info={"id": str(item.id)},
)
response = await router.tasks_update(request, repo, Encoder(), _Notify(), _Handler(matched=False))
assert response.status == web.HTTPBadRequest.status_code
refreshed = await repo.get(item.id)
assert refreshed is not None
assert refreshed.timer == "0 0 * * *"
@pytest.mark.asyncio
async def test_patch_requires_timer_without_handler(repo) -> None:
item = await repo.create({"name": "Patch Timer", "url": "https://example.com/b", "timer": "0 0 * * *"})
request = _json_request(
"PATCH",
f"/api/tasks/{item.id}",
{"timer": ""},
match_info={"id": str(item.id)},
)
response = await router.tasks_patch(request, repo, Encoder(), _Notify(), _Handler(matched=False))
assert response.status == web.HTTPBadRequest.status_code
refreshed = await repo.get(item.id)
assert refreshed is not None
assert refreshed.timer == "0 0 * * *"
@pytest.mark.asyncio
async def test_patch_requires_timer_when_handler_disabled(repo) -> None:
item = await repo.create({"name": "Disabled Handler", "url": "https://example.com/c", "timer": "0 0 * * *"})
request = _json_request(
"PATCH",
f"/api/tasks/{item.id}",
{"timer": "", "handler_enabled": False},
match_info={"id": str(item.id)},
)
response = await router.tasks_patch(request, repo, Encoder(), _Notify(), _Handler(matched=True))
assert response.status == web.HTTPBadRequest.status_code
assert b"handler is disabled" in response.body

View file

@ -1,6 +1,6 @@
from app.library.log import get_logger
import logging
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def cron_time(timer: str) -> str:
@ -12,9 +12,5 @@ def cron_time(timer: str) -> str:
cs = CronSim(timer, datetime.now(UTC))
return cs.explain()
except Exception as exc:
LOG.exception(
"Failed to explain task timer '%s'.",
timer,
extra={"timer": timer, "exception_type": type(exc).__name__},
)
LOG.exception(exc)
return timer

View file

@ -1,12 +1,12 @@
import logging
import os
import threading
import time
from pathlib import Path
from app.library.log import get_logger
from app.library.Singleton import ThreadSafe
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("Archiver")
class _Entry:
@ -169,21 +169,12 @@ class Archiver(metaclass=ThreadSafe):
continue
ids.add(s)
except OSError as e:
LOG.exception(
"Failed to read archive file '%s'.",
key,
extra={"archive_file": key, "operation": "read", "exception_type": type(e).__name__},
)
LOG.error(f"Failed to read archive file '{key}': {e!s}")
ids = set()
try:
elapsed_ms: float = (time.perf_counter() - start) * 1000.0
LOG.debug(
"_ensure_loaded took %.2fms (loaded=%s)",
elapsed_ms,
len(ids),
extra={"archive_file": key, "elapsed_ms": round(elapsed_ms, 2), "loaded_count": len(ids)},
)
LOG.debug(f"_ensure_loaded took {elapsed_ms:.2f}ms (loaded={len(ids)})")
except Exception:
pass
@ -292,17 +283,7 @@ class Archiver(metaclass=ThreadSafe):
f.write("".join(f"{x}\n" for x in new_ids))
except OSError as e:
LOG.exception(
"Failed to write %s item(s) to archive file '%s'.",
len(new_ids),
key,
extra={
"archive_file": key,
"operation": "add",
"item_count": len(new_ids),
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to write to archive file '{key}': {e!s}")
return False
entry.ids.update(new_ids)
@ -354,17 +335,7 @@ class Archiver(metaclass=ThreadSafe):
continue
kept_lines.append(line)
except OSError as e:
LOG.exception(
"Failed to read archive file '%s' before deleting %s item(s).",
key,
len(remove_ids),
extra={
"archive_file": key,
"operation": "delete_read",
"item_count": len(remove_ids),
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed reading archive for delete '{key}': {e!s}")
return False
if not changed:
@ -380,17 +351,7 @@ class Archiver(metaclass=ThreadSafe):
with path.open("w", encoding="utf-8") as f:
f.writelines(kept_lines)
except OSError as e:
LOG.exception(
"Failed to write archive file '%s' after deleting %s item(s).",
key,
len(remove_ids),
extra={
"archive_file": key,
"operation": "delete_write",
"item_count": len(remove_ids),
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed writing archive after delete '{key}': {e!s}")
return False
if entry.loaded:

View file

@ -1,9 +1,7 @@
import asyncio
import functools
import logging
import multiprocessing
import pickle
import sys
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path
from typing import Any
@ -12,51 +10,10 @@ from aiohttp import web
from app.features.ytdlp.utils import _DATA, LogWrapper, get_archive_id
from app.features.ytdlp.ytdlp import YTDLP
from app.library.log import get_logger
from app.library.Services import Services
from app.library.Singleton import Singleton
LOG = get_logger()
LIVE_REEXTRACT_STATUSES: set[str] = {"is_live", "post_live"}
REEXTRACT_INFO_KEY = "_ytptube_reextract"
def _ytdlp_logger(target: logging.Logger):
return _YTDLPLogger(target)
class _YTDLPLogger:
def __init__(self, target: logging.Logger) -> None:
self.target = target
def __call__(self, level: int, msg: str, *args: Any, **kwargs: Any) -> None:
kwargs.setdefault("stacklevel", 4)
if level <= logging.DEBUG and isinstance(msg, str) and msg.startswith("[debug] "):
self.target.debug(msg.removeprefix("[debug] "), *args, **kwargs)
return
if level <= logging.DEBUG:
self.target.info(msg, *args, **kwargs)
return
self.target.log(level, msg, *args, **kwargs)
class _LogCapture:
def __init__(self, logs: list[str]) -> None:
self.logs = logs
def __call__(self, _: int, msg: str, *args: Any, **__: Any) -> None:
self.logs.append(msg % args if args else msg)
def _get_process_pool_kwargs() -> dict[str, Any]:
"""Use a fork-based pool for frozen Linux builds."""
if sys.platform == "linux" and getattr(sys, "frozen", False):
return {"mp_context": multiprocessing.get_context("fork")}
return {}
LOG: logging.Logger = logging.getLogger("downloads.extractor")
class ExtractorConfig:
@ -82,17 +39,6 @@ class ExtractorConfig:
self.wait_threshold = wait_threshold
def _sleep_timeout(config: dict[str, Any], timeout: float, budget_sleep: bool) -> float:
if not budget_sleep:
return timeout
sleep_requests = config.get("sleep_interval_requests")
if not isinstance(sleep_requests, int | float) or sleep_requests <= 0:
return timeout
return timeout + min(float(sleep_requests) * 20, 300.0)
class ExtractorPool(metaclass=Singleton):
"""
Manages process pool and semaphore for video information extraction.
@ -120,7 +66,7 @@ class ExtractorPool(metaclass=Singleton):
def attach(self, app: web.Application) -> None:
"""Attach the extractor pool to the application (no-op for now)."""
app.on_shutdown.append(self.shutdown)
Services.get_instance().add(type(self).__name__, self)
Services.get_instance().add(__class__.__name__, self)
def _ensure_initialized(self, config: ExtractorConfig) -> None:
"""
@ -137,7 +83,7 @@ class ExtractorPool(metaclass=Singleton):
self._semaphore = asyncio.Semaphore(config.concurrency)
if self._pool is None:
self._pool = ProcessPoolExecutor(max_workers=config.concurrency, **_get_process_pool_kwargs())
self._pool = ProcessPoolExecutor(max_workers=config.concurrency)
LOG.info("Initialized extractor process pool with %s workers", config.concurrency)
def get_pool(self, config: ExtractorConfig) -> ProcessPoolExecutor:
@ -181,10 +127,7 @@ class ExtractorPool(metaclass=Singleton):
self._pool.shutdown(wait=False, cancel_futures=False)
LOG.debug("Extractor process pool shutdown complete")
except Exception as exc:
LOG.exception(
"Failed to shut down the extractor process pool.",
extra={"exception_type": type(exc).__name__},
)
LOG.error("Error shutting down extractor process pool: %s", exc)
else:
self._pool = None
@ -254,23 +197,6 @@ def _sanitize_config(config: dict[str, Any]) -> dict[str, Any]:
return sanitized
def needs_reextract(info: dict[str, Any]) -> bool:
return bool(info.get("is_live") or info.get("live_status") in LIVE_REEXTRACT_STATUSES)
def _process_safe_info(info: dict[str, Any]) -> dict[str, Any]:
if needs_reextract(info):
info = dict(info)
info[REEXTRACT_INFO_KEY] = True
for key in ("formats", "requested_formats", "requested_downloads", "fragments"):
info.pop(key, None)
if _is_picklable(info):
return info
return YTDLP.sanitize_info(info, remove_private_keys=False)
def extract_info_sync(
config: dict[str, Any],
url: str,
@ -279,9 +205,8 @@ def extract_info_sync(
follow_redirect: bool = False,
sanitize_info: bool = False,
capture_logs: int | None = None,
process_safe: bool = False,
**kwargs,
) -> tuple[dict[str, Any] | None, list[str]]:
) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]:
"""
Extract video information from a URL.
@ -293,11 +218,10 @@ def extract_info_sync(
follow_redirect: Follow URL redirects
sanitize_info: Sanitize the extracted information
capture_logs: If provided (e.g., logging.WARNING), capture logs at this level.
process_safe: Strip non-pickleable data for safe inter-process communication.
**kwargs: Additional arguments
Returns:
tuple[dict | None, list[str]]: Extracted information and captured logs.
tuple[dict | None, list[dict]]: Extracted information and captured logs.
"""
params: dict[str, Any] = {**config, **_DATA.YTDLP_PARAMS, "simulate": True}
@ -306,81 +230,67 @@ def extract_info_sync(
params["verbose"] = True
params.pop("quiet", None)
suppress: tuple[str, ...] = ()
patterns = kwargs.get("suppress_logs")
if isinstance(patterns, list | tuple):
suppress = tuple(value for value in patterns if isinstance(value, str) and value)
log_wrapper = LogWrapper(suppress=suppress)
log_wrapper = LogWrapper()
id_dict: dict[str, str | None] = get_archive_id(url=url)
archive_id: str | None = f".{id_dict['id']}" if id_dict.get("id") else None
logger_name: str = f"yt-dlp{archive_id or '.extract_info'}"
try:
log_wrapper.add_target(
target=logging.getLogger(f"yt-dlp{archive_id or '.extract_info'}"),
level=logging.DEBUG if debug else logging.WARNING,
)
captured_logs: list[str] = kwargs.get("captured_logs", [])
if capture_logs is not None:
log_wrapper.add_target(
target=_ytdlp_logger(logging.getLogger(logger_name)),
level=logging.DEBUG,
name=logger_name,
target=lambda _, msg: captured_logs.append(msg),
level=capture_logs,
name="log-capture",
)
captured_logs: list[str] = kwargs.get("captured_logs", [])
if capture_logs is not None:
log_wrapper.add_target(
target=_LogCapture(captured_logs),
level=capture_logs,
name="log-capture",
)
if log_wrapper.has_targets():
if "logger" in params:
log_wrapper.add_target(target=params["logger"], level=logging.DEBUG)
if log_wrapper.has_targets():
if "logger" in params:
log_wrapper.add_target(target=params["logger"], level=logging.DEBUG)
params["logger"] = log_wrapper
params["logger"] = log_wrapper
if kwargs.get("no_log", False):
params["logger"] = LogWrapper()
params["quiet"] = True
params["no_warnings"] = True
if kwargs.get("no_log", False):
params["logger"] = LogWrapper()
params["quiet"] = True
params["no_warnings"] = True
if no_archive and "download_archive" in params:
del params["download_archive"]
if no_archive and "download_archive" in params:
del params["download_archive"]
data: dict[str, Any] | None = YTDLP(params=params).extract_info(url, download=False)
data: dict[str, Any] | None = YTDLP(params=params).extract_info(url, download=False)
if data and follow_redirect and "_type" in data and "url" == data["_type"]:
return extract_info_sync(
config,
data["url"],
debug=debug,
no_archive=no_archive,
follow_redirect=follow_redirect,
sanitize_info=sanitize_info,
capture_logs=capture_logs,
captured_logs=captured_logs,
**kwargs,
)
if data and follow_redirect and "_type" in data and "url" == data["_type"]:
return extract_info_sync(
config,
data["url"],
debug=debug,
no_archive=no_archive,
follow_redirect=follow_redirect,
sanitize_info=sanitize_info,
capture_logs=capture_logs,
process_safe=process_safe,
captured_logs=captured_logs,
**kwargs,
)
if not data:
return (data, captured_logs)
if not data:
return (data, captured_logs)
try:
from app.features.ytdlp.mini_filter import match_str
try:
from app.features.ytdlp.mini_filter import match_str
data["is_premiere"] = match_str("media_type=video & duration & is_live", data)
if not data["is_premiere"]:
data["is_premiere"] = "video" == data.get("media_type") and "is_upcoming" == data.get("live_status")
except ImportError:
pass
data["is_premiere"] = match_str("media_type=video & duration & is_live", data)
if not data["is_premiere"]:
data["is_premiere"] = "video" == data.get("media_type") and "is_upcoming" == data.get("live_status")
except ImportError:
pass
result = YTDLP.sanitize_info(data, remove_private_keys=True) if sanitize_info else data
result = YTDLP.sanitize_info(data, remove_private_keys=True) if sanitize_info else data
if process_safe and isinstance(result, dict):
result = _process_safe_info(result)
return (result, captured_logs)
finally:
logging.Logger.manager.loggerDict.pop(logger_name, None)
return (result, captured_logs)
async def fetch_info(
@ -392,9 +302,8 @@ async def fetch_info(
sanitize_info: bool = False,
capture_logs: int | None = None,
extractor_config: ExtractorConfig | None = None,
budget_sleep: bool = False,
**kwargs,
) -> tuple[dict[str, Any] | None, list[str]]:
) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]:
"""
Extract video information from a URL.
@ -410,11 +319,10 @@ async def fetch_info(
sanitize_info: Sanitize the extracted information
capture_logs: If provided (e.g., logging.WARNING), capture logs
extractor_config: Configuration for the extractor
budget_sleep: Whether to add extra timeout budget for request-sleep-heavy extraction
**kwargs: Additional arguments
Returns:
tuple[dict | None, list[str]]: Extracted information and captured logs.
tuple[dict | None, list[dict]]: Extracted information and captured logs.
"""
if extractor_config is None:
@ -434,9 +342,6 @@ async def fetch_info(
loop = asyncio.get_running_loop()
safe_config = _sanitize_config(config)
safe_kwargs = _sanitize_picklable(kwargs)
safe_kwargs.pop("process_safe", None)
timeout = _sleep_timeout(safe_config, extractor_config.timeout, budget_sleep)
try:
try:
@ -454,34 +359,14 @@ async def fetch_info(
follow_redirect=follow_redirect,
sanitize_info=sanitize_info,
capture_logs=capture_logs,
process_safe=True,
**safe_kwargs,
**kwargs,
),
),
timeout=timeout,
timeout=extractor_config.timeout,
)
except TimeoutError:
raise
except Exception as exc:
LOG.warning(
"yt-dlp extraction for '%s' fell back to the thread pool after the process pool failed.",
url,
extra={
"url": url,
"timeout": timeout,
"concurrency": extractor_config.concurrency,
"pool": "process",
"fallback": "thread",
"follow_redirect": follow_redirect,
"no_archive": no_archive,
"sanitize_info": sanitize_info,
"capture_logs_level": capture_logs,
"exception_type": type(exc).__name__,
},
exc_info=True,
)
LOG.warning("extract_info process pool failed, falling back to thread pool url=%s error=%s", url, exc)
return await asyncio.wait_for(
fut=loop.run_in_executor(
None,
@ -497,7 +382,7 @@ async def fetch_info(
**kwargs,
),
),
timeout=timeout,
timeout=extractor_config.timeout,
)
finally:
semaphore.release()

View file

@ -1,14 +1,9 @@
from __future__ import annotations
import operator
import re
from typing import TYPE_CHECKING, Any
from typing import Any
if TYPE_CHECKING:
from collections.abc import Callable
type TOKEN = tuple[str, str]
type AST_NODE = tuple[Any, ...]
TOKEN = tuple[str, str]
AST_NODE = tuple[str, ...]
def match_str(expr: str, dct: dict) -> bool:
@ -47,7 +42,7 @@ class MiniFilter:
ASTNode = tuple[str, ...]
# Supported string operators
STRING_OPERATORS: dict[str, Callable[[Any, Any], bool]] = {
STRING_OPERATORS: dict[str, callable] = {
"*=": operator.contains,
"^=": lambda attr, value: isinstance(attr, str) and attr.startswith(value),
"$=": lambda attr, value: isinstance(attr, str) and attr.endswith(value),
@ -55,7 +50,7 @@ class MiniFilter:
}
# Comparison operators (numeric + string)
COMPARISON_OPERATORS: dict[str, Callable[[Any, Any], bool]] = {
COMPARISON_OPERATORS: dict[str, callable] = {
**STRING_OPERATORS,
"<=": operator.le,
"<": operator.lt,
@ -65,7 +60,7 @@ class MiniFilter:
}
# Unary operators (for presence/absence checks)
UNARY_OPERATORS: dict[str, Callable[[Any], bool]] = {
UNARY_OPERATORS: dict[str, callable] = {
"": lambda v: (v is True) if isinstance(v, bool) else (v is not None),
"!": lambda v: (v is False) if isinstance(v, bool) else (v is None),
}
@ -80,12 +75,9 @@ class MiniFilter:
self.tokens: list[TOKEN] = self._tokenize(expr)
self.pos: int = 0
self.ast: AST_NODE = self._parse_or()
if self.pos != len(self.tokens):
msg = f"Unexpected token {self.tokens[self.pos][1]!r}"
raise SyntaxError(msg)
@staticmethod
def run(expr: str, dct: dict[str, Any] | bool = False) -> bool:
def run(expr: str, dct: dict | bool = False) -> bool:
"""
Convenience method to evaluate an expression directly.
@ -95,15 +87,14 @@ class MiniFilter:
"""
return MiniFilter(expr).evaluate(dct)
def evaluate(self, dct: dict[str, Any] | bool = False) -> bool:
def evaluate(self, dct: dict | bool = False) -> bool:
"""
Evaluate the parsed expression against a dictionary.
:param dct: Dictionary of attributes (video metadata, etc.)
:return: True/False result of expression
"""
data: dict[str, Any] = dct if isinstance(dct, dict) else {}
return self._eval(self.ast, data)
return self._eval(self.ast, dct)
def export(self) -> list[str]:
"""
@ -124,123 +115,33 @@ class MiniFilter:
def _tokenize(self, expr: str) -> list[TOKEN]:
"""
Split expression into tokens (AND, OR, LPAREN, RPAREN, ATOM).
Parentheses, OR, and AND are only treated as syntax outside quoted values.
Ensures that OR (||) and AND (&) are recognized before ATOM.
"""
tokens: list[TOKEN] = []
atom: list[str] = []
quote: str | None = None
escaped: bool = False
idx: int = 0
def flush_atom() -> None:
if not atom:
return
normalized_atom: str = self._normalize_atom("".join(atom))
if normalized_atom:
tokens.append(("ATOM", normalized_atom))
atom.clear()
while idx < len(expr):
char: str = expr[idx]
if quote:
atom.append(char)
if escaped:
escaped = False
elif "\\" == char:
escaped = True
elif quote == char:
quote = None
idx += 1
continue
if escaped:
atom.append(char)
escaped = False
idx += 1
continue
if "\\" == char:
atom.append(char)
escaped = True
idx += 1
continue
if char in {'"', "'"}:
atom.append(char)
quote = char
idx += 1
continue
if expr.startswith("||", idx):
flush_atom()
tokens.append(("OR", "||"))
idx += 2
continue
if "&" == char:
flush_atom()
tokens.append(("AND", "&"))
idx += 1
continue
if "(" == char:
flush_atom()
tokens.append(("LPAREN", "("))
idx += 1
continue
if ")" == char:
flush_atom()
tokens.append(("RPAREN", ")"))
idx += 1
continue
if char.isspace() and not atom:
idx += 1
continue
atom.append(char)
idx += 1
if quote:
msg = "Unterminated quoted string in filter expression"
raise SyntaxError(msg)
flush_atom()
return tokens
def _normalize_atom(self, atom: str) -> str:
normalized_atom: str = atom.strip()
# First, let's normalize spaces around operators to make parsing easier
# Replace spaced operators with non-spaced ones
normalized_expr: str = expr
for op in ["<=", ">=", "<", ">", "*=", "^=", "$=", "~=", "="]:
# Replace "key op value" with "key op value" (removing extra spaces)
pattern: str = rf"([a-z_]+)\s*(!?)\s*({re.escape(op)})\s*"
normalized_atom = re.sub(pattern, r"\1\2\3", normalized_atom, count=1)
replacement = r"\1\2\3"
normalized_expr = re.sub(pattern, replacement, normalized_expr)
quote: str | None = None
escaped: bool = False
for idx, char in enumerate(normalized_atom):
if quote:
if escaped:
escaped = False
elif "\\" == char:
escaped = True
elif quote == char:
quote = None
continue
if char in {'"', "'"}:
quote = char
continue
if char.isspace():
unexpected: str = normalized_atom[idx:].strip()
if unexpected:
msg = f"Unexpected token {unexpected!r}"
raise SyntaxError(msg)
return normalized_atom
token_spec: list[set] = [
(r"\|\|", "OR"), # must come before ATOM
(r"&", "AND"),
(r"\(", "LPAREN"),
(r"\)", "RPAREN"),
(r"[^\s&|()]+", "ATOM"), # don't allow whitespace in ATOM
(r"\s+", None), # skip whitespace
]
regex: str = "|".join(f"(?P<{name}>{pat})" for pat, name in token_spec if name)
master: re.Pattern[str] = re.compile(regex)
tokens: list[TOKEN] = []
for m in master.finditer(normalized_expr):
kind: str | None = m.lastgroup
if kind:
tokens.append((kind, m.group()))
return tokens
def _parse_or(self) -> AST_NODE:
"""
@ -296,7 +197,7 @@ class MiniFilter:
raise SyntaxError("Expected " + kind)
def _eval(self, node: AST_NODE, dct: dict[str, Any]) -> bool:
def _eval(self, node: AST_NODE, dct: dict) -> bool:
"""
Recursively evaluate AST node against dict.
"""
@ -333,9 +234,7 @@ class MiniFilter:
raise ValueError("Invalid AST node " + node_type)
def _match_one(self, filter_part: str, dct: dict[str, Any]) -> bool:
filter_part = filter_part.replace(r"\&", "&")
def _match_one(self, filter_part: str, dct: dict) -> bool:
operator_rex: re.Pattern[str] = re.compile(
r"""(?x)
(?P<key>[a-z_]+)

View file

@ -1,113 +0,0 @@
from __future__ import annotations
import re
import secrets
import string
from typing import TYPE_CHECKING, Any
from yt_dlp.utils._utils import STR_FORMAT_RE_TMPL, STR_FORMAT_TYPES
if TYPE_CHECKING:
from collections.abc import Callable
OUTTMPL_RX: re.Pattern[str] = re.compile(STR_FORMAT_RE_TMPL.format("[^)]*", f"[{STR_FORMAT_TYPES}ljhqBUDS]"))
CALL_RX: re.Pattern[str] = re.compile(r"^(?P<name>ytp_[A-Za-z0-9_]+)(?P<args>(?::[A-Za-z0-9_]+)*)(?P<rest>.*)$")
OPERATORS: tuple[str, ...] = (",", "&", "|", ">", "+", "-", "*")
def random_text(args: tuple[str, ...], _info_dict: dict[str, Any], _state: dict[str, Any]) -> str:
if not args:
msg = "ytp_random requires a length argument. Use %(ytp_random:<length>)s."
raise ValueError(msg)
if len(args) > 2:
msg = "ytp_random accepts at most 2 arguments: length and optional mode."
raise ValueError(msg)
try:
length = int(args[0])
except ValueError as exc:
msg: str = f"ytp_random length must be an integer, got {args[0]!r}."
raise ValueError(msg) from exc
if length < 1:
msg = "ytp_random length must be greater than 0."
raise ValueError(msg)
mode = args[1].lower() if len(args) > 1 else "mixed"
if mode in ("mixed", "m"):
alphabet = string.ascii_letters + string.digits
elif mode in ("d", "digit", "digits", "int", "ints", "number", "numbers"):
alphabet = string.digits
elif mode in ("s", "str", "string", "strings", "alpha", "letter", "letters"):
alphabet = string.ascii_letters
else:
msg = f"ytp_random mode must be one of mixed, d, or s. Got {args[1]!r}."
raise ValueError(msg)
return "".join(secrets.choice(alphabet) for _ in range(length))
CALLS: dict[str, Callable[[tuple[str, ...], dict[str, Any], dict[str, Any]], Any]] = {
"ytp_random": random_text,
}
def split_call(key: str) -> tuple[str, tuple[str, ...], str] | None:
if not key.startswith("ytp_"):
return None
if not (match := CALL_RX.match(key)):
msg = f"Invalid YTPTube output template callable {key!r}."
raise ValueError(msg)
name: str | Any = match.group("name")
rest: str | Any = match.group("rest") or ""
if rest and rest[0] not in OPERATORS:
msg = f"Invalid YTPTube output template callable {key!r}."
raise ValueError(msg)
if name not in CALLS:
msg = f"Unsupported YTPTube output template callable {name!r}."
raise ValueError(msg)
raw_args: str | Any = match.group("args")
args: tuple[str | Any, ...] = tuple(part for part in raw_args.split(":") if part)
return name, args, rest
def rewrite_outtmpl(
outtmpl: str,
info_dict: dict[str, Any],
cache: dict[str, Any] | None = None,
) -> tuple[str, dict[str, Any]]:
if "%(ytp_" not in outtmpl:
return outtmpl, info_dict
state: dict[str, Any] = {}
values: dict[str, Any] = {} if cache is None else cache
fields: dict[str, str] = {}
enriched = dict(info_dict)
def replace(match: re.Match[str]) -> str:
if not (key := match.group("key")):
return match.group(0)
parsed = split_call(key)
if not parsed:
return match.group(0)
name, args, rest = parsed
call_key: str = ":".join((name, *args)) if args else name
if call_key not in values:
values[call_key] = CALLS[name](args, enriched, state)
if call_key not in fields:
fields[call_key] = f"__ytptube_outtmpl_{len(fields)}"
synthetic_key: str = fields[call_key]
enriched[synthetic_key] = values[call_key]
return f"{match.group('prefix')}%({synthetic_key}{rest}){match.group('format')}"
return OUTTMPL_RX.sub(replace, outtmpl), enriched

View file

@ -1,109 +0,0 @@
import subprocess
import sys
from typing import Any
from app.library.log import get_logger
LOG = get_logger()
def patch_metadataparser() -> None:
"""
Patches yt_dlp MetadataParserPP action to handle subprocess pickling issues.
"""
try:
from yt_dlp.postprocessor.metadataparser import MetadataParserPP
from yt_dlp.utils import Namespace
except Exception as exc:
LOG.warning(
"Unable to import yt-dlp metadata parser for patching: %s",
exc,
extra={"patch": "metadata_parser", "exception_type": type(exc).__name__},
exc_info=True,
)
return
if getattr(MetadataParserPP.Actions, "_ytptube_patched", False):
return
class _ActionNS(Namespace):
_ACTIONS_STR: list[str] = []
@staticmethod
def _get_name(func) -> str | None:
if not callable(func):
return None
target = getattr(func, "__func__", func)
module_name = getattr(target, "__module__", None)
qual_name = getattr(target, "__qualname__", getattr(target, "__name__", None))
return f"{module_name}.{qual_name}" if module_name and qual_name else None
def __contains__(self, candidate: object) -> bool:
if candidate in self.__dict__.values():
return True
if func_name := _ActionNS._get_name(candidate):
if len(_ActionNS._ACTIONS_STR) < 1:
_ActionNS._ACTIONS_STR.extend(
[value for value in (_ActionNS._get_name(value) for value in self.__dict__.values()) if value]
)
return func_name in _ActionNS._ACTIONS_STR
return False
actions_dict: dict[str, Any] = dict(MetadataParserPP.Actions.items_)
MetadataParserPP.Actions = _ActionNS(**actions_dict)
MetadataParserPP.Actions._ytptube_patched = True
LOG.debug("MetadataParserPP action namespace patch applied successfully.")
def patch_windows_popen_wait() -> None:
if sys.platform != "win32":
return
try:
from yt_dlp.utils import Popen
except Exception as exc:
LOG.warning(
"Unable to import yt-dlp Popen for patching: %s",
exc,
extra={"patch": "windows_popen_wait", "exception_type": type(exc).__name__},
exc_info=True,
)
return
if getattr(Popen, "_ytptube_wait_patched", False):
return
original_wait = Popen.wait
# Windows subprocess waits can swallow the synthetic interrupt we use to
# stop live downloads, especially while yt-dlp is blocked on ffmpeg.
def interruptible_wait(self, timeout=None):
if timeout is not None:
return original_wait(self, timeout=timeout)
while True:
try:
return original_wait(self, timeout=0.1)
except subprocess.TimeoutExpired:
continue
Popen.wait = interruptible_wait
Popen._ytptube_wait_patched = True
LOG.debug("yt_dlp Popen.wait Windows patch applied successfully.")
def apply_ytdlp_patches() -> None:
try:
patch_metadataparser()
except Exception as exc:
LOG.debug("Metadata parser patch failed to apply: %s", exc)
try:
patch_windows_popen_wait()
except Exception as exc:
LOG.debug("Windows Popen wait patch failed to apply: %s", exc)

View file

@ -1,4 +1,3 @@
import asyncio
import json
import logging
import time
@ -18,11 +17,10 @@ from app.library.cache import Cache
from app.library.config import Config
from app.library.encoder import Encoder
from app.library.ItemDTO import Item
from app.library.log import get_logger
from app.library.router import route
from app.library.Utils import validate_url
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def _get_preset_archive(preset: str) -> str | None:
@ -38,12 +36,7 @@ def _get_preset_archive(preset: str) -> str | None:
try:
opts: dict = YTDLPOpts.get_instance().preset(preset).get_all()
except Exception as e:
LOG.exception(
"Failed to build yt-dlp options for preset '%s': %s.",
preset,
e,
extra={"preset": preset, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to build yt-dlp opts for preset '{preset}'. {e!s}")
return None
if not (archive_file := opts.get("download_archive")):
@ -131,19 +124,7 @@ async def archiver_get(request: Request) -> Response:
data={"file": archive_file, "items": data, "count": len(data)}, status=web.HTTPOk.status_code
)
except Exception as e:
LOG.exception(
"Failed to read archive file '%s' for preset '%s': %s.",
archive_file,
preset,
e,
extra={
"route": "api/archiver/",
"action": "read_archive",
"preset": preset,
"archive_file": archive_file,
"ids": ids,
},
)
LOG.exception(e)
return web.json_response(
data={"error": f"Failed to read archive file for preset '{preset}'.", "message": str(e)},
status=web.HTTPInternalServerError.status_code,
@ -189,21 +170,7 @@ async def archiver_add(request: Request) -> Response:
status=web.HTTPOk.status_code if status else web.HTTPNotModified.status_code,
)
except Exception as e:
LOG.exception(
"Failed to add %s item(s) to archive file '%s' for preset '%s': %s.",
len(items),
archive_file,
preset,
e,
extra={
"route": "api/archiver/",
"action": "add_archive_items",
"preset": preset,
"archive_file": archive_file,
"item_count": len(items),
"skip_check": skip_check,
},
)
LOG.exception(e)
return web.json_response(
data={"error": f"Failed to add items to archive for preset '{preset}'.", "message": str(e)},
status=web.HTTPInternalServerError.status_code,
@ -247,20 +214,7 @@ async def archiver_delete(request: Request) -> Response:
status=web.HTTPOk.status_code if status else web.HTTPNotModified.status_code,
)
except Exception as e:
LOG.exception(
"Failed to delete %s item(s) from archive file '%s' for preset '%s': %s.",
len(items),
archive_file,
preset,
e,
extra={
"route": "api/archiver/",
"action": "delete_archive_items",
"preset": preset,
"archive_file": archive_file,
"item_count": len(items),
},
)
LOG.exception(e)
return web.json_response(
data={"error": f"Failed to delete items from archive for preset '{preset}'.", "message": str(e)},
status=web.HTTPInternalServerError.status_code,
@ -286,7 +240,7 @@ async def convert(request: Request) -> Response:
return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code)
try:
response: dict[str, Any] = {"opts": {}, "output_template": None, "download_path": None}
response = {"opts": {}, "output_template": None, "download_path": None}
data = arg_converter(args, dumps=True)
@ -347,7 +301,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
)
try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
validate_url(url, allow_internal=config.allow_internal_urls)
except ValueError as e:
return web.json_response(
data={"status": False, "message": str(e), "error": str(e)},
@ -384,8 +338,6 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
if cache.has(key) and not request.query.get("force", False):
data: Any | None = cache.get(key)
if data is None:
data = {}
data["_cached"] = {
"status": "hit",
"preset": preset,
@ -395,7 +347,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
"ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(),
"expires": data.get("_cached", {}).get("expires", time.time() + 300),
}
return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
ytdlp_opts: dict = opts.get_all()
@ -452,21 +404,10 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
cache.set(key=key, value=data, ttl=300)
return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
except Exception as e:
LOG.exception(
"Failed to get video info for '%s': %s.",
url,
e,
extra={
"route": "api/yt-dlp/url/info/",
"action": "get_video_info",
"url": url,
"preset": preset,
"cache_key": key if "key" in locals() else None,
"has_cli_args": bool(cli_args),
},
)
LOG.exception(e)
LOG.error(f"Error encountered while getting video info for '{url}'. '{e!s}'.")
return web.json_response(
data={
"error": "failed to get video info.",
@ -488,7 +429,7 @@ async def get_options() -> Response:
"""
from app.features.ytdlp.ytdlp import ytdlp_options
return web.json_response(text=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
return web.json_response(body=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
@route("POST", "api/yt-dlp/archive_id/", "get_archive_ids")
@ -510,13 +451,9 @@ async def get_archive_ids(request: Request, config: Config) -> Response:
response = []
for i, url in enumerate(data):
dct: dict[str, Any] = {"index": i, "url": url}
if not isinstance(url, str):
dct.update({"id": None, "ie_key": None, "archive_id": None, "error": "URL must be a string."})
response.append(dct)
continue
dct = {"index": i, "url": url}
try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
validate_url(url, allow_internal=config.allow_internal_urls)
dct.update(get_archive_id(url))
except ValueError as e:
dct.update({"id": None, "ie_key": None, "archive_id": None, "error": str(e)})
@ -558,19 +495,7 @@ async def make_command(request: Request, config: Config, encoder: Encoder) -> Re
try:
command, info = YTDLPCli(item=it, config=config).build()
except Exception as e:
LOG.exception(
"Failed to build yt-dlp command for '%s': %s.",
it.url,
e,
extra={
"route": "api/yt-dlp/command/",
"action": "build_command",
"url": it.url,
"preset": it.preset,
"has_cookies": bool(it.cookies),
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
return web.json_response(
data={"error": "Failed to build CLI command"},
status=web.HTTPBadRequest.status_code,

View file

@ -1,60 +1,6 @@
import logging
import pickle
from unittest.mock import MagicMock, patch
from app.features.ytdlp.extractor import (
ExtractorConfig,
ExtractorPool,
REEXTRACT_INFO_KEY,
_LogCapture,
_get_process_pool_kwargs,
_process_safe_info,
_ytdlp_logger,
extract_info_sync,
)
from app.features.ytdlp.utils import LogWrapper
class TestProcessPoolConfiguration:
def setup_method(self):
ExtractorPool._reset_singleton()
def test_uses_fork_context_for_frozen_linux(self, monkeypatch):
monkeypatch.setattr("app.features.ytdlp.extractor.sys.platform", "linux")
monkeypatch.setattr("app.features.ytdlp.extractor.sys.frozen", True, raising=False)
context = object()
get_context = MagicMock(return_value=context)
monkeypatch.setattr("app.features.ytdlp.extractor.multiprocessing.get_context", get_context)
kwargs = _get_process_pool_kwargs()
assert kwargs == {"mp_context": context}
get_context.assert_called_once_with("fork")
def test_default_context_linux(self, monkeypatch):
monkeypatch.setattr("app.features.ytdlp.extractor.sys.platform", "linux")
monkeypatch.delattr("app.features.ytdlp.extractor.sys.frozen", raising=False)
get_context = MagicMock()
monkeypatch.setattr("app.features.ytdlp.extractor.multiprocessing.get_context", get_context)
assert _get_process_pool_kwargs() == {}
get_context.assert_not_called()
def test_init_pool_kwargs(self, monkeypatch):
context = object()
monkeypatch.setattr("app.features.ytdlp.extractor._get_process_pool_kwargs", lambda: {"mp_context": context})
executor = MagicMock()
executor_cls = MagicMock(return_value=executor)
monkeypatch.setattr("app.features.ytdlp.extractor.ProcessPoolExecutor", executor_cls)
pool = ExtractorPool.get_instance()
pool._ensure_initialized(ExtractorConfig(concurrency=3))
executor_cls.assert_called_once_with(max_workers=3, mp_context=context)
assert pool.get_pool(ExtractorConfig(concurrency=3)) is executor
from app.features.ytdlp.extractor import extract_info_sync
class TestExtractInfo:
@ -88,145 +34,3 @@ class TestExtractInfo:
(result, logs) = extract_info_sync(config, url, debug=True)
assert isinstance(result, dict), "Result should be a dictionary"
assert isinstance(logs, list), "Logs should be a list"
@patch("app.features.ytdlp.extractor.YTDLP")
def test_extract_info_mirrors_debug_to_console(self, mock_ytdlp_class):
seen: list[tuple[int, str]] = []
def fake_extract_info(url, download=False): # noqa: ARG001
logger = mock_ytdlp_class.call_args.kwargs["params"]["logger"]
logger.debug("[generic_browser] Using remote browser for https://example.com/video")
logger.debug("[debug] [generic_browser] Loading page https://example.com/video")
logger.warning("[generic_browser] Browser fallback warning")
return {"title": "Test Video", "id": "test123"}
mock_ytdlp = MagicMock()
mock_ytdlp.extract_info.side_effect = fake_extract_info
mock_ytdlp_class.return_value = mock_ytdlp
logger = logging.getLogger("yt-dlp.extract_info")
with patch.object(
logger, "info", side_effect=lambda msg, *a, **k: seen.append((logging.INFO, msg % a if a else msg))
):
with patch.object(
logger,
"debug",
side_effect=lambda msg, *a, **k: seen.append((logging.DEBUG, msg % a if a else msg)),
):
with patch.object(
logger,
"log",
side_effect=lambda level, msg, *a, **k: seen.append((level, msg % a if a else msg)),
):
(result, logs) = extract_info_sync(
{}, "https://example.com/video", debug=True, capture_logs=logging.WARNING
)
assert result is not None
assert result["id"] == "test123"
assert logs == ["[generic_browser] Browser fallback warning"]
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
assert (logging.DEBUG, "[generic_browser] Loading page https://example.com/video") in seen
assert (logging.WARNING, "[generic_browser] Browser fallback warning") in seen
@patch("app.features.ytdlp.extractor.YTDLP")
def test_extract_info_mirrors_screen_logs_without_debug(self, mock_ytdlp_class):
seen: list[tuple[int, str]] = []
def fake_extract_info(url, download=False): # noqa: ARG001
logger = mock_ytdlp_class.call_args.kwargs["params"]["logger"]
logger.debug("[generic_browser] Using remote browser for https://example.com/video")
logger.warning("[generic_browser] Browser fallback warning")
return {"title": "Test Video", "id": "test123"}
mock_ytdlp = MagicMock()
mock_ytdlp.extract_info.side_effect = fake_extract_info
mock_ytdlp_class.return_value = mock_ytdlp
logger = logging.getLogger("yt-dlp.extract_info")
with patch.object(
logger, "info", side_effect=lambda msg, *a, **k: seen.append((logging.INFO, msg % a if a else msg))
):
with patch.object(
logger,
"log",
side_effect=lambda level, msg, *a, **k: seen.append((level, msg % a if a else msg)),
):
(result, logs) = extract_info_sync(
{}, "https://example.com/video", debug=False, capture_logs=logging.WARNING
)
assert result is not None
assert result["id"] == "test123"
assert logs == ["[generic_browser] Browser fallback warning"]
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
assert (logging.WARNING, "[generic_browser] Browser fallback warning") in seen
def test_process_safe_live(self) -> None:
data = {
"id": "live-id",
"is_live": True,
"formats": [{"format_id": "dash", "fragments": ({"url": "https://example.test/sq/1"} for _ in range(1))}],
"requested_formats": [{"format_id": "dash"}],
}
result = _process_safe_info(data)
assert result[REEXTRACT_INFO_KEY] is True
assert "formats" not in result
assert "requested_formats" not in result
pickle.dumps(result)
def test_process_safe_post_live(self) -> None:
data = {
"id": "post-live-id",
"live_status": "post_live",
"formats": [{"format_id": "dash", "fragments": ({"url": "https://example.test/sq/1"} for _ in range(1))}],
}
result = _process_safe_info(data)
assert result[REEXTRACT_INFO_KEY] is True
assert "formats" not in result
pickle.dumps(result)
def test_process_safe_lazy(self) -> None:
from yt_dlp.utils import LazyList
data = {"id": "video-id", "formats": LazyList({"format_id": str(i)} for i in range(2))}
result = _process_safe_info(data)
assert result["formats"] == [{"format_id": "0"}, {"format_id": "1"}]
pickle.dumps(result)
class TestYtdlpLogger:
def test_debug_prefix_uses_debug(self) -> None:
logger = MagicMock()
_ytdlp_logger(logger)(logging.DEBUG, "[debug] hello")
logger.debug.assert_called_once_with("hello", stacklevel=4)
def test_screen_style_debug_uses_info(self) -> None:
logger = MagicMock()
_ytdlp_logger(logger)(logging.DEBUG, "screen line")
logger.info.assert_called_once_with("screen line", stacklevel=4)
def test_targets_are_picklable(self) -> None:
logs: list[str] = []
wrapper = LogWrapper()
wrapper.add_target(_ytdlp_logger(logging.getLogger("yt-dlp.test")), level=logging.DEBUG, name="yt-dlp.test")
wrapper.add_target(_LogCapture(logs), level=logging.WARNING, name="log-capture")
pickle.dumps(wrapper)
def test_capture_formats_args(self) -> None:
logs: list[str] = []
_LogCapture(logs)(logging.WARNING, "hello %s", "world")
assert logs == ["hello world"]

View file

@ -1,8 +1,6 @@
import unittest
from typing import Any
import pytest
from app.features.ytdlp.mini_filter import MiniFilter
@ -195,7 +193,7 @@ class TestMiniFilter(unittest.TestCase):
self._test("filesize>=1GiB", {"filesize": 2**30}, expected_result=True, test_name="filesize_1gib_positive")
self._test("filesize>=1GiB", {"filesize": 1000000}, expected_result=False, test_name="filesize_1gib_negative")
def test_duration_or_grouping(self):
def test_complex_duration_units_with_or_and_grouping(self):
"""Test complex expressions with duration units, OR operations, and grouping. Skip yt-dlp due to known parsing bugs."""
# Test grouping with duration units
expr = "(filesize>1MB & duration<10m) || uploader='BBC'"
@ -276,65 +274,6 @@ class TestMiniFilter(unittest.TestCase):
self._test("uploader^='CNN'", d, expected_result=False, test_name="string_startswith_negative")
self._test("uploader$='BBC'", d, expected_result=False, test_name="string_endswith_negative")
def test_regex_inline_flags_tokenization(self):
expr = "artist ~= '(?i)^[0-9A-L]'"
parser = MiniFilter(expr)
assert parser.tokens == [("ATOM", "artist~='(?i)^[0-9A-L]'")]
assert parser.ast == ("ATOM", "artist~='(?i)^[0-9A-L]'")
assert parser.export() == ["artist~='(?i)^[0-9A-L]'"]
self._test(expr, {"artist": "Adele"}, expected_result=True, test_name="regex_inline_flags_match")
self._test(expr, {"artist": "muse"}, expected_result=False, test_name="regex_inline_flags_no_match")
def test_quoted_values_with_spaces(self):
expr = "uploader = 'BBC News'"
parser = MiniFilter(expr)
assert parser.tokens == [("ATOM", "uploader='BBC News'")]
assert parser.export() == ["uploader='BBC News'"]
self._test(expr, {"uploader": "BBC News"}, expected_result=True, test_name="quoted_spaces_match")
self._test(expr, {"uploader": "BBC"}, expected_result=False, test_name="quoted_spaces_no_match")
def test_quoted_values_with_parentheses(self):
expr = "title='a(b)c'"
parser = MiniFilter(expr)
assert parser.tokens == [("ATOM", "title='a(b)c'")]
assert parser.export() == ["title='a(b)c'"]
self._test(expr, {"title": "a(b)c"}, expected_result=True, test_name="quoted_parentheses_match")
self._test(expr, {"title": "abc"}, expected_result=False, test_name="quoted_parentheses_no_match")
def test_quoted_regex_ampersand(self):
expr = r"description~='(?i)\bcats \& dogs\b'"
parser = MiniFilter(expr)
assert parser.tokens == [("ATOM", r"description~='(?i)\bcats \& dogs\b'")]
assert parser.export() == [r"description~='(?i)\bcats \& dogs\b'"]
self._test(
expr,
{"description": "Cats & Dogs are great"},
expected_result=True,
test_name="quoted_regex_escaped_ampersand_match",
)
self._test(
expr,
{"description": "Cats and dogs are great"},
expected_result=False,
test_name="quoted_regex_escaped_ampersand_no_match",
)
def test_parser_rejects_trailing_tokens(self):
with pytest.raises(SyntaxError, match="Unexpected token"):
MiniFilter("uploader='BBC' stray")
def test_spaces_around_operators(self):
"""Test that spaces around operators are handled correctly."""
d: dict[str, str] = {"channel_id": "UC-7oMv6E4Uz2tF51w5Sj49w", "uploader": "BBC"}
@ -528,7 +467,7 @@ class TestMiniFilter(unittest.TestCase):
def test_export_and_yt_dlp_compat(self):
from yt_dlp.utils import match_str
d: dict[str, Any] = {"filesize": 2000000, "duration": 200, "uploader": "BBC"}
d: dict[str, str] = {"filesize": 2000000, "duration": 200, "uploader": "BBC"}
# Use numeric values to avoid yt-dlp unit parsing inconsistencies
expr = "(filesize>1000000 & duration<600) || uploader='BBC'"

View file

@ -1,23 +1,11 @@
import importlib
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, Mock, patch
import pytest
from yt_dlp.globals import extractors as ytdlp_extractors
from app.features.ytdlp.outtmpl import rewrite_outtmpl
from app.features.ytdlp.patches import patch_windows_popen_wait
from app.features.ytdlp.utils import _DATA
from app.features.ytdlp.ytdlp import YTDLP, _ArchiveProxy, ytdlp_options
def _archive_path(tmp_path: Path) -> str:
return str(tmp_path / "archive.txt")
class TestArchiveProxy:
def test_bool_and_falsey_cases(self, tmp_path: Path) -> None:
def test_bool_and_falsey_cases(self) -> None:
# No file path means proxy is falsey and operations return False
p = _ArchiveProxy(file=None)
assert bool(p) is False
@ -25,24 +13,22 @@ class TestArchiveProxy:
assert p.add("id") is False
# Empty item also returns False
p2 = _ArchiveProxy(file=_archive_path(tmp_path))
p2 = _ArchiveProxy(file="/tmp/archive.txt")
assert bool(p2) is True
assert ("" in p2) is False
assert p2.add("") is False
@patch("app.features.ytdlp.archiver.Archiver.get_instance")
def test_delegates_to_archiver(self, mock_get_instance, tmp_path: Path) -> None:
def test_contains_and_add_delegate_to_archiver(self, mock_get_instance) -> None:
arch = MagicMock()
mock_get_instance.return_value = arch
file = _archive_path(tmp_path)
p = _ArchiveProxy(file=file)
p = _ArchiveProxy(file="/tmp/archive.txt")
# contains -> read(file, [item]) and check membership
arch.read.return_value = ["abc"]
assert ("abc" in p) is True
arch.read.assert_called_with(file, ["abc"])
arch.read.assert_called_with("/tmp/archive.txt", ["abc"])
arch.read.return_value = []
assert ("xyz" in p) is False
@ -50,14 +36,14 @@ class TestArchiveProxy:
# add -> add(file, [item]) returns boolean
arch.add.return_value = True
assert p.add("abc") is True
arch.add.assert_called_with(file, ["abc"])
arch.add.assert_called_with("/tmp/archive.txt", ["abc"])
arch.add.return_value = False
assert p.add("xyz") is False
class TestYtDlpOptions:
def test_options_shape(self) -> None:
def test_options_structure_and_no_suppresshelp(self) -> None:
opts = ytdlp_options()
assert isinstance(opts, list)
@ -104,12 +90,11 @@ class TestYTDLP:
return ytdlp
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
def test_init_archive_param(self, mock_super_init, tmp_path: Path) -> None:
def test_init_patches_download_archive_param(self, mock_super_init) -> None:
"""Test that __init__ removes download_archive before calling super, then restores it."""
mock_super_init.return_value = None
file = _archive_path(tmp_path)
params = {"download_archive": file, "quiet": True}
params = {"download_archive": "/tmp/archive.txt", "quiet": True}
ytdlp = YTDLP(params=params, auto_init=True)
# Set params as it would be after super().__init__
@ -123,16 +108,16 @@ class TestYTDLP:
assert call_kwargs["auto_init"] is True
# Our __init__ code manually sets these after super()
ytdlp.params["download_archive"] = file
ytdlp.params["download_archive"] = "/tmp/archive.txt"
assert ytdlp.params["download_archive"] == file, "Verify download_archive was restored to params"
assert ytdlp.params["download_archive"] == "/tmp/archive.txt", "Verify download_archive was restored to params"
# Verify archive proxy was set up
assert isinstance(ytdlp.archive, _ArchiveProxy)
assert ytdlp.archive._file == file
assert ytdlp.archive._file == "/tmp/archive.txt"
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
def test_init_no_archive(self, mock_super_init) -> None:
def test_init_handles_no_download_archive(self, mock_super_init) -> None:
"""Test __init__ works correctly when download_archive is not in params."""
mock_super_init.return_value = None
@ -150,7 +135,7 @@ class TestYTDLP:
assert not ytdlp.archive
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
def test_init_none_params(self, mock_super_init) -> None:
def test_init_handles_none_params(self, mock_super_init) -> None:
"""Test __init__ handles None params gracefully."""
mock_super_init.return_value = None
@ -160,68 +145,12 @@ class TestYTDLP:
assert isinstance(ytdlp.archive, _ArchiveProxy)
assert not ytdlp.archive
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
def test_init_patches_wait(self, mock_super_init) -> None:
mock_super_init.return_value = None
class FakePopen:
def wait(self, timeout=None):
return timeout
with patch("app.features.ytdlp.patches.sys.platform", "win32"):
with patch("yt_dlp.utils.Popen", FakePopen):
YTDLP(params={})
assert getattr(FakePopen, "_ytptube_wait_patched", False) is True
def test_wait_patch_polls(self) -> None:
calls: list[float | None] = []
class FakePopen:
_ytptube_wait_patched = False
def wait(self, timeout=None):
calls.append(timeout)
if len(calls) < 3:
raise TimeoutError
return 0
with patch("app.features.ytdlp.patches.sys.platform", "win32"):
with (
patch("yt_dlp.utils.Popen", FakePopen),
patch("app.features.ytdlp.patches.subprocess.TimeoutExpired", TimeoutError),
):
patch_windows_popen_wait()
result = FakePopen().wait()
assert result == 0
assert calls == [0.1, 0.1, 0.1]
def test_wait_patch_timeout(self) -> None:
calls: list[float | None] = []
class FakePopen:
_ytptube_wait_patched = False
def wait(self, timeout=None):
calls.append(timeout)
return 0
with patch("app.features.ytdlp.patches.sys.platform", "win32"):
with patch("yt_dlp.utils.Popen", FakePopen):
patch_windows_popen_wait()
result = FakePopen().wait(timeout=5)
assert result == 0
assert calls == [5]
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL._delete_downloaded_files")
def test_delete_interrupted(self, mock_super_delete) -> None:
def test_delete_downloaded_files_skips_when_interrupted(self, mock_super_delete) -> None:
"""Test _delete_downloaded_files skips cleanup when _interrupted is True."""
with patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__", return_value=None):
ytdlp = YTDLP(params={})
mock_obj: Any = ytdlp
mock_obj.to_screen = Mock()
ytdlp.to_screen = Mock()
# Set interrupted flag
ytdlp._interrupted = True
@ -231,12 +160,12 @@ class TestYTDLP:
# Should not call super method
mock_super_delete.assert_not_called()
# Should show message
mock_obj.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
ytdlp.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
# Should return None
assert result is None
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL._delete_downloaded_files")
def test_delete_calls_super(self, mock_super_delete) -> None:
def test_delete_downloaded_files_calls_super_when_not_interrupted(self, mock_super_delete) -> None:
"""Test _delete_downloaded_files calls super when not interrupted."""
mock_super_delete.return_value = "cleanup_result"
@ -251,7 +180,7 @@ class TestYTDLP:
# Should return super's result
assert result == "cleanup_result"
def test_record_archive_missing(self) -> None:
def test_record_download_archive_does_nothing_without_download_archive_param(self) -> None:
"""Test record_download_archive returns early when download_archive is not set."""
ytdlp = self._create_ytdlp(params={})
ytdlp.archive = Mock()
@ -261,9 +190,9 @@ class TestYTDLP:
# Should not interact with archive
ytdlp.archive.add.assert_not_called()
def test_record_archive_adds_id(self, tmp_path: Path) -> None:
def test_record_download_archive_adds_archive_id(self) -> None:
"""Test record_download_archive adds the archive ID."""
ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
ytdlp.write_debug = Mock()
ytdlp.archive = Mock()
ytdlp._make_archive_id = Mock(return_value="youtube test123")
@ -279,9 +208,9 @@ class TestYTDLP:
ytdlp.archive.add.assert_called_once_with("youtube test123")
ytdlp.write_debug.assert_called_with("Adding to archive: youtube test123")
def test_record_archive_old_ids(self, tmp_path: Path) -> None:
def test_record_download_archive_handles_old_archive_ids(self) -> None:
"""Test record_download_archive adds _old_archive_ids when present."""
ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
ytdlp.write_debug = Mock()
ytdlp.archive = Mock()
ytdlp._make_archive_id = Mock(return_value="youtube new123")
@ -306,9 +235,9 @@ class TestYTDLP:
# Should not add duplicate (youtube new123 appears only once)
assert calls.count("youtube new123") == 1
def test_record_archive_empty_old_ids(self, tmp_path: Path) -> None:
def test_record_download_archive_skips_empty_old_archive_ids(self) -> None:
"""Test record_download_archive handles empty or invalid _old_archive_ids."""
ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
ytdlp.write_debug = Mock()
ytdlp.archive = Mock()
ytdlp._make_archive_id = Mock(return_value="youtube test123")
@ -332,9 +261,9 @@ class TestYTDLP:
ytdlp.record_download_archive(info_dict)
assert ytdlp.archive.add.call_count == 1 # Only main ID
def test_record_archive_empty_id(self, tmp_path: Path) -> None:
def test_record_download_archive_returns_early_on_empty_archive_id(self) -> None:
"""Test record_download_archive returns early when _make_archive_id returns empty."""
ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
ytdlp.archive = Mock()
ytdlp._make_archive_id = Mock(return_value=None)
@ -342,124 +271,3 @@ class TestYTDLP:
# Should not add anything
ytdlp.archive.add.assert_not_called()
def test_outtmpl_callable(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
result = ytdlp.evaluate_outtmpl("%(ytp_random:8)s", {"title": "x"})
assert len(result) == 8
assert result.isalnum()
def test_exec_init(self) -> None:
YTDLP(
params={
"compat_opts": set(),
"postprocessors": [{"key": "Exec", "exec_cmd": "echo %(title)q"}],
}
)
def test_outtmpl_reuses_value(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
result = ytdlp.evaluate_outtmpl("%(ytp_random:8)s/%(ytp_random:8)s", {"title": "x"})
first, second = result.split("/")
assert first == second
def test_outtmpl_new_value(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
first = ytdlp.evaluate_outtmpl("%(ytp_random:8)s", {"title": "x"})
second = ytdlp.evaluate_outtmpl("%(ytp_random:8)s", {"title": "y"})
assert first != second
def test_prepare_filename_sidecars(self) -> None:
ytdlp = YTDLP(
params={
"outtmpl": {
"default": "%(ytp_random:6)s.%(ext)s",
"thumbnail": "%(ytp_random:6)s.%(ext)s",
"subtitle": "%(ytp_random:6)s.%(ext)s",
"infojson": "%(ytp_random:6)s.%(ext)s",
}
}
)
info = {"id": "abc123", "title": "Example", "ext": "mp4"}
default_name = ytdlp.prepare_filename(info)
thumbnail_name = ytdlp.prepare_filename(info, "thumbnail")
subtitle_name = ytdlp.prepare_filename(info, "subtitle")
infojson_name = ytdlp.prepare_filename(info, "infojson")
default_base = default_name.rsplit(".", 1)[0]
thumbnail_base = thumbnail_name.rsplit(".", 1)[0]
subtitle_base = subtitle_name.rsplit(".", 1)[0]
infojson_base = infojson_name.removesuffix(".info.json")
assert default_base == thumbnail_base
assert default_base == subtitle_base
assert default_base == infojson_base
def test_prepare_filename_resets(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(ytp_random:8)s.%(ext)s"}})
first = ytdlp.prepare_filename({"id": "one", "title": "One", "ext": "mp4"})
second = ytdlp.prepare_filename({"id": "two", "title": "Two", "ext": "mp4"})
assert first != second
@pytest.mark.parametrize(
("template", "expected"),
[
("%(ytp_random:8|fallback)s", 8),
("%(ytp_random:8&{} - |)s", 11),
("%(ytp_random:8)S", 8),
],
)
def test_outtmpl_suffix(self, template: str, expected: int) -> None:
ytdlp = YTDLP(
params={
"outtmpl": {"default": "%(title)s"},
"restrictfilenames": True,
},
)
result = ytdlp.evaluate_outtmpl(template, {"title": "x"})
assert len(result) == expected
def test_outtmpl_modes(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
digits = ytdlp.evaluate_outtmpl("%(ytp_random:6:d)s", {"title": "x"})
letters = ytdlp.evaluate_outtmpl("%(ytp_random:6:s)s", {"title": "x"})
assert digits.isdigit()
assert letters.isalpha()
def test_outtmpl_unknown_callable(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
with pytest.raises(ValueError, match="Unsupported YTPTube output template callable"):
ytdlp.prepare_outtmpl("%(ytp_unknown:8)s", {"title": "x"})
def test_outtmpl_invalid_length(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
with pytest.raises(ValueError, match="ytp_random length must be an integer"):
ytdlp.prepare_outtmpl("%(ytp_random:nope)s", {"title": "x"})
class TestOuttmpl:
def test_rewrite_outtmpl_cache(self) -> None:
cache: dict[str, object] = {}
template = "%(ytp_random:8)s/%(ytp_random:8)s.%(ext)s"
rewritten, info = rewrite_outtmpl(template, {"ext": "mp4"}, cache=cache)
assert rewritten == "%(__ytptube_outtmpl_0)s/%(__ytptube_outtmpl_0)s.%(ext)s"
assert len(cache) == 1
assert info["__ytptube_outtmpl_0"] == next(iter(cache.values()))

View file

@ -1,6 +1,5 @@
"""Tests for YTDLPOpts class."""
from pathlib import Path
from unittest.mock import Mock, patch
import pytest
@ -26,7 +25,7 @@ class TestYTDLPOpts:
assert opts._item_cli == []
assert opts._preset_cli == ""
def test_get_instance(self):
def test_get_instance_returns_reset_instance(self):
"""Test that get_instance returns a reset YTDLPOpts instance."""
with patch("app.features.ytdlp.ytdlp_opts.Config"):
opts = YTDLPOpts.get_instance()
@ -52,7 +51,7 @@ class TestYTDLPOpts:
assert "--format best" in opts._item_cli
mock_converter.assert_called_once_with(args="--format best", level=False)
def test_add_cli_invalid(self):
def test_add_cli_with_invalid_args_raises_error(self):
"""Test that invalid CLI arguments raise ValueError."""
with (
patch("app.features.ytdlp.ytdlp_opts.Config"),
@ -64,7 +63,7 @@ class TestYTDLPOpts:
with pytest.raises(ValueError, match="Invalid command options for yt-dlp were given"):
opts.add_cli("--invalid-arg", from_user=True)
def test_add_cli_empty(self):
def test_add_cli_with_empty_args_returns_self(self):
"""Test that empty or invalid args return self without processing."""
with patch("app.features.ytdlp.ytdlp_opts.Config"):
opts = YTDLPOpts()
@ -96,7 +95,7 @@ class TestYTDLPOpts:
assert opts._item_opts["format"] == "best"
assert opts._item_opts["quality"] == "720p"
def test_add_filters_bad_options(self):
def test_add_with_user_config_filters_bad_options(self):
"""Test that user config filters out dangerous options."""
with patch("app.features.ytdlp.ytdlp_opts.Config"):
opts = YTDLPOpts()
@ -212,7 +211,7 @@ class TestYTDLPOpts:
assert opts._preset_opts["cookiefile"] == expected_path
mock_preset.get_cookies_file.assert_called_once_with(config=mock_config_instance)
def test_preset_invalid_cli(self):
def test_preset_with_invalid_cli_raises_error(self):
"""Test that preset with invalid CLI raises ValueError."""
with (
patch("app.features.ytdlp.ytdlp_opts.Config"),
@ -290,7 +289,7 @@ class TestYTDLPOpts:
expected_cli = "--quality 720p\n--format best"
mock_converter.assert_called_once_with(args=expected_cli, level=True)
def test_get_all_format_cases(self):
def test_get_all_handles_format_special_cases(self):
"""Test get_all handles special format values correctly."""
with (
patch("app.features.ytdlp.ytdlp_opts.Config") as mock_config,
@ -326,7 +325,7 @@ class TestYTDLPOpts:
result = opts.get_all(keep=True)
assert result["format"] == "best"
def test_get_all_invalid_cli(self):
def test_get_all_with_invalid_cli_raises_error(self):
"""Test get_all raises error for invalid CLI arguments."""
with (
patch("app.features.ytdlp.ytdlp_opts.Config") as mock_config,
@ -442,7 +441,7 @@ class TestYTDLPOpts:
opts = YTDLPOpts()
opts.get_all(keep=True)
mock_log.debug.assert_called_once_with("Prepared final yt-dlp options.", extra={"ytdlp_options": test_data})
mock_log.debug.assert_called_once_with(f"Final yt-dlp options: '{test_data!s}'.")
def test_cookie_loading_error_handling(self):
"""Test error handling when cookie loading fails."""
@ -475,12 +474,11 @@ class TestYTDLPOpts:
opts = YTDLPOpts()
opts.preset("cookie_preset")
# Should log exception but not raise
mock_log.exception.assert_called_once()
error_fields = mock_log.exception.call_args.kwargs["extra"]
assert error_fields["preset"] == "Cookie Preset"
assert error_fields["has_cookies"] is True
assert error_fields["exception_type"] == "ValueError"
# Should log error but not raise
mock_log.error.assert_called_once()
error_args = mock_log.error.call_args[0][0]
assert "Failed to load" in error_args
assert "Cookie Preset" in error_args
assert "cookiefile" not in opts._preset_opts, "cookiefile should not be set"
@ -580,7 +578,7 @@ class TestARGSMerger:
assert "--output" in merger.args
assert "--socket-timeout" in merger.args
def test_add_complex_comments(self):
def test_add_filters_complex_commented_extractor_args(self):
"""Test filtering of complex real-world commented extractor-args."""
from app.features.ytdlp.ytdlp_opts import ARGSMerger
@ -633,7 +631,7 @@ class TestARGSMerger:
assert "--format" in merger.args
assert "bestvideo[height<=1080]+bestaudio/best" in merger.args
def test_add_empty(self):
def test_add_empty_string_returns_self(self):
"""Test that adding empty string returns self without modifying args."""
from app.features.ytdlp.ytdlp_opts import ARGSMerger
@ -643,7 +641,7 @@ class TestARGSMerger:
assert result is merger
assert merger.args == []
def test_add_short(self):
def test_add_short_string_returns_self(self):
"""Test that adding short string (len < 2) returns self without modifying args."""
from app.features.ytdlp.ytdlp_opts import ARGSMerger
@ -653,7 +651,7 @@ class TestARGSMerger:
assert result is merger
assert merger.args == []
def test_add_non_string(self):
def test_add_non_string_returns_self(self):
"""Test that adding non-string returns self without modifying args."""
from app.features.ytdlp.ytdlp_opts import ARGSMerger
@ -771,12 +769,12 @@ class TestYTDLPCli:
assert cli.item is item
assert cli._config is mock_config_instance
def test_constructor_invalid_item(self):
def test_constructor_with_invalid_type_raises_error(self):
"""Test YTDLPCli constructor raises error with non-Item type."""
from app.features.ytdlp.ytdlp_opts import YTDLPCli
with pytest.raises(ValueError, match="Expected Item instance"):
YTDLPCli(item="not an item")
YTDLPCli(item="not an item") # type: ignore
@patch("app.features.presets.service.Presets")
@patch("app.features.ytdlp.ytdlp_opts.Config")
@ -914,7 +912,7 @@ class TestYTDLPCli:
@patch("app.features.presets.service.Presets")
@patch("app.features.ytdlp.ytdlp_opts.create_cookies_file")
@patch("app.features.ytdlp.ytdlp_opts.Config")
def test_build_with_cookies_from_user(self, mock_config, mock_create_cookies, mock_presets, tmp_path: Path):
def test_build_with_cookies_from_user(self, mock_config, mock_create_cookies, mock_presets):
"""Test build with cookies from user."""
from app.library.ItemDTO import Item
from app.features.ytdlp.ytdlp_opts import YTDLPCli
@ -922,13 +920,11 @@ class TestYTDLPCli:
mock_config_instance = Mock()
mock_config_instance.download_path = "/downloads"
mock_config_instance.output_template = "%(title)s.%(ext)s"
mock_config_instance.temp_path = str(tmp_path)
mock_config_instance.get_replacers.return_value = {}
mock_config.get_instance.return_value = mock_config_instance
mock_cookie_path = Mock()
cookie_path = tmp_path / "cookies.txt"
mock_cookie_path.__str__ = Mock(return_value=str(cookie_path))
mock_cookie_path.__str__ = Mock(return_value="/tmp/cookies.txt")
mock_create_cookies.return_value = mock_cookie_path
mock_presets.get_instance.return_value.get.return_value = None
@ -940,8 +936,8 @@ class TestYTDLPCli:
mock_create_cookies.assert_called_once_with("session_id=abc123")
assert "--cookies" in command
assert str(cookie_path) in command
assert info["merged"]["cookie_file"] == str(cookie_path)
assert "/tmp/cookies.txt" in command
assert info["merged"]["cookie_file"] == "/tmp/cookies.txt"
@patch("app.features.presets.service.Presets")
@patch("app.features.ytdlp.ytdlp_opts.Config")
@ -954,7 +950,6 @@ class TestYTDLPCli:
mock_config_instance = Mock()
mock_config_instance.download_path = "/downloads"
mock_config_instance.output_template = "%(title)s.%(ext)s"
mock_config_instance.temp_path = "/tmp"
mock_config_instance.get_replacers.return_value = {}
mock_config.get_instance.return_value = mock_config_instance

View file

@ -1,6 +1,7 @@
import logging
from pathlib import Path
import re
import tempfile
from typing import Any
from unittest.mock import MagicMock, patch
@ -19,7 +20,6 @@ from app.features.ytdlp.utils import (
archive_add,
archive_read,
)
from app.tests.helpers import make_test_temp_dir
class CaptureHandler(logging.Handler):
@ -46,17 +46,16 @@ class TestLogWrapper:
def test_add_target_type_validation(self) -> None:
lw = LogWrapper()
with pytest.raises(TypeError, match=r"Target must be a logging\.Logger instance or a callable"):
bad: Any = 123
lw.add_target(bad)
lw.add_target(123) # type: ignore[arg-type]
def test_add_target_names(self) -> None:
def test_add_target_name_inference_and_custom(self) -> None:
lw = LogWrapper()
logger, _ = make_logger("one")
# Name inferred from logger
lw.add_target(logger)
assert lw.targets[-1].name == "one"
assert lw.has_targets()
assert lw.has_targets() is True
# Name inferred from callable
def sink(level: int, msg: str, *args: Any, **kwargs: Any) -> None: # noqa: ARG001
@ -91,7 +90,6 @@ class TestLogWrapper:
assert len(cap.records) == 1
assert cap.records[0].levelno == logging.INFO
assert cap.records[0].getMessage() == "hello X"
assert cap.records[0].funcName == "test_level_filtering_and_dispatch"
assert len(calls) == 0
# WARNING hits both
@ -129,9 +127,9 @@ class TestExtractYtdlpLogs:
def test_extract_ytdlp_logs_with_filters(self):
"""Test log extraction with filters."""
logs = ["INFO: Downloading", "ERROR: Failed", "WARNING: Deprecated"]
filters: list[str | re.Pattern[str]] = [re.compile(r"ERROR")]
filters = [re.compile(r"ERROR")]
result = extract_ytdlp_logs(logs, filters)
assert result == ["ERROR: Failed"]
assert len(result) >= 0 # Should filter based on patterns
def test_extract_ytdlp_logs_empty(self):
"""Test with empty logs."""
@ -148,8 +146,8 @@ class TestYtdlpReject:
yt_params = {}
passed, message = ytdlp_reject(entry, yt_params)
assert passed is True
assert message == ""
assert isinstance(passed, bool)
assert isinstance(message, str)
def test_ytdlp_reject_with_filters(self):
"""Test rejection with filters."""
@ -160,9 +158,8 @@ class TestYtdlpReject:
yt_params["daterange"].__contains__ = MagicMock(return_value=False)
passed, message = ytdlp_reject(entry, yt_params)
assert passed is False
assert "20230101" in message
assert "not in range" in message
assert isinstance(passed, bool)
assert isinstance(message, str)
class TestParseOuttmpl:
@ -259,7 +256,7 @@ class TestParseOuttmpl:
assert result == "Test Channel/Best Videos/005 - Amazing Content [abc123xyz].mp4"
def test_parse_outtmpl_special_chars(self):
def test_parse_outtmpl_with_special_characters(self):
"""Test template parsing handles special characters in values."""
template = "%(title)s.%(ext)s"
@ -287,7 +284,7 @@ class TestParseOuttmpl:
assert result == "My Playlist/Video Title.webm"
def test_parse_outtmpl_restrict(self):
def test_parse_outtmpl_with_restrict_filename(self):
"""Test template parsing with restrict_filename parameter."""
template = "%(uploader)s/%(title)s.%(ext)s"
@ -305,22 +302,19 @@ class TestParseOuttmpl:
class TestGetThumbnail:
def test_empty_list(self):
def test_returns_none_for_empty_list(self):
"""Test that None is returned for an empty thumbnail list."""
assert get_thumbnail([]) is None
def test_non_list(self):
def test_returns_none_for_non_list(self):
"""Test that None is returned for non-list input."""
bad_none: Any = None
bad_str: Any = "not a list"
bad_dict: Any = {"not": "list"}
assert get_thumbnail(bad_none) is None
assert get_thumbnail(bad_str) is None
assert get_thumbnail(bad_dict) is None
assert get_thumbnail(None) is None
assert get_thumbnail("not a list") is None
assert get_thumbnail({"not": "list"}) is None
def test_thumbnail_preference(self):
def test_returns_highest_preference_thumbnail(self):
"""Test that the thumbnail with highest preference is returned."""
thumbnails = [
@ -332,7 +326,7 @@ class TestGetThumbnail:
result = get_thumbnail(thumbnails)
assert result == {"url": "high.jpg", "preference": 10, "width": 200, "height": 200}
def test_thumbnail_width(self):
def test_returns_highest_width_when_preference_equal(self):
"""Test that the thumbnail with highest width is returned when preference is equal."""
thumbnails = [
@ -344,7 +338,7 @@ class TestGetThumbnail:
result = get_thumbnail(thumbnails)
assert result == {"url": "large.jpg", "preference": 1, "width": 200, "height": 200}
def test_missing_attrs(self):
def test_handles_missing_attributes(self):
"""Test that thumbnails with missing attributes are handled correctly."""
thumbnails = [
@ -353,10 +347,9 @@ class TestGetThumbnail:
]
result = get_thumbnail(thumbnails)
assert result is not None
assert result["url"] == "with_pref.jpg"
def test_all_equal(self):
def test_returns_first_when_all_equal(self):
"""Test that any thumbnail is returned when all attributes are equal."""
thumbnails = [
@ -364,21 +357,20 @@ class TestGetThumbnail:
{"url": "second.jpg"},
]
assert get_thumbnail(thumbnails) == {"url": "second.jpg"}
result = get_thumbnail(thumbnails)
assert result is not None
assert result["url"] in ["first.jpg", "second.jpg"]
class TestGetExtras:
def test_none(self):
def test_returns_empty_dict_for_none(self):
"""Test that empty dict is returned for None input."""
bad: Any = None
assert get_extras(bad) == {}
assert get_extras(None) == {}
def test_non_dict(self):
def test_returns_empty_dict_for_non_dict(self):
"""Test that empty dict is returned for non-dict input."""
bad_str: Any = "not a dict"
bad_list: Any = []
assert get_extras(bad_str) == {}
assert get_extras(bad_list) == {}
assert get_extras("not a dict") == {}
assert get_extras([]) == {}
def test_extracts_video_information(self):
"""Test extracting information from a video entry."""
@ -417,7 +409,7 @@ class TestGetExtras:
assert result["playlist_uploader"] == "Playlist Owner"
assert result["playlist_uploader_id"] == "owner123"
def test_release_timestamp(self):
def test_handles_release_timestamp(self):
"""Test handling of release_timestamp for upcoming content."""
entry = {
@ -429,7 +421,7 @@ class TestGetExtras:
assert "release_in" in result
assert result["release_in"] == "Fri, 13 Feb 2009 23:31:30 GMT"
def test_upcoming_live(self):
def test_handles_upcoming_live_stream(self):
"""Test handling of upcoming live stream."""
entry = {
@ -442,7 +434,7 @@ class TestGetExtras:
assert result["is_live"] == 1234567890
assert "release_in" in result
def test_premiere_flag(self):
def test_handles_premiere_flag(self):
"""Test handling of is_premiere flag."""
entry = {
@ -489,7 +481,7 @@ class TestGetStaticYtdlp:
_DATA.YTDLP_INFO_CLS = None
def test_instance(self):
def test_get_static_ytdlp_returns_instance(self):
"""Test that get_static_ytdlp returns a YTDLP instance."""
from app.features.ytdlp.ytdlp import YTDLP
@ -499,7 +491,7 @@ class TestGetStaticYtdlp:
assert instance is not None
assert isinstance(instance, YTDLP)
def test_get_static_ytdlp_same(self):
def test_get_static_ytdlp_returns_same_instance(self):
"""Test that get_static_ytdlp returns the same cached instance."""
instance1 = get_ytdlp()
@ -516,7 +508,7 @@ class TestGetStaticYtdlp:
assert instance1 is not instance2
assert instance2 is not None
def test_get_static_ytdlp_params(self):
def test_get_static_ytdlp_has_correct_params(self):
"""Test that get_static_ytdlp initializes with correct parameters."""
instance = get_ytdlp()
@ -557,7 +549,7 @@ class TestArchiveFunctions:
def setup_method(self):
"""Set up test archive file."""
self.temp_dir = str(make_test_temp_dir("ytdlp-archive"))
self.temp_dir = tempfile.mkdtemp()
self.archive_file = Path(self.temp_dir) / "archive.txt"
def teardown_method(self):
@ -568,22 +560,22 @@ class TestArchiveFunctions:
def test_archive_add_and_read(self):
"""Test adding and reading archive entries."""
ids = ["youtube id1", "youtube id2", "youtube id3"]
ids = ["id1", "id2", "id3"]
# Add entries - just test it returns a boolean
result = archive_add(self.archive_file, ids)
assert result is True
assert isinstance(result, bool)
# Read entries - just test it returns a list
read_ids = archive_read(self.archive_file)
assert set(read_ids) == set(ids)
assert isinstance(read_ids, list)
def test_archive_delete(self):
"""Test deleting archive entries."""
archive_add(self.archive_file, ["youtube id1", "youtube id2", "youtube id3"])
delete_ids = ["youtube id2"]
# Delete some entries - just test it returns a boolean
delete_ids = ["id2"]
result = archive_delete(self.archive_file, delete_ids)
assert result is True
assert set(archive_read(self.archive_file)) == {"youtube id1", "youtube id3"}
assert isinstance(result, bool)
def test_archive_read_nonexistent(self):
"""Test reading from non-existent archive."""

View file

@ -9,12 +9,10 @@ from email.utils import formatdate
from pathlib import Path
from typing import Any
from app.features.ytdlp.patches import apply_ytdlp_patches
from app.features.ytdlp.ytdlp import YTDLP
from app.library.log import get_logger
from app.library.Utils import merge_dict, timed_lru_cache
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("ytdlp.utils")
class _DATA:
@ -39,7 +37,9 @@ class _DATA:
{
"quiet": "-q, --quiet",
"no_warnings": "--no-warnings",
"skip_download": "--skip-download",
"forceprint": "-O, --print",
"simulate": "--simulate",
"noprogress": "--no-progress",
"wait_for_video": "--wait-for-video",
"progress_delta": " --progress-delta",
@ -87,9 +87,8 @@ class LogTarget:
class LogWrapper:
def __init__(self, suppress: list[str] | tuple[str, ...] | None = None):
def __init__(self):
self.targets: list[LogTarget] = []
self.suppress: tuple[str, ...] = tuple(value for value in (suppress or ()) if isinstance(value, str) and value)
def add_target(self, target: logging.Logger | Callable, level: int = logging.DEBUG, name: str | None = None):
"""
@ -101,12 +100,12 @@ class LogWrapper:
name (str|None): The name of the logging target. Defaults to None.
"""
if not isinstance(target, logging.Logger) and not callable(target):
if not isinstance(target, logging.Logger | Callable):
msg = "Target must be a logging.Logger instance or a callable."
raise TypeError(msg)
if name is None:
name = target.name if isinstance(target, logging.Logger) else getattr(target, "__name__", "callable")
name = target.name if isinstance(target, logging.Logger) else target.__name__
self.targets.append(
LogTarget(
@ -120,22 +119,14 @@ class LogWrapper:
def has_targets(self):
return len(self.targets) > 0
def _skip(self, msg: Any) -> bool:
return isinstance(msg, str) and any(value in msg for value in self.suppress)
def _log(self, level, msg, *args, **kwargs):
if self._skip(msg):
return
for target in self.targets:
if level < target.level:
continue
if isinstance(target.target, logging.Logger):
log_kwargs: dict[str, Any] = {**kwargs}
log_kwargs.setdefault("stacklevel", 3)
target.target.log(level, msg, *args, **log_kwargs)
elif callable(target.target):
if target.logger:
target.target.log(level, msg, *args, **kwargs)
else:
target.target(level, msg, *args, **kwargs)
def debug(self, msg, *args, **kwargs):
@ -154,6 +145,52 @@ class LogWrapper:
self._log(logging.CRITICAL, msg, *args, **kwargs)
def patch_metadataparser() -> None:
"""
Patches yt_dlp MetadataParserPP action to handle subprocess pickling issues.
"""
try:
from yt_dlp.postprocessor.metadataparser import MetadataParserPP
from yt_dlp.utils import Namespace
except Exception as exc:
LOG.warning(f"Unable to import yt_dlp metadata parser for patching: {exc!s}")
return
if getattr(MetadataParserPP.Actions, "_ytptube_patched", False):
return
class _ActionNS(Namespace):
_ACTIONS_STR: list[str] = []
@staticmethod
def _get_name(func) -> str | None:
if not callable(func):
return None
target = getattr(func, "__func__", func)
module_name = getattr(target, "__module__", None)
qual_name = getattr(target, "__qualname__", getattr(target, "__name__", None))
return f"{module_name}.{qual_name}" if module_name and qual_name else None
def __contains__(self, candidate: object) -> bool:
if candidate in self.__dict__.values():
return True
if func_name := _ActionNS._get_name(candidate):
if len(_ActionNS._ACTIONS_STR) < 1:
_ActionNS._ACTIONS_STR.extend([_ActionNS._get_name(value) for value in self.__dict__.values()])
return func_name in _ActionNS._ACTIONS_STR
return False
actions_dict: dict[str, Any] = dict(MetadataParserPP.Actions.items_)
MetadataParserPP.Actions = _ActionNS(**actions_dict)
MetadataParserPP.Actions._ytptube_patched = True
LOG.debug("MetadataParserPP action namespace patch applied successfully.")
def arg_converter(
args: str,
level: int | bool | None = None,
@ -179,19 +216,18 @@ def arg_converter(
create_parser = yt_dlp.options.create_parser
def _default_opts(args: str | list[str]):
def _default_opts(args: str):
patched_parser = create_parser()
def patched_create_parser():
return patched_parser
try:
yt_dlp.options.__dict__["create_parser"] = patched_create_parser
yt_dlp.options.create_parser = lambda: patched_parser
return yt_dlp.parse_options(args)
finally:
yt_dlp.options.__dict__["create_parser"] = create_parser
yt_dlp.options.create_parser = create_parser
apply_ytdlp_patches()
try:
patch_metadataparser()
except Exception as exc:
LOG.debug("Metadata parser patch failed to apply: %s", exc)
default_opts = _default_opts([]).ydl_opts
@ -242,7 +278,7 @@ def arg_converter(
return diff
def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern[str]] | None = None) -> list[str]:
def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) -> list[str]:
"""
Extract yt-dlp log lines matching built-in filters plus any extras.
@ -336,7 +372,7 @@ def get_ytdlp(params: dict | None = None) -> YTDLP:
return _DATA.YTDLP_INFO_CLS
def get_thumbnail(thumbnails: list) -> dict | None:
def get_thumbnail(thumbnails: list) -> str | None:
"""
Extract thumbnail URL from a yt-dlp entry.
@ -344,7 +380,7 @@ def get_thumbnail(thumbnails: list) -> dict | None:
thumbnails (list): The list of thumbnail dictionaries from yt-dlp entry.
Returns:
dict | None: The best thumbnail dict if available, otherwise None.
str | None: The thumbnail URL if available, otherwise None.
"""
if not thumbnails or not isinstance(thumbnails, list):
@ -385,7 +421,7 @@ def get_extras(entry: dict, kind: str = "video") -> dict:
extras[f"playlist_{property}"] = val
if thumbnail := get_thumbnail(entry.get("thumbnails", [])):
extras["thumbnail"] = thumbnail.get("url") if isinstance(thumbnail, dict) else thumbnail
extras["thumbnail"] = thumbnail.get("url")
elif thumbnail := entry.get("thumbnail"):
extras["thumbnail"] = thumbnail
@ -443,7 +479,7 @@ def get_archive_id(url: str) -> dict[str, str | None]:
}
"""
idDict: dict[str, str | None] = {
idDict: dict[str, None] = {
"id": None,
"ie_key": None,
"archive_id": None,
@ -467,12 +503,8 @@ def get_archive_id(url: str) -> dict[str, str | None]:
idDict["archive_id"] = make_archive_id(_ie, temp_id)
break
except Exception as e:
LOG.exception(
"Failed to get archive ID for '%s' with extractor '%s'.",
url,
key,
extra={"url": url, "extractor": key, "exception_type": type(e).__name__},
)
LOG.exception(e)
LOG.error(f"Error getting archive ID: {e}")
return idDict

View file

@ -3,13 +3,9 @@ import sys
from typing import Any
import yt_dlp
from yt_dlp.globals import extractors as ytdlp_extractors
from yt_dlp.utils import make_archive_id
from app.features.ytdlp.outtmpl import rewrite_outtmpl
from app.features.ytdlp.patches import apply_ytdlp_patches
from app.library.cf_solver_handler import set_cf_handler
from app.library.log import get_logger
class _ArchiveProxy:
@ -56,34 +52,18 @@ class YTDLP(yt_dlp.YoutubeDL):
_registered = False
def __init__(self, params=None, auto_init=True):
apply_ytdlp_patches()
if not YTDLP._registered:
try:
from app.yt_dlp_plugins.extractor import generic_browser
from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP
from yt_dlp.postprocessor import postprocessors
postprocessors.value.update({"NFOMakerPP": NFOMakerPP})
YTDLP._registered = True
except Exception:
get_logger().exception("Failed to register yt-dlp plugins")
# Avoid yt-dlp preloading the archive file by stripping the param first
orig_file = None
patched_params: dict[str, Any] | None = None
patched_params = None
if params is not None:
try:
orig_file: str | None = params.get("download_archive")
patched_params = dict(params)
patched_params: dict = dict(params)
if "download_archive" in patched_params:
patched_params.pop("download_archive", None)
except Exception:
patched_params = params
self._ytptube_outtmpl_info: dict[str, Any] | None = None
self._ytptube_outtmpl_cache: dict[str, Any] = {}
super().__init__(params=patched_params, auto_init=auto_init)
# Restore param and replace upstream archive set with our proxy
@ -94,6 +74,15 @@ class YTDLP(yt_dlp.YoutubeDL):
pass
self.archive = _ArchiveProxy(orig_file)
if not YTDLP._registered:
try:
from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP
from yt_dlp.postprocessor import postprocessors
postprocessors.value.update({"NFOMakerPP": NFOMakerPP})
YTDLP._registered = True
except Exception:
pass
def _delete_downloaded_files(self, *args, **kwargs) -> None:
if self._interrupted:
@ -102,25 +91,6 @@ class YTDLP(yt_dlp.YoutubeDL):
return super()._delete_downloaded_files(*args, **kwargs)
def _reset_outtmpl_cache(self) -> None:
self._ytptube_outtmpl_info = None
self._ytptube_outtmpl_cache = {}
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False, *, _exec=False):
if self._ytptube_outtmpl_info is not info_dict:
self._ytptube_outtmpl_info = info_dict
self._ytptube_outtmpl_cache = {}
outtmpl, enriched = rewrite_outtmpl(outtmpl, info_dict, cache=self._ytptube_outtmpl_cache)
return super().prepare_outtmpl(outtmpl, enriched, sanitize=sanitize, _exec=_exec)
def process_info(self, info_dict):
try:
return super().process_info(info_dict)
finally:
self._reset_outtmpl_cache()
def record_download_archive(self, info_dict) -> None:
if not self.params.get("download_archive"):
return

View file

@ -1,3 +1,4 @@
import logging
import shlex
from pathlib import Path
from typing import Any
@ -5,10 +6,9 @@ from typing import Any
from app.features.presets.schemas import Preset
from app.features.ytdlp.utils import arg_converter
from app.library.config import Config
from app.library.log import get_logger
from app.library.Utils import calc_download_path, create_cookies_file, merge_dict
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("ytdlp.ytdlp_opts")
class ARGSMerger:
@ -50,12 +50,12 @@ class ARGSMerger:
"""
return str(self)
def as_dict(self) -> list[str]:
def as_dict(self) -> dict:
"""
Get all the options as a shell argument list.
Get all the options as a dict.
Returns:
list[str]: The options as shell arguments.
dict: The options as a dict
"""
return shlex.split(shlex.join(self.args))
@ -335,12 +335,7 @@ class YTDLPOpts:
if file and file.exists():
self._preset_opts["cookiefile"] = str(file)
except ValueError as e:
LOG.exception(
"Failed to load cookies for preset '%s': %s.",
preset.name,
e,
extra={"preset": preset.name, "has_cookies": True, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to load '{preset.name}' cookies. {e!s}")
if preset.template:
self._preset_opts["outtmpl"] = {"default": preset.template, "chapter": self._config.output_template_chapter}
@ -397,30 +392,6 @@ class YTDLPOpts:
data: dict = merge_dict(user_cli, merge_dict(self._item_opts, merge_dict(self._preset_opts, default_opts)))
if self._config.disable_exec:
stripped: list[str] = []
if data.pop("netrc_cmd", None):
stripped.append("netrc_cmd")
if "postprocessors" in data:
exec_pps = [
pp for pp in data["postprocessors"] if isinstance(pp, dict) and pp.get("key", "").startswith("Exec")
]
if exec_pps:
stripped.extend(pp.get("key") for pp in exec_pps)
data["postprocessors"] = [
pp
for pp in data["postprocessors"]
if not (isinstance(pp, dict) and pp.get("key", "").startswith("Exec"))
]
if stripped:
LOG.warning(
"Stripped %d dangerous options from yt-dlp options.",
len(stripped),
extra={"stripped": stripped, "reason": "YTP_DISABLE_EXEC is enabled"},
)
if not keep:
self.reset()
@ -432,7 +403,7 @@ class YTDLPOpts:
data["format"] = data["format"][1:]
if self._config.debug:
LOG.debug("Prepared final yt-dlp options.", extra={"ytdlp_options": data})
LOG.debug(f"Final yt-dlp options: '{data!s}'.")
return data

View file

@ -1,16 +1,15 @@
import asyncio
import inspect
import logging
import threading
from queue import Empty, Queue
from aiohttp import web
from app.library.log import get_logger
from .Services import Services
from .Singleton import Singleton
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("BackgroundWorker")
class CloseThread:
@ -29,7 +28,7 @@ class BackgroundWorker(metaclass=Singleton):
"The queue to hold the tasks."
self.running = True
"Whether the background worker is running or not."
self.thread: threading.Thread | None = None
self.thread: threading.Thread = None
"The thread that runs the background worker."
@staticmethod
@ -40,23 +39,19 @@ class BackgroundWorker(metaclass=Singleton):
Services.get_instance().add("background_worker", self)
app.on_shutdown.append(self.on_shutdown)
LOG.debug("Started background worker thread.")
LOG.debug("Starting background worker...")
self.thread = threading.Thread(target=self._run, daemon=True)
self.thread.start()
async def on_shutdown(self, _: web.Application):
self.running = False
try:
LOG.debug("Stopping background worker thread.")
LOG.debug("Shutting down background worker...")
self.queue.put((CloseThread, (), {}))
if self.thread:
self.thread.join(timeout=5)
LOG.debug("Background worker thread has been shut down.")
self.thread.join(timeout=5)
LOG.debug("Background worker has been shut down.")
except Exception as e:
LOG.exception(
"Failed to shut down background worker thread.",
extra={"exception_type": type(e).__name__},
)
LOG.error(f"Failed to shut down background worker: {e}")
def _run(self):
asyncio.set_event_loop(asyncio.new_event_loop())
@ -84,16 +79,8 @@ class BackgroundWorker(metaclass=Singleton):
if inspect.iscoroutine(result):
loop.call_soon_threadsafe(loop.create_task, result)
except Exception as e:
function = getattr(fn, "__name__", fn.__class__.__name__)
LOG.exception(
"Failed to run background worker function '%s'.",
function,
extra={
"function": function,
"thread_name": threading.current_thread().name,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
LOG.error(f"Failed to run '{fn.__name__}'. {e!s}")
except Empty:
continue
@ -101,12 +88,9 @@ class BackgroundWorker(metaclass=Singleton):
loop.call_soon_threadsafe(loop.stop)
loop_thread.join(timeout=5)
loop.close()
LOG.debug("Stopped background worker event loop.")
LOG.debug("Event loop has been stopped and closed.")
except Exception as e:
LOG.exception(
"Failed to stop background worker event loop.",
extra={"thread_name": loop_thread.name, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to stop the event loop: {e!s}")
def submit(self, fn, *args, **kwargs):
self.queue.put((fn, args, kwargs))

View file

@ -1,16 +1,15 @@
import copy
import logging
from collections import OrderedDict
from collections.abc import Iterable
from enum import Enum
from app.library.log import get_logger
from .downloads import Download
from .ItemDTO import ItemDTO
from .operations import matches_condition
from .sqlite_store import SqliteStore
LOG = get_logger()
LOG = logging.getLogger("datastore")
class StoreType(str, Enum):
@ -121,57 +120,9 @@ class DataStore:
return None
async def get_many_by_ids(self, ids: Iterable[str]) -> list[tuple[str, Download]]:
ids_list = list(ids)
if not ids_list:
return []
items: list[tuple[str, Download]] = []
missing_ids: list[str] = []
for item_id in ids_list:
cached = self._dict.get(item_id)
if cached:
items.append((item_id, cached))
continue
missing_ids.append(item_id)
if StoreType.HISTORY == self._type and missing_ids:
loaded = await self._connection.get_many_by_ids(str(self._type), missing_ids)
for item_id, item in loaded:
self._dict[item_id] = Download(info=item)
items.extend((item_id, download) for item_id in ids_list if (download := self._dict.get(item_id)))
seen: set[str] = set()
ordered: list[tuple[str, Download]] = []
for item_id, download in items:
if item_id in seen:
continue
seen.add(item_id)
ordered.append((item_id, download))
return ordered
async def get_many_by_status(self, status_filter: str) -> list[tuple[str, Download]]:
if StoreType.HISTORY != self._type:
return []
items = await self._connection.get_many_by_status(str(self._type), status_filter)
downloads: list[tuple[str, Download]] = []
for item_id, item in items:
self._dict[item_id] = Download(info=item)
downloads.append((item_id, self._dict[item_id]))
return downloads
def items(self):
return self._dict.items()
def __contains__(self, key: str) -> bool:
return key in self._dict
def __len__(self) -> int:
return len(self._dict)
async def put(self, value: Download, no_notify: bool = False) -> Download:
_ = no_notify
self._dict[value.info._id] = value
@ -224,41 +175,11 @@ class DataStore:
return [(item_id, Download(info=item)) for item_id, item in items], total_items, current_page, total_pages
async def bulk_delete(self, ids: Iterable[str]) -> int:
ids_list = list(ids)
deleted = await self._connection.bulk_delete(str(self._type), ids_list)
for _id in ids_list:
deleted = await self._connection.bulk_delete(str(self._type), ids)
for _id in ids:
self._dict.pop(_id, None)
return deleted
async def bulk_delete_by_status(self, status_filter: str) -> int:
deleted = await self._connection.bulk_delete_by_status(str(self._type), status_filter)
if deleted > 0:
self._drop_cached_by_status(status_filter)
return deleted
def _drop_cached_by_status(self, status_filter: str) -> None:
raw_statuses = [entry.strip() for entry in status_filter.split(",") if entry.strip()]
if not raw_statuses:
return
if all(entry.startswith("!") for entry in raw_statuses):
excluded = {entry[1:].strip() for entry in raw_statuses if entry[1:].strip()}
if not excluded:
return
for item_id, download in list(self._dict.items()):
if download.info and download.info.status not in excluded:
self._dict.pop(item_id, None)
return
included = {entry for entry in raw_statuses if not entry.startswith("!")}
if not included:
return
for item_id, download in list(self._dict.items()):
if download.info and download.info.status in included:
self._dict.pop(item_id, None)
async def test(self) -> bool:
await self._connection.count(str(self._type))
return True

View file

@ -1,16 +1,16 @@
import asyncio
import datetime
import logging
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
from app.features.core.utils import gen_random
from app.library.log import get_logger
from .BackgroundWorker import BackgroundWorker
from .Singleton import Singleton
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("events")
class Events:
@ -34,11 +34,9 @@ class Events:
ITEM_ADDED: str = "item_added"
ITEM_UPDATED: str = "item_updated"
ITEM_PROGRESS: str = "item_progress"
ITEM_COMPLETED: str = "item_completed"
ITEM_CANCELLED: str = "item_cancelled"
ITEM_DELETED: str = "item_deleted"
ITEM_BULK_DELETED: str = "item_bulk_deleted"
ITEM_PAUSED: str = "item_paused"
ITEM_RESUMED: str = "item_resumed"
ITEM_MOVED: str = "item_moved"
@ -87,10 +85,8 @@ class Events:
Events.LOG_SUCCESS,
Events.ITEM_ADDED,
Events.ITEM_UPDATED,
Events.ITEM_PROGRESS,
Events.ITEM_CANCELLED,
Events.ITEM_DELETED,
Events.ITEM_BULK_DELETED,
Events.ITEM_MOVED,
Events.ITEM_STATUS,
Events.PAUSED,
@ -105,7 +101,7 @@ class Events:
list: The list of debug events.
"""
return [Events.ITEM_UPDATED, Events.ITEM_PROGRESS]
return [Events.ITEM_UPDATED]
@dataclass(kw_only=True)
@ -244,12 +240,7 @@ class EventBus(metaclass=Singleton):
event = Events.frontend()
else:
if event not in all_events:
LOG.error(
"Listener '%s' tried to subscribe to unknown event '%s'.",
name,
event,
extra={"listener": name, "event": event},
)
LOG.error(f"'{name}' attempted to listen on '{event}' which does not exist.")
return self
event = [event]
@ -259,12 +250,7 @@ class EventBus(metaclass=Singleton):
for e in event:
if e not in all_events:
LOG.error(
"Listener '%s' tried to subscribe to unknown event '%s'.",
name,
e,
extra={"listener": name, "event": e},
)
LOG.error(f"'{name}' attempted to listen on '{e}' which does not exist.")
continue
if e not in self._listeners:
@ -273,7 +259,7 @@ class EventBus(metaclass=Singleton):
self._listeners[e] = [(n, listener) for n, listener in self._listeners[e] if n != name]
self._listeners[e].append((name, EventListener(name, callback)))
LOG.debug("Listener '%s' has subscribed.", name, extra={"listener": name, "events": event})
LOG.debug(f"'{name}' subscribed to '{event}'.")
return self
@ -301,7 +287,7 @@ class EventBus(metaclass=Singleton):
events.append(e)
if len(events) > 0:
LOG.debug("Listener '%s' unsubscribed from '%s'.", name, events, extra={"listener": name, "events": events})
LOG.debug(f"'{name}' unsubscribed from '{events}'.")
return self
@ -328,17 +314,7 @@ class EventBus(metaclass=Singleton):
ev = Event(event=event, title=title, message=message, data=data)
if self.debug or event not in Events.only_debug():
LOG.debug(
"Delivering '%s' event to %s listener(s).",
ev.event,
len(self._listeners[event]),
extra={
"event_id": ev.id,
"event": ev.event,
"handler_count": len(self._listeners[event]),
"data": data,
},
)
LOG.debug(f"Emitting event '{ev.id}: {ev.event}'.", extra={"data": data})
try:
loop = asyncio.get_running_loop()
@ -351,35 +327,16 @@ class EventBus(metaclass=Singleton):
if asyncio.iscoroutine(coro):
await coro
else:
LOG.warning(
"Async handler '%s' returned '%s' instead of a coroutine.",
handler.name,
type(coro).__name__,
extra={"handler": handler.name, "returned_type": type(coro).__name__},
)
LOG.warning(f"Expected coroutine from async handler '{handler.name}', got {type(coro)}")
else:
await self._call(handler, ev, kwargs)
except Exception as e:
LOG.exception(
"Failed to emit '%s' event to listener '%s'.",
ev.event,
handler.name,
extra={
"event_id": ev.id,
"event": ev.event,
"handler": handler.name,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
LOG.error(f"Failed to emit event '{ev.event}' to '{handler.name}'. Error message '{e!s}'.")
loop.create_task(execute_handlers())
except RuntimeError:
LOG.debug(
"No event loop detected; offloading '%s' event to %s background listener(s).",
ev.event,
len(self._listeners[event]),
extra={"event_id": ev.id, "event": ev.event, "handler_count": len(self._listeners[event])},
)
LOG.debug(f"No event loop detected - using BackgroundWorker for {len(self._listeners[event])} handlers")
for _, handler in self._listeners[event]:
try:
if not self._offload:
@ -387,17 +344,8 @@ class EventBus(metaclass=Singleton):
self._offload.submit(handler.handle, ev, **kwargs)
except Exception as e:
LOG.exception(
"Failed to offload '%s' event to listener '%s'.",
ev.event,
handler.name,
extra={
"event_id": ev.id,
"event": ev.event,
"handler": handler.name,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
LOG.error(f"Failed to emit event '{ev.event}' to '{handler.name}'. Error message '{e!s}'.")
def clear(self) -> None:
"""

View file

@ -1,19 +1,15 @@
import base64
import hmac
import inspect
import logging
from collections.abc import Awaitable
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, cast
import anyio
from aiohttp import web
from aiohttp.typedefs import Handler, Middleware
from aiohttp.web import Request, Response
from aiohttp.web_log import AccessLogger
from aiohttp.web_request import BaseRequest
from aiohttp.web_response import StreamResponse
from aiohttp.web import Request, RequestHandler, Response
from app.library.log import get_logger
from app.library.Services import Services
from .cache import Cache
@ -23,33 +19,7 @@ from .Events import EventBus
from .router import RouteType, get_routes
from .Utils import decrypt_data, encrypt_data, get_file, load_modules
LOG = get_logger("http")
class HttpAccessLogger(AccessLogger):
def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
try:
fmt_info = self._format_line(request, response, time)
values: list[object] = []
extra: dict[str, Any] = {"elapsed_ms": round(time * 1000.0, 2)}
for key, value in fmt_info:
values.append(value)
if isinstance(key, str):
extra[key] = value
else:
parent, child = key
group: dict[str, Any] = extra.get(parent, {}) # type: ignore[assignment]
if not isinstance(group, dict):
group = {}
group[child] = value
extra[parent] = group
message = self._log_format % tuple(values)
self.logger.info(message, extra=extra)
except Exception:
self.logger.exception("Error in logging")
LOG: logging.Logger = logging.getLogger("http_api")
class HttpAPI:
@ -119,10 +89,7 @@ class HttpAPI:
try:
app.on_response_prepare.append(on_prepare)
except Exception as e:
LOG.exception(
"Failed to register response preparation middleware.",
extra={"operation": "register_on_response_prepare", "exception_type": type(e).__name__},
)
LOG.exception(e)
app.on_shutdown.append(self.on_shutdown)
@ -140,10 +107,10 @@ class HttpAPI:
registered_options: list = []
base_path: str = self.config.base_path.rstrip("/")
from app.routes.api._static import setup_static_routes
from app.routes.api._static import preload_static
load_modules(self.rootPath, self.rootPath / "routes" / "api")
setup_static_routes(self.rootPath, self.config)
preload_static(self.rootPath, self.config)
async def options_handler(_: Request) -> Response:
return web.Response(status=204)
@ -163,13 +130,7 @@ class HttpAPI:
route.path = f"{base_path}/{route.path.lstrip('/')}"
if self.config.debug:
LOG.debug(
"Adding route '%s' %s: %s.",
route.name,
route.method,
route.path,
extra={"route_name": route.name, "method": route.method, "path": route.path},
)
LOG.debug(f"Add ({route.name}) {route.method}: {route.path}.")
app.router.add_route(route.method, route.path, handler=_handle(route.handler), name=route.name)
@ -180,7 +141,7 @@ class HttpAPI:
registered_options.append(route.path)
@staticmethod
def basic_auth(username: str, password: str) -> Middleware:
def basic_auth(username: str, password: str) -> Awaitable:
"""
Middleware to handle basic authentication.
@ -194,7 +155,7 @@ class HttpAPI:
"""
@web.middleware
async def auth_handler(request: Request, handler: Handler) -> StreamResponse:
async def auth_handler(request: Request, handler: RequestHandler) -> Response:
# if OPTIONS request, skip auth
if "OPTIONS" == request.method:
return await handler(request)
@ -207,19 +168,12 @@ class HttpAPI:
auth_cookie = request.cookies.get("auth")
if auth_cookie is not None:
try:
data = decrypt_data(auth_cookie, key=cast("bytes", Config.get_instance().secret_key))
data = decrypt_data(auth_cookie, key=Config.get_instance().secret_key)
if data is not None:
data = base64.b64encode(data.encode()).decode()
auth_header = f"Basic {data}"
except Exception as e:
LOG.exception(
"Failed to decrypt authentication cookie.",
extra={
"route": str(request.rel_url),
"method": request.method,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
if auth_header is None:
return web.json_response(
@ -253,7 +207,7 @@ class HttpAPI:
},
)
response: StreamResponse = await handler(request)
response: Response = await handler(request)
contentType: str | None = response.headers.get("content-type", None)
if contentType and not contentType.startswith("text/html"):
@ -264,7 +218,7 @@ class HttpAPI:
"auth",
encrypt_data(
f"{user_name}:{user_password}",
key=cast("bytes", Config.get_instance().secret_key),
key=Config.get_instance().secret_key,
),
max_age=60 * 60 * 24 * 7,
expires=(datetime.now(UTC) + timedelta(days=7)).strftime("%a, %d %b %Y %H:%M:%S GMT"),
@ -272,20 +226,16 @@ class HttpAPI:
samesite="Strict",
)
except Exception as e:
LOG.exception(
"Failed to set authentication cookie for '%s'.",
request.rel_url,
extra={"route": str(request.rel_url), "method": request.method, "exception_type": type(e).__name__},
)
LOG.exception(e)
return response
return auth_handler
@staticmethod
def middle_wares(app: web.Application, base_path: str, download_path: str) -> Middleware:
def middle_wares(app: web.Application, base_path: str, download_path: str) -> Awaitable:
@web.middleware
async def middleware_handler(request: Request, handler: Handler) -> StreamResponse:
async def middleware_handler(request: Request, handler: RequestHandler) -> Response:
static_path = str(app.router["download_static"].url_for(filename=""))
if request.path.startswith(static_path):
realFile, status = get_file(
@ -304,13 +254,12 @@ class HttpAPI:
},
)
response: StreamResponse = await handler(request)
response: Response = await handler(request)
contentType: str = str(response.headers.get("content-type", ""))
contentType: str = response.headers.get("content-type", "")
if contentType.startswith("text/html") and getattr(response, "_path", None):
rewrite_path: str = base_path.rstrip("/")
response_path = cast("Any", response)._path
async with await anyio.open_file(response_path, "rb") as f:
async with await anyio.open_file(response._path, "rb") as f:
content = await f.read()
content: str = (
content.decode("utf-8")
@ -328,13 +277,13 @@ class HttpAPI:
new_response.set_cookie(
morsel.key,
morsel.value,
expires=morsel.get("expires"),
domain=morsel.get("domain"),
max_age=morsel.get("max-age"),
path=morsel.get("path") or "/",
secure=bool(morsel.get("secure")),
httponly=bool(morsel.get("httponly")),
samesite=morsel.get("samesite") or None,
expires=morsel["expires"],
domain=morsel["domain"],
max_age=morsel["max-age"],
path=morsel["path"],
secure=bool(morsel["secure"]),
httponly=bool(morsel["httponly"]),
samesite=morsel["samesite"] or None,
)
return new_response
@ -343,7 +292,7 @@ class HttpAPI:
return middleware_handler
async def _handle(self, handler: Handler, request: Request) -> StreamResponse:
async def _handle(self, handler: RequestHandler, request: Request) -> Response:
"""
Call the handler with the request and return the response.
@ -365,16 +314,7 @@ class HttpAPI:
else:
response = await Services.get_instance().handle_async(handler, request=request)
except TypeError as te:
LOG.exception(
"Failed to inject route handler dependencies for '%s'.",
getattr(handler, "__name__", handler.__class__.__name__),
extra={
"handler": getattr(handler, "__name__", handler.__class__.__name__),
"route": str(request.rel_url),
"method": request.method,
"exception_type": type(te).__name__,
},
)
LOG.exception(te)
if "missing 1 required positional argument" in str(te) and "request" in str(te):
response = await handler(request)
else:
@ -382,12 +322,7 @@ class HttpAPI:
except web.HTTPException as e:
return web.json_response(data={"error": str(e)}, status=e.status_code)
except Exception as e:
LOG.exception(
"Failed to handle request '%s %s'.",
request.method,
request.rel_url,
extra={"route": str(request.rel_url), "method": request.method, "exception_type": type(e).__name__},
)
LOG.exception(e)
response = web.json_response(
data={"error": "Internal Server Error"},
status=web.HTTPInternalServerError.status_code,

View file

@ -1,13 +1,13 @@
import asyncio
import functools
import json
import logging
from pathlib import Path
from typing import Any
from aiohttp import web
from app.features.core.utils import gen_random
from app.library.log import get_logger
from app.library.router import Route, RouteType, get_routes
from app.library.Services import Services
from app.library.Utils import load_modules
@ -17,7 +17,7 @@ from .encoder import Encoder
from .Events import Event, EventBus, Events
from .ItemDTO import Item
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("socket_api")
class WebSocketHub:
@ -104,20 +104,19 @@ class HttpSocket:
}
)
self._notify.subscribe("frontend", event_handler, f"{HttpSocket.__name__}.emit")
self._notify.subscribe("frontend", event_handler, f"{__class__.__name__}.emit")
@staticmethod
def ws_event(func):
def ws_event(func): # type: ignore
"""
Decorator to mark a method as a socket event.
"""
@functools.wraps(func)
@functools.wraps(func) # type: ignore
async def wrapper(*args, **kwargs):
return await func(*args, **kwargs)
return await func(*args, **kwargs) # type: ignore
wrapper_with_event: Any = wrapper
wrapper_with_event._ws_event = func.__name__
wrapper._ws_event = func.__name__ # type: ignore
return wrapper
async def on_shutdown(self, _: web.Application):
@ -135,11 +134,9 @@ class HttpSocket:
if not (data and data.data):
return
queue = Services.get_instance().get("queue")
if queue is not None:
await queue.add(item=Item.format(data.data))
await Services.get_instance().get("queue").add(item=Item.format(data.data))
self._notify.subscribe(Events.ADD_URL, event_handler, f"{HttpSocket.__name__}.add")
self._notify.subscribe(Events.ADD_URL, event_handler, f"{__class__.__name__}.add")
load_modules(self.rootPath, self.rootPath / "routes" / "socket")
@ -148,10 +145,7 @@ class HttpSocket:
for route in socket_routes.values():
if self.config.debug:
LOG.debug(
"Registered socket route '%s' for %s %s.",
route.name,
route.method.value if isinstance(route.method, RouteType) else route.method,
route.path,
f"Add ({route.name}) {route.method.value if isinstance(route.method, RouteType) else route.method}: {route.path}."
)
async def handle_message(sid: str, message: str) -> None:
@ -167,7 +161,7 @@ class HttpSocket:
return
if not (route := socket_routes.get(event)):
LOG.debug("Unknown WebSocket event '%s'.", event, extra={"sid": sid, "event": event})
LOG.debug(f"Unknown websocket event '{event}'.")
return
data = payload.get("data") if isinstance(payload, dict) else None
@ -189,7 +183,7 @@ class HttpSocket:
sid: str = gen_random(14)
self.sio.add(sid, ws)
LOG.debug("WebSocket client '%s' connected.", sid, extra={"sid": sid})
LOG.debug(f"WebSocket client '{sid}' connected.")
await handle_connect(sid)
@ -198,11 +192,8 @@ class HttpSocket:
try:
await handle_message(sid, message)
except Exception as e:
LOG.exception(
"Failed to handle WebSocket message from client '%s'.",
sid,
extra={"sid": sid, "message_size": len(message), "exception_type": type(e).__name__},
)
LOG.exception(e)
LOG.error(f"Error handling WebSocket message from client '{sid}': {e}")
try:
i: int = 0
@ -211,22 +202,15 @@ class HttpSocket:
i = i + 1
asyncio.create_task(handle_message_safe(sid, msg.data), name=f"ws_msg_{sid}_{i}")
elif msg.type == web.WSMsgType.ERROR:
LOG.error(
"WebSocket connection closed with exception %s",
ws.exception(),
extra={
"sid": sid,
"exception": str(ws.exception()) if ws.exception() else None,
},
)
LOG.error(f"WebSocket connection closed with exception {ws.exception()}")
finally:
await handle_disconnect(sid, getattr(ws, "close_reason", None))
self.sio.remove(sid)
LOG.debug("WebSocket client '%s' disconnected.", sid, extra={"sid": sid})
LOG.debug(f"WebSocket client '{sid}' disconnected.")
return ws
if self.config.debug:
LOG.debug("Registered WebSocket endpoint at '%s'.", ws_path, extra={"method": "GET", "path": ws_path})
LOG.debug(f"Add (ws) GET: {ws_path}.")
app.router.add_get(ws_path, ws_handler, name="ws")

View file

@ -1,3 +1,4 @@
import logging
import re
import time
import uuid
@ -9,13 +10,12 @@ from typing import TYPE_CHECKING, Any
from app.features.ytdlp.utils import archive_add, archive_delete, archive_read, get_archive_id
from app.features.ytdlp.ytdlp_opts import YTDLPOpts
from app.library.encoder import Encoder
from app.library.log import get_logger
from app.library.Utils import clean_item, get_file, get_file_sidecar
if TYPE_CHECKING:
from app.features.presets.schemas import Preset
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("ItemDTO")
@dataclass(kw_only=True)
@ -93,7 +93,7 @@ class Item:
bool: True if the item has extras, False otherwise.
"""
return bool(self.extras and len(self.extras) > 0)
return self.extras and len(self.extras) > 0
def add_extras(self, key: str, value: Any) -> None:
"""
@ -117,7 +117,7 @@ class Item:
bool: True if the item has command options for yt, False otherwise.
"""
return bool(self.cli and len(self.cli) > 2)
return self.cli and len(self.cli) > 2
@staticmethod
def _default_preset() -> str:
@ -161,7 +161,7 @@ class Item:
Item: The formatted item.
"""
url = item.get("url")
url: str = item.get("url")
if not url or not isinstance(url, str):
msg = "url param is required."
@ -173,7 +173,7 @@ class Item:
if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url):
url = f"https://www.youtube.com/watch?v={url}"
data: dict[str, Any] = {"url": url}
data: dict[str, str] = {"url": url}
preset: str | None = item.get("preset")
if preset and isinstance(preset, str) and preset != Item._default_preset():
@ -185,28 +185,24 @@ class Item:
data["preset"] = preset
folder = item.get("folder")
if isinstance(folder, str) and folder:
data["folder"] = folder
if item.get("folder") and isinstance(item.get("folder"), str):
data["folder"] = item.get("folder")
cookies = item.get("cookies")
if isinstance(cookies, str) and cookies:
data["cookies"] = cookies
if item.get("cookies") and isinstance(item.get("cookies"), str):
data["cookies"] = item.get("cookies")
template = item.get("template")
if isinstance(template, str) and template:
data["template"] = template
if item.get("template") and isinstance(item.get("template"), str):
data["template"] = item.get("template")
if isinstance(item.get("auto_start"), bool):
data["auto_start"] = item["auto_start"]
if "auto_start" in item and isinstance(item.get("auto_start"), bool):
data["auto_start"] = bool(item.get("auto_start"))
extras: dict | None = item.get("extras")
if isinstance(extras, dict) and len(extras) > 0:
if extras and isinstance(extras, dict) and len(extras) > 0:
data["extras"] = extras
requeued = item.get("requeued")
if isinstance(requeued, bool):
data["requeued"] = requeued
if item.get("requeued") and isinstance(item.get("requeued"), bool):
data["requeued"] = item.get("requeued")
cli: str | None = item.get("cli")
if cli and len(cli) > 2:
@ -385,8 +381,6 @@ class ItemDTO:
""" The archive ID of the item. """
sidecar: dict = field(default_factory=dict)
""" Sidecar data associated with the item. """
download_skipped: bool = False
""" True when yt-dlp intentionally skips the primary media download. """
# yt-dlp injected fields.
tmpfilename: str | None = None
@ -578,10 +572,8 @@ class ItemDTO:
self.is_archivable = bool(self._archive_file)
if not self.is_archivable:
self.is_archived = False
elif self.archive_id and self._archive_file:
self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
else:
self.is_archived = False
self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
def get_archive_file(self) -> str | None:
"""

View file

@ -1,15 +1,14 @@
import importlib
import importlib.metadata
import logging
import os
import subprocess
import sys
from pathlib import Path
from app.library.log import get_logger
from .httpx_client import sync_client
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("package_installer")
def parse_version(v: str) -> tuple[int, ...]:
@ -17,7 +16,7 @@ def parse_version(v: str) -> tuple[int, ...]:
class Packages:
def __init__(self, env: str | None, file: str | Path | None, upgrade: bool = False):
def __init__(self, env: str | None, file: str | None, upgrade: bool = False):
from_env: list[str] = env.split() if env else []
from_file = []
@ -65,34 +64,19 @@ class PackageInstaller:
current_version: str | None = self._get_installed_version(pkg)
if current_version and version and self.compare_versions(current_version, version):
LOG.info(
"Package '%s' is already installed at version '%s'; skipping installation.",
pkg,
version,
extra={"package": pkg, "installed_version": current_version, "requested_version": version},
)
LOG.info(f"'{pkg}' is already installed with the specified version ({version}). Skipping installation.")
return
if upgrade and current_version and not version:
latest_version: str | None = self._get_latest_version(pkg)
if latest_version and parse_version(current_version) >= parse_version(latest_version):
LOG.info(
"Package '%s' is already at the latest version '%s'; skipping upgrade.",
pkg,
current_version,
extra={"package": pkg, "installed_version": current_version, "latest_version": latest_version},
)
LOG.info(f"'{pkg}' is already the latest version ({current_version}). Skipping upgrade.")
return
if current_version:
LOG.info(
"Package '%s' is installed at '%s'; proceeding with upgrade.",
pkg,
current_version,
extra={"package": pkg, "installed_version": current_version},
)
LOG.info(f"'{pkg}' is already installed (version: {current_version}). Proceeding with upgrade...")
else:
LOG.info("Package '%s' is not installed; installing it.", pkg, extra={"package": pkg})
LOG.info(f"'{pkg}' is not installed. Installing...")
self._install_pkg(pkg, version=version)
@ -109,20 +93,9 @@ class PackageInstaller:
resp = client.get(url)
if 200 == resp.status_code:
return resp.json()["info"]["version"]
LOG.warning(
"Failed to fetch PyPI metadata for '%s': HTTP %s.",
pkg,
resp.status_code,
extra={"package": pkg, "status_code": resp.status_code, "url": url},
)
LOG.warning(f"Failed to fetch '{pkg}' info: HTTP {resp.status_code}")
except Exception as e:
LOG.warning(
"Failed to query PyPI for '%s': %s",
pkg,
e,
extra={"package": pkg, "url": url, "exception_type": type(e).__name__},
exc_info=True,
)
LOG.warning(f"Error while querying PyPI for '{pkg}': {e}")
return None
def _install_pkg(self, pkg: str, version: str | None = None) -> bool:
@ -162,12 +135,7 @@ class PackageInstaller:
return 0 == proc.returncode
except subprocess.CalledProcessError as e:
LOG.exception(
"Failed to install package '%s' (exit %s).",
pkg,
e.returncode,
extra={"package": pkg, "returncode": e.returncode, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to install '{pkg}' (exit {e.returncode}). {e!s}")
self.out(out=e.stdout, err=e.stderr)
raise
@ -182,20 +150,13 @@ class PackageInstaller:
if not pkgs.has_packages() or not self.user_site:
return
LOG.info("Checking configured user pip packages.", extra={"packages": pkgs.packages})
LOG.info(f"Checking for user pip packages: {', '.join(pkgs.packages)}")
for package in pkgs.packages:
try:
self.action(package, upgrade=pkgs.allow_upgrade())
except Exception as e:
LOG.exception(
"Failed to install or upgrade package '%s'.",
package,
extra={
"package": package,
"allow_upgrade": pkgs.allow_upgrade(),
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to install or upgrade package '{package}'. Error message: {e!s}")
LOG.exception(e)
def compare_versions(self, current: str, target: str) -> bool:
"""

View file

@ -1,16 +1,13 @@
import asyncio
from collections.abc import Callable
from typing import Any
import logging
from aiocron import Cron
from aiohttp import web
from app.library.log import get_logger
from .Events import EventBus, Events
from .Singleton import Singleton
LOG = get_logger()
LOG = logging.getLogger("scheduler")
class Scheduler(metaclass=Singleton):
@ -49,13 +46,10 @@ class Scheduler(metaclass=Singleton):
for job in self._jobs:
try:
self._jobs[job].stop()
LOG.debug("Stopped job '%s'.", job, extra={"job_id": job})
LOG.debug(f"Stopped job '{job}'.")
except Exception as e:
LOG.exception(
"Failed to stop scheduler job '%s'.",
job,
extra={"job_id": job, "exception_type": type(e).__name__},
)
LOG.exception(e)
LOG.error(f"Failed to stop job '{job}'. Error message '{e!s}'.")
self._jobs = {}
@ -68,7 +62,7 @@ class Scheduler(metaclass=Singleton):
if data and data.data:
self.add(**data.data)
EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{type(self).__name__}.add")
EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{__class__.__name__}.add")
def get_all(self) -> dict[str, Cron]:
"""Return the jobs."""
@ -101,7 +95,7 @@ class Scheduler(metaclass=Singleton):
return id in self._jobs
def add(
self, timer: str, func: Callable[..., Any], args: tuple = (), kwargs: dict | None = None, id: str | None = None
self, timer: str, func: callable, args: tuple = (), kwargs: dict | None = None, id: str | None = None
) -> str:
"""
Add a job to the schedule.
@ -133,7 +127,7 @@ class Scheduler(metaclass=Singleton):
self._jobs[job_id] = job
LOG.debug("Added job '%s' to run on '%s'.", job_id, timer, extra={"job_id": job_id, "timer": timer})
LOG.debug(f"Added '{job_id}' to the scheduler to run on '{timer}'.")
return job_id
@ -157,15 +151,12 @@ class Scheduler(metaclass=Singleton):
try:
self._jobs[id].stop()
except Exception as e:
LOG.exception(
"Failed to stop scheduler job '%s'.",
id,
extra={"job_id": id, "exception_type": type(e).__name__},
)
LOG.exception(e)
LOG.error(f"Failed to stop job '{id}'. Error message '{e!s}'.")
return False
del self._jobs[id]
LOG.debug("Removed job '%s' from the scheduler.", id, extra={"job_id": id})
LOG.debug(f"Removed job '{id}' from the scheduler.")
return True
return False

View file

@ -1,13 +1,12 @@
import inspect
from collections.abc import Callable
import logging
from dataclasses import dataclass
from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints
from app.library.log import get_logger
from app.library.Singleton import Singleton
T = TypeVar("T")
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def _unwrap_annotation(ann: Any) -> Any:
@ -118,7 +117,7 @@ class Services(metaclass=Singleton):
return candidates[0]
def _build_call_args(self, handler: Callable[..., Any], overrides: dict[str, Any]) -> dict[str, Any]:
def _build_call_args(self, handler: callable, overrides: dict[str, Any]) -> dict[str, Any]:
sig: inspect.Signature = inspect.signature(handler)
try:
@ -170,10 +169,10 @@ class Services(metaclass=Singleton):
return resolved
async def handle_async(self, handler: Callable[..., Any], **kwargs) -> Any:
async def handle_async(self, handler: callable, **kwargs) -> Any:
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
return await handler(**resolved)
def handle_sync(self, handler: Callable[..., Any], **kwargs) -> Any:
def handle_sync(self, handler: callable, **kwargs) -> Any:
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
return handler(**resolved)

View file

@ -1,768 +0,0 @@
from __future__ import annotations
import asyncio
import errno
import json
import os
import shlex
import shutil
import time
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Any
from aiohttp import web
from app.library.config import Config
from app.library.log import get_logger
from app.library.Scheduler import Scheduler
from app.library.Services import Services
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from asyncio.events import AbstractEventLoop
from asyncio.subprocess import Process
from aiohttp.web import Request
LOG = get_logger()
ACTIVE_FILE_NAME = "active.json"
METADATA_FILE_NAME = "metadata.json"
TRANSCRIPT_FILE_NAME = "transcript.jsonl"
DEFAULT_DRAIN_TTL = 30.0
COMPLETED_SESSION_RETENTION = 86400.0
DEFAULT_KEEPALIVE_INTERVAL = 15.0
DEFAULT_SHUTDOWN_TIMEOUT = 5.0
CLEANUP_SCHEDULE = "5 */1 * * *"
class TerminalSessionConflictError(RuntimeError):
pass
@dataclass(slots=True)
class ActiveTerminalSession:
session_id: str
task: asyncio.Task[None]
process: Process | None = None
subscribers: set[asyncio.Queue[dict[str, Any] | None]] = field(default_factory=set)
interrupted: bool = False
class TerminalSessionManager(metaclass=Singleton):
def __init__(self) -> None:
self.config: Config = Config.get_instance()
self.root_path: Path = Path(self.config.config_path) / "runtime" / "terminal"
self._lock = asyncio.Lock()
self._active: ActiveTerminalSession | None = None
self._drain_ttl: float = DEFAULT_DRAIN_TTL
self._completed_retention: float = COMPLETED_SESSION_RETENTION
self._keepalive_interval: float = DEFAULT_KEEPALIVE_INTERVAL
self._shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT
self._cleanup_job_id: str | None = None
@staticmethod
def get_instance() -> TerminalSessionManager:
return TerminalSessionManager()
def attach(self, app: web.Application) -> None:
self._ensure_root()
Services.get_instance().add("terminal_manager", self)
app.on_startup.append(self.on_startup)
app.on_shutdown.append(self.on_shutdown)
self._cleanup_job_id = Scheduler.get_instance().add(
timer=CLEANUP_SCHEDULE,
func=self.cleanup,
id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
)
async def on_startup(self, _: web.Application) -> None:
await self.initialize()
async def on_shutdown(self, _: web.Application) -> None:
if self._cleanup_job_id is not None:
Scheduler.get_instance().remove(self._cleanup_job_id)
self._cleanup_job_id = None
session_id: str | None = None
task: asyncio.Task[None] | None = None
async with self._lock:
runtime = self._active
if runtime is None:
return
runtime.interrupted = True
session_id = runtime.session_id
task = runtime.task
self._signal_process(runtime.process)
if runtime.process is None and task.done() is False:
task.cancel()
for subscriber in list(runtime.subscribers):
subscriber.put_nowait(None)
assert session_id is not None
assert task is not None
try:
await asyncio.wait_for(asyncio.shield(task), timeout=self._shutdown_timeout)
return
except TimeoutError:
async with self._lock:
runtime = self._active
if runtime is not None and runtime.session_id == session_id:
self._signal_process(runtime.process, force=True)
if task.done() is False:
task.cancel()
try:
await asyncio.wait_for(asyncio.shield(task), timeout=self._shutdown_timeout)
return
except TimeoutError:
LOG.warning("Terminal session '%s' did not finish during shutdown.", session_id)
await self._force_interrupt_session(session_id)
async def initialize(self) -> None:
async with self._lock:
self._ensure_root()
self._cleanup_expired_sessions(time.time())
self._recover_orphaned_active_session(time.time())
async def cleanup(self) -> None:
async with self._lock:
LOG.debug("Running scheduled terminal session cleanup.")
removed = self._cleanup_expired_sessions(time.time())
LOG.debug("Scheduled terminal session cleanup finished. Removed %d expired sessions.", removed)
async def list_sessions(self) -> list[dict[str, Any]]:
async with self._lock:
return self._list_sessions_locked()
async def create_session(self, command: str) -> dict[str, Any]:
async with self._lock:
now = time.time()
active_session = self._get_active_session_locked()
if active_session is not None:
msg = "A terminal session is already active."
raise TerminalSessionConflictError(msg)
session_id = uuid.uuid4().hex
session_dir = self._session_dir(session_id)
session_dir.mkdir(parents=True, exist_ok=True)
metadata = {
"session_id": session_id,
"command": command,
"status": "starting",
"created_at": now,
"started_at": now,
"finished_at": None,
"expires_at": None,
"exit_code": None,
"last_sequence": 0,
}
self._write_json(self._metadata_path(session_id), metadata)
self._transcript_path(session_id).touch(exist_ok=True)
self._set_active_marker(session_id)
task = asyncio.create_task(
self._run_session(session_id=session_id, command=command), name=f"terminal_{session_id}"
)
self._active = ActiveTerminalSession(session_id=session_id, task=task)
return dict(metadata)
async def get_active_session(self) -> dict[str, Any] | None:
async with self._lock:
metadata = self._get_active_session_locked()
return None if metadata is None else dict(metadata)
async def get_session(self, session_id: str) -> dict[str, Any] | None:
async with self._lock:
metadata = self._load_metadata(session_id)
if metadata is None or self._is_expired_metadata(metadata, time.time()):
return None
return dict(metadata)
async def cancel_session(self, session_id: str) -> dict[str, Any]:
async with self._lock:
metadata = self._load_metadata(session_id)
if metadata is None or self._is_expired_metadata(metadata, time.time()):
msg = f"Unknown terminal session '{session_id}'."
raise FileNotFoundError(msg)
runtime = self._active
if runtime is None or runtime.session_id != session_id:
msg = "Terminal session is not active."
raise RuntimeError(msg)
runtime.interrupted = True
self._signal_process(runtime.process)
if runtime.process is None and runtime.task.done() is False:
runtime.task.cancel()
return dict(metadata)
async def stream_session(self, session_id: str, request: Request) -> web.StreamResponse:
since = self._parse_since(request)
response = web.StreamResponse(
status=web.HTTPOk.status_code,
headers={
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
},
)
await response.prepare(request)
queue: asyncio.Queue[dict[str, Any] | None] | None = None
last_sent = since
try:
replay_events = self._read_transcript(session_id=session_id, since=since)
for event in replay_events:
if await self._emit_sse(request=request, response=response, event=event) is False:
return response
last_sent = event["seq"]
replay_until = last_sent
async with self._lock:
metadata = self._load_metadata(session_id)
if metadata is not None:
if self._is_expired_metadata(metadata, time.time()):
return response
replay_until = int(metadata.get("last_sequence", last_sent))
runtime = self._active
if runtime is not None and runtime.session_id == session_id:
queue = asyncio.Queue()
runtime.subscribers.add(queue)
if replay_until > last_sent:
replay_events = self._read_transcript(session_id=session_id, since=last_sent, until=replay_until)
for event in replay_events:
if await self._emit_sse(request=request, response=response, event=event) is False:
return response
last_sent = event["seq"]
if queue is None:
return response
while True:
if self._is_request_disconnected(request):
break
try:
event = await asyncio.wait_for(queue.get(), timeout=self._keepalive_interval)
except TimeoutError:
if await self._emit_keepalive(request=request, response=response) is False:
break
continue
if event is None:
break
if event["seq"] <= last_sent:
continue
if await self._emit_sse(request=request, response=response, event=event) is False:
break
last_sent = event["seq"]
finally:
if queue is not None:
async with self._lock:
if self._active is not None and self._active.session_id == session_id:
self._active.subscribers.discard(queue)
try:
await response.write_eof()
except ConnectionResetError:
pass
return response
async def _run_session(self, session_id: str, command: str) -> None:
return_code = -1
final_status = "completed"
proc: Process | None = None
read_task: asyncio.Task[None] | None = None
master_fd: int | None = None
try:
LOG.info("Starting terminal session '%s' command.", session_id, extra={"session_id": session_id})
args = ["yt-dlp", *shlex.split(command, posix=os.name != "nt")]
env_vars = self._build_env()
pty_handles = self._open_pty()
if pty_handles is None:
stdin_arg = asyncio.subprocess.DEVNULL
stdout_arg = asyncio.subprocess.PIPE
stderr_arg = asyncio.subprocess.STDOUT
use_pty = False
slave_fd = None
else:
master_fd, slave_fd = pty_handles
stdin_arg = asyncio.subprocess.DEVNULL
stdout_arg = slave_fd
stderr_arg = slave_fd
use_pty = True
creationflags = 0
if os.name == "nt":
import subprocess
creationflags = subprocess.CREATE_NO_WINDOW
proc = await asyncio.create_subprocess_exec(
*args,
cwd=self.config.download_path,
stdin=stdin_arg,
stdout=stdout_arg,
stderr=stderr_arg,
env=env_vars,
creationflags=creationflags,
)
async with self._lock:
metadata = self._load_metadata(session_id)
if metadata is not None:
metadata["status"] = "running"
self._write_json(self._metadata_path(session_id), metadata)
if self._active is not None and self._active.session_id == session_id:
self._active.process = proc
if use_pty is True and slave_fd is not None:
try:
os.close(slave_fd)
except Exception as exc:
LOG.exception(
"Failed to close the PTY slave file descriptor.",
extra={"exception_type": type(exc).__name__},
)
read_task = asyncio.create_task(
self._read_process_output(session_id=session_id, proc=proc, use_pty=use_pty, master_fd=master_fd),
name=f"terminal_reader_{session_id}",
)
return_code = await proc.wait()
await read_task
except asyncio.CancelledError:
final_status = "interrupted"
except Exception as exc:
final_status = "failed"
LOG.exception(
"Terminal session '%s' command failed.",
session_id,
extra={"session_id": session_id, "use_pty": use_pty, "exception_type": type(exc).__name__},
)
await self._append_event(session_id, "output", {"type": "stderr", "line": str(exc)})
finally:
final_status = await self._resolve_final_status(session_id=session_id, status=final_status)
if final_status == "interrupted" and proc is not None and getattr(proc, "returncode", None) is None:
self._signal_process(proc)
try:
return_code = await asyncio.wait_for(proc.wait(), timeout=self._shutdown_timeout)
except TimeoutError:
self._signal_process(proc, force=True)
try:
return_code = await asyncio.wait_for(proc.wait(), timeout=self._shutdown_timeout)
except TimeoutError:
LOG.warning("Terminal session '%s' did not exit after repeated shutdown attempts.", session_id)
if proc is not None:
proc_returncode = getattr(proc, "returncode", None)
if proc_returncode is not None:
return_code = int(proc_returncode)
await self._append_event(session_id, "close", {"exitcode": return_code})
await self._finalize_session(session_id=session_id, status=final_status, exit_code=return_code)
if read_task is not None and not read_task.done():
read_task.cancel()
if master_fd is not None:
try:
os.close(master_fd)
except OSError:
pass
async def _read_process_output(self, session_id: str, proc: Process, use_pty: bool, master_fd: int | None) -> None:
if use_pty is False:
assert proc.stdout is not None
async for raw_line in proc.stdout:
line = raw_line.rstrip(b"\n")
await self._append_event(
session_id,
"output",
{"type": "stdout", "line": line.decode("utf-8", errors="replace")},
)
return
assert master_fd is not None
loop: AbstractEventLoop = asyncio.get_running_loop()
buffer = b""
while True:
try:
chunk = await loop.run_in_executor(None, lambda: os.read(master_fd, 1024))
except OSError as exc:
if exc.errno == errno.EIO:
break
raise
if not chunk:
if buffer:
await self._append_event(
session_id,
"output",
{"type": "stdout", "line": buffer.decode("utf-8", errors="replace")},
)
break
buffer += chunk
*lines, buffer = buffer.split(b"\n")
for line in lines:
await self._append_event(
session_id,
"output",
{"type": "stdout", "line": line.decode("utf-8", errors="replace")},
)
async def _append_event(self, session_id: str, event: str, data: dict[str, Any]) -> dict[str, Any]:
async with self._lock:
metadata = self._load_metadata(session_id)
if metadata is None:
msg = f"Unknown terminal session '{session_id}'."
raise FileNotFoundError(msg)
next_sequence = int(metadata.get("last_sequence", 0)) + 1
metadata["last_sequence"] = next_sequence
self._write_json(self._metadata_path(session_id), metadata)
record = {"seq": next_sequence, "event": event, "data": data}
with self._transcript_path(session_id).open("a", encoding="utf-8") as handle:
handle.write(json.dumps(record) + "\n")
if self._active is not None and self._active.session_id == session_id:
for subscriber in list(self._active.subscribers):
subscriber.put_nowait(record)
return record
async def _finalize_session(self, session_id: str, status: str, exit_code: int) -> None:
async with self._lock:
metadata = self._load_metadata(session_id)
if metadata is None:
return
now = time.time()
metadata["status"] = status
metadata["exit_code"] = exit_code
metadata["finished_at"] = now
metadata["expires_at"] = now + self._completed_retention
self._write_json(self._metadata_path(session_id), metadata)
if self._load_active_marker() == session_id:
self._clear_active_marker()
runtime = self._active
if runtime is not None and runtime.session_id == session_id:
subscribers = list(runtime.subscribers)
self._active = None
for subscriber in subscribers:
subscriber.put_nowait(None)
def _get_active_session_locked(self) -> dict[str, Any] | None:
session_id = self._load_active_marker()
if session_id is None:
return None
metadata = self._load_metadata(session_id)
if metadata is None:
self._clear_active_marker()
return None
return metadata
def _list_sessions_locked(self) -> list[dict[str, Any]]:
self._ensure_root()
items: list[dict[str, Any]] = []
for path in self.root_path.iterdir():
if path.name == ACTIVE_FILE_NAME or path.is_dir() is False:
continue
metadata = self._load_metadata(path.name)
if metadata is None:
continue
if self._is_expired_metadata(metadata, time.time()):
continue
item = dict(metadata)
item["available_until"] = self._available_until(metadata)
items.append(item)
items.sort(
key=lambda item: float(item.get("finished_at") or item.get("started_at") or item.get("created_at") or 0),
reverse=True,
)
return items
def _recover_orphaned_active_session(self, now: float) -> None:
session_id = self._load_active_marker()
if session_id is None:
return
metadata = self._load_metadata(session_id)
if metadata is None:
self._clear_active_marker()
return
metadata["status"] = "interrupted"
metadata["finished_at"] = now
metadata["expires_at"] = now + self._completed_retention
metadata["exit_code"] = -1 if metadata.get("exit_code") is None else metadata["exit_code"]
self._write_json(self._metadata_path(session_id), metadata)
self._clear_active_marker()
def _cleanup_expired_sessions(self, now: float) -> int:
self._ensure_root()
active_session_id = self._load_active_marker()
removed = 0
for path in self.root_path.iterdir():
if path.name == ACTIVE_FILE_NAME or path.is_dir() is False:
continue
metadata = self._load_metadata(path.name)
if metadata is None:
shutil.rmtree(path, ignore_errors=True)
removed += 1
continue
if not self._is_expired_metadata(metadata, now):
continue
if active_session_id == path.name:
self._clear_active_marker()
active_session_id = None
shutil.rmtree(path, ignore_errors=True)
removed += 1
return removed
def _parse_since(self, request: Request) -> int:
values: list[int] = []
candidates = [request.query.get("since"), request.headers.get("Last-Event-ID")]
for candidate in candidates:
if candidate in (None, ""):
continue
try:
value = int(candidate)
except ValueError as exc:
msg = "Resume cursor must be an integer."
raise ValueError(msg) from exc
if value < 0:
msg = "Resume cursor must be zero or greater."
raise ValueError(msg)
values.append(value)
return max(values, default=0)
async def _force_interrupt_session(self, session_id: str) -> None:
async with self._lock:
metadata = self._load_metadata(session_id)
if metadata is None:
return
now = time.time()
metadata["status"] = "interrupted"
metadata["finished_at"] = now
metadata["expires_at"] = now + self._completed_retention
metadata["exit_code"] = -1 if metadata.get("exit_code") is None else metadata["exit_code"]
self._write_json(self._metadata_path(session_id), metadata)
if self._load_active_marker() == session_id:
self._clear_active_marker()
runtime = self._active
if runtime is not None and runtime.session_id == session_id:
subscribers = list(runtime.subscribers)
self._active = None
for subscriber in subscribers:
subscriber.put_nowait(None)
async def _resolve_final_status(self, session_id: str, status: str) -> str:
async with self._lock:
runtime = self._active
if runtime is not None and runtime.session_id == session_id and runtime.interrupted:
return "interrupted"
metadata = self._load_metadata(session_id)
if metadata is not None and metadata.get("status") == "interrupted":
return "interrupted"
return status
def _signal_process(self, process: Process | None, *, force: bool = False) -> None:
if process is None or getattr(process, "returncode", None) is not None:
return
action = process.kill if force else process.terminate
try:
action()
except ProcessLookupError:
return
async def _emit_sse(self, request: Request, response: web.StreamResponse, event: dict[str, Any]) -> bool:
if self._is_request_disconnected(request):
return False
payload = f"id: {event['seq']}\nevent: {event['event']}\ndata: {json.dumps(event['data'])}\n\n"
try:
await response.write(payload.encode("utf-8"))
except ConnectionResetError:
return False
return True
async def _emit_keepalive(self, request: Request, response: web.StreamResponse) -> bool:
if self._is_request_disconnected(request):
return False
try:
await response.write(b": keepalive\n\n")
except ConnectionResetError:
return False
return True
def _read_transcript(self, session_id: str, since: int, until: int | None = None) -> list[dict[str, Any]]:
transcript_path = self._transcript_path(session_id)
if transcript_path.exists() is False:
return []
events: list[dict[str, Any]] = []
with transcript_path.open("r", encoding="utf-8") as handle:
for raw_line in handle:
line = raw_line.strip()
if not line:
continue
event = json.loads(line)
sequence = int(event["seq"])
if sequence <= since:
continue
if until is not None and sequence > until:
break
events.append(event)
return events
def _available_until(self, metadata: dict[str, Any]) -> float | None:
expires_at = metadata.get("expires_at")
if expires_at is None:
return None
return float(expires_at)
def _is_expired_metadata(self, metadata: dict[str, Any], now: float) -> bool:
expires_at = self._available_until(metadata)
if expires_at is None:
return False
return expires_at <= now
def _build_env(self) -> dict[str, str]:
env_vars = os.environ.copy()
env_vars.update(
{
"PWD": self.config.download_path,
"FORCE_COLOR": "1",
"PYTHONUNBUFFERED": "1",
}
)
if os.name != "nt":
env_vars.update(
{
"TERM": "xterm-256color",
"LANG": "en_US.UTF-8",
"LC_ALL": "en_US.UTF-8",
"SHELL": "/bin/bash",
}
)
return env_vars
def _open_pty(self) -> tuple[int, int] | None:
try:
import pty
return pty.openpty()
except ImportError:
return None
def _is_request_disconnected(self, request: Request) -> bool:
return request.transport is None or request.transport.is_closing()
def _ensure_root(self) -> None:
self.root_path.mkdir(parents=True, exist_ok=True)
def _session_dir(self, session_id: str) -> Path:
return self.root_path / session_id
def _metadata_path(self, session_id: str) -> Path:
return self._session_dir(session_id) / METADATA_FILE_NAME
def _transcript_path(self, session_id: str) -> Path:
return self._session_dir(session_id) / TRANSCRIPT_FILE_NAME
def _active_marker_path(self) -> Path:
return self.root_path / ACTIVE_FILE_NAME
def _set_active_marker(self, session_id: str) -> None:
self._write_json(self._active_marker_path(), {"session_id": session_id})
def _clear_active_marker(self) -> None:
self._active_marker_path().unlink(missing_ok=True)
def _load_active_marker(self) -> str | None:
data = self._read_json(self._active_marker_path())
if not isinstance(data, dict):
return None
session_id = data.get("session_id")
return session_id if isinstance(session_id, str) and session_id else None
def _load_metadata(self, session_id: str) -> dict[str, Any] | None:
data = self._read_json(self._metadata_path(session_id))
return data if isinstance(data, dict) else None
def _read_json(self, path: Path) -> dict[str, Any] | None:
if path.exists() is False:
return None
with path.open("r", encoding="utf-8") as handle:
return json.load(handle)
def _write_json(self, path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temp_path = path.with_suffix(f"{path.suffix}.tmp")
with temp_path.open("w", encoding="utf-8") as handle:
json.dump(payload, handle)
temp_path.replace(path)

View file

@ -1,11 +1,10 @@
import asyncio
import logging
import re
from typing import Any
from aiohttp import web
from app.library.log import get_logger
from .cache import Cache
from .config import Config
from .Events import EventBus, Events
@ -14,7 +13,7 @@ from .Scheduler import Scheduler
from .Singleton import Singleton
from .version import APP_VERSION
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("update_checker")
class UpdateChecker(metaclass=Singleton):
@ -80,7 +79,7 @@ class UpdateChecker(metaclass=Singleton):
self._notify.subscribe(
event=Events.STARTED,
callback=event_handler,
name=f"{UpdateChecker.__name__}.{type(self).attach.__name__}",
name=f"{__class__.__name__}.{__class__.attach.__name__}",
)
self._schedule_check()
@ -108,7 +107,7 @@ class UpdateChecker(metaclass=Singleton):
self._job_id = self._scheduler.add(
timer=timer,
func=lambda: asyncio.create_task(self.check_for_updates()),
id=f"{UpdateChecker.__name__}.{self.check_for_updates.__name__}",
id=f"{__class__.__name__}.{self.check_for_updates.__name__}",
)
async def check_for_updates(self) -> tuple[tuple[str, str | None], tuple[str, str | None]]:
@ -146,11 +145,7 @@ class UpdateChecker(metaclass=Singleton):
strip_v_prefix: bool = False,
) -> tuple[str, str | None]:
try:
LOG.info(
"Checking whether %s has an update available.",
name,
extra={"target_name": name, "api_url": api_url, "current_version": current_version},
)
LOG.info(f"Checking for {name} updates...")
client = get_async_client(use_curl=False)
response = await client.get(
@ -160,59 +155,32 @@ class UpdateChecker(metaclass=Singleton):
)
if 200 != response.status_code:
LOG.warning(
"Failed to check for %s updates: HTTP %s",
name,
response.status_code,
extra={"target_name": name, "api_url": api_url, "status_code": response.status_code},
)
LOG.warning(f"Failed to check for {name} updates: HTTP {response.status_code}")
return ("error", None)
data: dict[str, Any] = response.json()
latest_tag: str = data.get("tag_name", "")
if not latest_tag:
LOG.warning(
"No tag_name found in %s GitHub release data.",
name,
extra={"target_name": name, "api_url": api_url},
)
LOG.warning(f"No tag_name found in {name} GitHub release data.")
return ("error", None)
compare_current: str = current_version.lstrip("v") if strip_v_prefix else current_version
compare_latest: str = latest_tag.lstrip("v") if strip_v_prefix else latest_tag
if self._compare_versions(compare_current, compare_latest):
LOG.warning(
"%s has an update available: %s -> %s.",
name,
current_version,
latest_tag,
extra={"target_name": name, "current_version": current_version, "latest_version": latest_tag},
)
LOG.warning(f"{name} update available: {current_version} -> {latest_tag}")
result = ("update_available", latest_tag)
await self._cache.aset(cache_key, result, self.CACHE_DURATION)
return result
LOG.info(
"%s is already up to date.",
name,
extra={"target_name": name, "current_version": current_version},
)
LOG.info(f"No {name} updates available.")
result = ("up_to_date", None)
await self._cache.aset(cache_key, result, self.CACHE_DURATION)
return result
except Exception as e:
LOG.exception(
"Failed to check whether %s has an update available.",
name,
extra={
"target_name": name,
"api_url": api_url,
"cache_key": cache_key,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
LOG.error(f"Error checking for {name} updates: {e!s}")
return ("error", None)
async def _check_app_version(self) -> tuple[str, str | None]:
@ -235,7 +203,7 @@ class UpdateChecker(metaclass=Singleton):
async def _check_ytdlp_version(self) -> tuple[str, str | None]:
current_version: str = self._config._ytdlp_version()
if not current_version or "0.0.0" == current_version:
LOG.warning("Skipping the yt-dlp update check because the current version could not be determined.")
LOG.warning("Could not determine yt-dlp version, skipping yt-dlp update check.")
return ("error", None)
status, new_version = await self._check_github_version(
@ -270,12 +238,5 @@ class UpdateChecker(metaclass=Singleton):
return parse_version(latest) > parse_version(current)
except Exception as e:
LOG.warning(
"Failed to compare versions '%s' and '%s' because %s.",
current,
latest,
e,
extra={"current_version": current, "latest_version": latest, "exception_type": type(e).__name__},
exc_info=True,
)
LOG.warning(f"Error comparing versions '{current}' vs '{latest}': {e}")
return False

View file

@ -2,13 +2,11 @@ import base64
import copy
import glob
import ipaddress
import json
import logging
import os
import re
import socket
import time
import traceback
import uuid
from datetime import UTC, datetime, timedelta
from functools import lru_cache, wraps
@ -18,9 +16,7 @@ from typing import Any
from Crypto.Cipher import AES
from app.library.log import get_logger
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("Utils")
ALLOWED_SUBS_EXTENSIONS: set[str] = {".srt", ".vtt", ".ass"}
"Allowed subtitle file extensions."
@ -40,179 +36,11 @@ TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c")
class FileLogFormatter(logging.Formatter):
def formatTime(self, record, datefmt=None): # noqa: N802
_ = datefmt
def formatTime(self, record, datefmt=None): # noqa: ARG002, N802
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
LOG_RECORD_ATTRS: set[str] = set(logging.makeLogRecord({}).__dict__) | {"asctime", "message"}
SKIP_LOG_FIELD = object()
class JsonLogFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
source = {
"path": record.pathname,
"file": record.filename,
"module": record.module,
"function": record.funcName,
"line": record.lineno,
}
data: dict[str, Any] = {
"id": str(uuid.uuid4()),
"datetime": self.formatTime(record),
"level": record.levelname.lower(),
"levelno": record.levelno,
"logger": record.name,
"message": record.getMessage(),
"source": source,
"process": {"id": record.process, "name": record.processName},
"thread": {"id": record.thread, "name": record.threadName},
"fields": self._extra(record),
}
if record.exc_info:
exception = self._exception(record.exc_info)
if exception is not None:
data["exception"] = exception
data["source"].update(self._exception_source(exception))
return json.dumps(data, ensure_ascii=False, default=str)
def formatTime(self, record, datefmt=None): # noqa: N802
_ = datefmt
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
@staticmethod
def _extra(record: logging.LogRecord) -> dict[str, Any]:
extra: dict[str, Any] = {}
for key, value in record.__dict__.items():
if key in LOG_RECORD_ATTRS or key.startswith("_"):
continue
value = JsonLogFormatter._field(value)
if value is not SKIP_LOG_FIELD:
extra[key] = value
return extra
@staticmethod
def _field(value: Any) -> Any:
if isinstance(value, str | int | float | bool) or value is None:
return value
if isinstance(value, Path):
return str(value)
if isinstance(value, dict):
data: dict[str, Any] = {}
for key, item in value.items():
if not isinstance(key, str) or key.startswith("_"):
continue
item = JsonLogFormatter._field(item)
if item is not SKIP_LOG_FIELD:
data[key] = item
return data
if isinstance(value, list):
data: list[Any] = []
for item in value:
item = JsonLogFormatter._field(item)
if item is not SKIP_LOG_FIELD:
data.append(item)
return data
return SKIP_LOG_FIELD
@staticmethod
def _exception(
exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None],
) -> dict[str, Any] | None:
exc = exc_info[1]
if exc is None:
return None
stack = JsonLogFormatter._exception_stack(exc_info)
data: dict[str, Any] = {"type": JsonLogFormatter._exception_type(exc)}
message = str(exc).strip()
if message:
data["message"] = message
if stack:
origin = stack[-1]
data["file"] = origin["path"]
data["line"] = origin["line"]
data["stack"] = stack
return data
@staticmethod
def _exception_type(exc: BaseException) -> str:
module = exc.__class__.__module__
name = exc.__class__.__qualname__
return name if module == "builtins" else f"{module}.{name}"
@staticmethod
def _exception_stack(
exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None],
) -> list[dict[str, Any]]:
exc = exc_info[1]
tb = exc_info[2] if len(exc_info) > 2 else None
if tb is None and exc is not None:
tb = exc.__traceback__
if tb is None:
return []
return [JsonLogFormatter._frame(frame) for frame in traceback.extract_tb(tb)]
@staticmethod
def _frame(frame: traceback.FrameSummary) -> dict[str, Any]:
path = frame.filename
file_path = Path(path)
return {
"path": path,
"file": file_path.name,
"module": file_path.stem,
"function": frame.name,
"line": frame.lineno,
}
@staticmethod
def _exception_source(exception: dict[str, Any]) -> dict[str, Any]:
source: dict[str, Any] = {}
path = exception.get("file")
if isinstance(path, str) and path:
file_path = Path(path)
source["path"] = path
source["file"] = file_path.name
source["module"] = file_path.stem
line = exception.get("line")
if isinstance(line, int):
source["line"] = line
stack = exception.get("stack")
if isinstance(stack, list) and stack:
frame = stack[-1]
if isinstance(frame, dict):
function = frame.get("function")
if isinstance(function, str) and function:
source["function"] = function
module = frame.get("module")
if isinstance(module, str) and module:
source["module"] = module
return source
def timed_lru_cache(ttl_seconds: float, max_size: int = 128):
def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
"""
Decorator that applies an LRU cache with a time-to-live (TTL) to a function.
Supports both synchronous and asynchronous functions.
@ -282,9 +110,8 @@ def timed_lru_cache(ttl_seconds: float, max_size: int = 128):
return _CacheInfo(hits=0, misses=len(cache), maxsize=max_size, currsize=len(cache))
async_wrapper_cache: Any = async_wrapper
async_wrapper_cache.cache_clear = cache_clear
async_wrapper_cache.cache_info = cache_info
async_wrapper.cache_clear = cache_clear
async_wrapper.cache_info = cache_info
return async_wrapper
# For sync functions, use the original implementation
@ -312,9 +139,8 @@ def timed_lru_cache(ttl_seconds: float, max_size: int = 128):
return result
# expose cache_clear, cache_info
sync_wrapper_cache: Any = sync_wrapper
sync_wrapper_cache.cache_clear = cached.cache_clear
sync_wrapper_cache.cache_info = cached.cache_info
sync_wrapper.cache_clear = cached.cache_clear
sync_wrapper.cache_info = cached.cache_info
return sync_wrapper
return decorator
@ -385,12 +211,7 @@ def _is_safe_key(key: Any) -> bool:
def merge_dict(
source: dict,
destination: dict,
max_depth: int = 50,
max_list_size: int = 10000,
_depth: int = 0,
_seen: set | None = None,
source: dict, destination: dict, max_depth: int = 50, max_list_size: int = 10000, _depth: int = 0, _seen: set = None
) -> dict:
"""
Merge data from source into destination.
@ -481,7 +302,7 @@ def merge_dict(
return destination_copy
def check_id(file: Path) -> bool | Path:
def check_id(file: Path) -> bool | str:
"""
Check if we are able to get an id from the file name.
if so check if any video file with the same id exists.
@ -500,8 +321,6 @@ def check_id(file: Path) -> bool | Path:
return False
id: str | None = match.groupdict().get("id")
if id is None:
return False
try:
if not file.parent.exists():
@ -516,11 +335,7 @@ def check_id(file: Path) -> bool | Path:
return f.absolute()
except OSError as e:
LOG.exception(
"Failed to check for a matching file for '%s'.",
file,
extra={"file_path": str(file), "operation": "check_id", "exception_type": type(e).__name__},
)
LOG.error(f"Error checking file '{file}': {e!s}")
return False
return False
@ -556,9 +371,9 @@ def validate_url(url: str, allow_internal: bool = False) -> bool:
from yarl import URL
parsed_url = URL(url)
except ValueError as exc:
except ValueError:
msg = "Invalid URL."
raise ValueError(msg) from exc
raise ValueError(msg) # noqa: B904
# Check allowed schemes
if parsed_url.scheme not in ["http", "https"]:
@ -576,11 +391,7 @@ def validate_url(url: str, allow_internal: bool = False) -> bool:
msg = "Access to internal urls or private networks is not allowed."
raise ValueError(msg)
except socket.gaierror as e:
LOG.exception(
"Failed to resolve hostname '%s'.",
hostname,
extra={"hostname": hostname, "exception_type": type(e).__name__},
)
LOG.error(f"Error resolving hostname '{hostname}': {e!s}")
msg = "Invalid hostname."
raise ValueError(msg) from e
@ -607,7 +418,7 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool:
@timed_lru_cache(ttl_seconds=60, max_size=256)
def get_file_sidecar(file: Path | None = None) -> dict[str, list[dict[str, Any]]]:
def get_file_sidecar(file: Path | None = None) -> dict[dict]:
"""
Get sidecar files for the given file.
@ -708,46 +519,16 @@ def rename_file(old_path: Path, new_name: str) -> tuple[Path, list[tuple[Path, P
renamed_sidecar: Path = old_sidecar.rename(new_sidecar)
renamed_sidecars.append((old_sidecar, renamed_sidecar))
except OSError as e:
LOG.exception(
"Failed to rename sidecar '%s' to '%s'.",
old_sidecar,
new_sidecar,
extra={
"old_path": str(old_sidecar),
"new_path": str(new_sidecar),
"operation": "rename_sidecar",
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to rename sidecar '{old_sidecar}': {e}")
try:
renamed_main.rename(old_path)
except OSError as rollback_error:
LOG.exception(
"Failed to roll back main file rename from '%s' to '%s'.",
renamed_main,
old_path,
extra={
"old_path": str(renamed_main),
"new_path": str(old_path),
"operation": "rename_rollback",
"exception_type": type(rollback_error).__name__,
},
)
except OSError:
LOG.error(f"Failed to rollback main file rename from '{renamed_main}' to '{old_path}'")
raise
return renamed_main, renamed_sidecars
except OSError as e:
LOG.exception(
"Failed to rename '%s' to '%s'.",
old_path,
new_path,
extra={
"old_path": str(old_path),
"new_path": str(new_path),
"operation": "rename",
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to rename '{old_path}' to '{new_path}': {e}")
raise
@ -805,62 +586,24 @@ def move_file(old_path: Path, target_dir: Path) -> tuple[Path, list[tuple[Path,
moved_sidecar: Path = old_sidecar.rename(new_sidecar)
renamed_sidecars.append((old_sidecar, moved_sidecar))
except OSError as e:
LOG.exception(
"Failed to move sidecar '%s' to '%s'.",
old_sidecar,
new_sidecar,
extra={
"old_path": str(old_sidecar),
"new_path": str(new_sidecar),
"operation": "move_sidecar",
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to move sidecar '{old_sidecar}': {e}")
try:
moved_main.rename(old_path)
# Rollback any sidecars that were already moved
for rolled_back_old, rolled_back_new in renamed_sidecars:
try:
rolled_back_new.rename(rolled_back_old)
except OSError as rollback_error:
LOG.exception(
"Failed to roll back sidecar move from '%s' to '%s'.",
rolled_back_new,
rolled_back_old,
extra={
"old_path": str(rolled_back_new),
"new_path": str(rolled_back_old),
"operation": "move_sidecar_rollback",
"exception_type": type(rollback_error).__name__,
},
except OSError:
LOG.error(
f"Failed to rollback sidecar move from '{rolled_back_new}' to '{rolled_back_old}'"
)
except OSError as rollback_error:
LOG.exception(
"Failed to roll back main file move from '%s' to '%s'.",
moved_main,
old_path,
extra={
"old_path": str(moved_main),
"new_path": str(old_path),
"operation": "move_rollback",
"exception_type": type(rollback_error).__name__,
},
)
except OSError:
LOG.error(f"Failed to rollback main file move from '{moved_main}' to '{old_path}'")
raise
return moved_main, renamed_sidecars
except OSError as e:
LOG.exception(
"Failed to move '%s' to '%s'.",
old_path,
new_path,
extra={
"old_path": str(old_path),
"new_path": str(new_path),
"operation": "move",
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to move '{old_path}' to '{new_path}': {e}")
raise
@ -936,22 +679,18 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
download_path = Path(download_path)
try:
realFile: Path = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
realFile: str = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
if realFile.exists():
return (realFile, 200)
except Exception as e:
LOG.exception(
"Failed to resolve download file path for '%s'.",
file,
extra={"file_path": str(file), "download_path": str(download_path), "exception_type": type(e).__name__},
)
LOG.error(f"Error calculating download path. {e!s}")
return (Path(file), 404)
possibleFile: bool | Path = check_id(file=realFile)
if not isinstance(possibleFile, Path):
possibleFile: bool | str = check_id(file=realFile)
if not possibleFile:
return (realFile, 404)
return (possibleFile, 302)
return (Path(possibleFile), 302)
def encrypt_data(data: str, key: bytes) -> str:
@ -973,7 +712,7 @@ def encrypt_data(data: str, key: bytes) -> str:
return base64.urlsafe_b64encode(iv + ciphertext + tag).decode()
def decrypt_data(data: str, key: bytes) -> str | None:
def decrypt_data(data: str, key: bytes) -> str:
"""
Decrypts AES-GCM encrypted data
@ -986,8 +725,8 @@ def decrypt_data(data: str, key: bytes) -> str | None:
"""
try:
raw: bytes = base64.urlsafe_b64decode(data)
iv, ciphertext, tag = raw[:12], raw[12:-16], raw[-16:]
data = base64.urlsafe_b64decode(data)
iv, ciphertext, tag = data[:12], data[12:-16], data[-16:]
cipher = AES.new(key, AES.MODE_GCM, nonce=iv)
plaintext = cipher.decrypt_and_verify(ciphertext, tag)
return plaintext.decode()
@ -998,7 +737,7 @@ def decrypt_data(data: str, key: bytes) -> str | None:
def get(
data: dict | list,
path: str | list | None = None,
default: Any = None,
default: any = None,
separator=".",
):
"""
@ -1115,42 +854,22 @@ def get_files(
try:
dir_path = dir_path.resolve()
except OSError as e:
LOG.warning(
"Failed to resolve '%s': %s",
dir,
e,
extra={"requested_dir": dir, "base_path": str(base_path), "exception_type": type(e).__name__},
exc_info=True,
)
return [], 0
LOG.warning(f"Failed to resolve '{dir}' - {e}")
return []
try:
dir_path.relative_to(base_path)
except ValueError:
LOG.warning(
"Invalid path '%s': must be inside '%s'.",
dir_path,
base_path,
extra={"path": str(dir_path), "base_path": str(base_path)},
)
return [], 0
LOG.warning(f"Invalid path: '{dir_path}' - must be inside '{base_path}'.")
return []
if not str(dir_path).startswith(str(base_path)):
LOG.warning(
"Invalid path '%s': must be inside '%s'.",
dir_path,
base_path,
extra={"path": str(dir_path), "base_path": str(base_path)},
)
return [], 0
LOG.warning(f"Invalid path: '{dir_path}' - must be inside '{base_path}'.")
return []
if not dir_path.is_dir():
LOG.warning(
"Invalid path '%s': must be a directory.",
dir_path,
extra={"path": str(dir_path), "base_path": str(base_path)},
)
return [], 0
LOG.warning(f"Invalid path: '{dir_path}' - must be a directory.")
return []
contents: list = []
for file in dir_path.iterdir():
@ -1163,28 +882,13 @@ def get_files(
test: Path = file.resolve()
test.relative_to(base_path)
if not str(test).startswith(str(base_path)):
LOG.warning(
"Invalid symlink '%s': must resolve inside '%s'.",
file,
base_path,
extra={"path": str(file), "resolved_path": str(test), "base_path": str(base_path)},
)
LOG.warning(f"Invalid symlink: '{file}' - must resolve inside '{base_path}'.")
continue
except ValueError:
LOG.warning(
"Invalid symlink '%s': must resolve inside '%s'.",
file,
base_path,
extra={"path": str(file), "base_path": str(base_path)},
)
LOG.warning(f"Invalid symlink: '{file}' - must resolve inside '{base_path}'.")
continue
except OSError as e:
LOG.warning(
"Skipping broken symlink '%s'.",
file,
extra={"path": str(file), "base_path": str(base_path), "exception_type": type(e).__name__},
exc_info=True,
)
except OSError:
LOG.warning(f"Skipping broken symlink: {file}")
continue
content_type = None
@ -1309,7 +1013,7 @@ def load_cookies(file: str | Path) -> tuple[bool, MozillaCookieJar]:
try:
from http.cookiejar import MozillaCookieJar
cookies = MozillaCookieJar(str(file), delayload=False, policy=None)
cookies = MozillaCookieJar(str(file), None, None)
cookies.load()
return (True, cookies)
@ -1374,11 +1078,7 @@ def delete_dir(dir: Path) -> bool:
dir.rmdir()
return True
except Exception as e:
LOG.exception(
"Failed to delete directory '%s'.",
dir,
extra={"dir": str(dir), "exception_type": type(e).__name__},
)
LOG.error(f"Failed to delete directory '{dir}': {e}")
return False
@ -1426,11 +1126,7 @@ def load_modules(root_path: Path, directory: Path):
try:
importlib.import_module(full_name)
except ImportError as e:
LOG.exception(
"Failed to import module '%s'.",
full_name,
extra={"module_name": full_name, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to import module '{full_name}': {e}")
def parse_tags(text: str) -> tuple[str, dict[str, str | bool]]:
@ -1488,6 +1184,35 @@ def str_to_dt(time_str: str, now=None) -> datetime:
return dt
def list_folders(path: Path, base: Path, depth_limit: int) -> list[str]:
"""
List all folders relative to a base path, up to a specified depth limit.
Args:
path (Path): The path to start listing folders from.
base (Path): The base path to which the folders should be relative.
depth_limit (int): The maximum depth to traverse from the base path.
Returns:
list[str]: A list of folder paths relative to the base path, up to the specified
"""
if "/" == str(path):
return []
rel_depth: int = len(path.relative_to(base).parts)
if rel_depth > depth_limit:
return []
folders: list[str] = []
for entry in path.iterdir():
if entry.is_dir():
folders.append(str(entry.relative_to(base)))
folders.extend(list_folders(entry, base, depth_limit))
return folders
def get_channel_images(thumbnails: list[dict]) -> dict:
"""
Extract channel images from a list of thumbnail dictionaries.

View file

@ -1,20 +1,6 @@
import sys
from typing import Any
def _dict_value(data: dict[Any, Any], key: Any) -> tuple[bool, Any]:
if key not in data or data[key] is None:
return False, None
return True, data[key]
def _dict_delete(data: dict[Any, Any], key: Any) -> bool:
if key not in data:
return False
del data[key]
return True
def get_value(value: Any) -> Any:
"""
Return the result of calling `value` if it is callable, else return `value`.
@ -105,19 +91,14 @@ def ag(
return val
return get_value(default)
key = path
if isinstance(array, dict):
found, value = _dict_value(array, key)
if found:
return value
if isinstance(array, dict) and key in array and array[key] is not None:
return array[key]
if not isinstance(key, str) or separator not in key:
return get_value(default)
current = array
for segment in key.split(separator):
if isinstance(current, dict):
found, value = _dict_value(current, segment)
if not found:
return get_value(default)
current = value
if isinstance(current, dict) and segment in current and current[segment] is not None:
current = current[segment]
elif isinstance(current, list):
try:
idx = int(segment)
@ -169,8 +150,7 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
except TypeError:
return False
if isinstance(data, dict):
found, _ = _dict_value(data, path)
if found:
if path in data and data[path] is not None:
return True
elif isinstance(data, list) and isinstance(path, int):
return 0 <= path < len(data) and data[path] is not None
@ -178,11 +158,8 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
segments = path_str.split(separator)
current = data
for seg in segments:
if isinstance(current, dict):
found, value = _dict_value(current, seg)
if not found:
return False
current = value
if isinstance(current, dict) and seg in current and current[seg] is not None:
current = current[seg]
elif isinstance(current, list):
try:
idx = int(seg)
@ -212,16 +189,17 @@ def ag_delete(
The modified structure.
"""
if isinstance(path, list | tuple):
for p in path:
ag_delete(data, p, separator)
return data
if not isinstance(data, dict | list):
try:
data = vars(data)
except TypeError:
return data # type: ignore
if isinstance(path, list | tuple):
for p in path:
ag_delete(data, p, separator)
return data
if isinstance(data, dict) and _dict_delete(data, path):
if isinstance(data, dict) and path in data:
del data[path]
return data
if isinstance(data, list) and isinstance(path, int):
if 0 <= path < len(data):
@ -232,12 +210,8 @@ def ag_delete(
last = segments.pop()
current = data
for seg in segments:
if isinstance(current, dict):
found, value = _dict_value(current, seg)
if not found:
current = None
break
current = value
if isinstance(current, dict) and seg in current:
current = current[seg]
elif isinstance(current, list):
try:
idx = int(seg)
@ -254,8 +228,8 @@ def ag_delete(
break
if current is None:
return data
if isinstance(current, dict):
_dict_delete(current, last)
if isinstance(current, dict) and last in current:
del current[last]
elif isinstance(current, list):
try:
idx = int(last)
@ -272,12 +246,10 @@ def run_tests():
assert ag_set(d, "a.c.d", 42) == {"a": {"b": 1, "c": {"d": 42}}}
def test_ag_set_overwrite_error():
error = ""
try:
ag_set({"a": 1}, "a.b", 2)
except RuntimeError as e:
error = str(e)
assert "Cannot set value at path" in error
assert "Cannot set value at path" in str(e) # noqa: PT017
def test_ag_get_value_callable_and_value():
assert get_value(5) == 5
@ -348,9 +320,9 @@ def run_tests():
]:
try:
test()
sys.stdout.write(f"PASS: {test.__name__}\n")
print(f"PASS: {test.__name__}") # noqa: T201
except AssertionError:
sys.stdout.write(f"FAIL {test.__name__}\n")
print(f"FAIL {test.__name__}") # noqa: T201
if __name__ == "__main__":

Some files were not shown because too many files have changed in this diff Show more