Compare commits

..

No commits in common. "master" and "v2.1.2" have entirely different histories.

257 changed files with 8102 additions and 22137 deletions

View file

@ -15,7 +15,7 @@ on:
env:
BUN_VERSION: latest
NODE_VERSION: 22
NODE_VERSION: 20
PYTHON_VERSION: "3.13"
jobs:

View file

@ -29,7 +29,7 @@ env:
DOCKERHUB_SLUG: arabcoders/ytptube
GHCR_SLUG: ghcr.io/arabcoders/ytptube
BUN_VERSION: latest
NODE_VERSION: 22
NODE_VERSION: 20
PYTHON_VERSION: "3.13"
jobs:

View file

@ -39,7 +39,7 @@ jobs:
env:
PYTHON_VERSION: "3.13"
BUN_VERSION: latest
NODE_VERSION: 22
NODE_VERSION: 20
TAG_NAME: ${{ github.event.inputs.tag || github.ref_name }}
steps:

396
API.md
View file

@ -24,8 +24,6 @@ This document describes the available endpoints and their usage. All endpoints r
- [DELETE /api/history](#delete-apihistory)
- [POST /api/history/{id}](#post-apihistoryid)
- [GET /api/history/{id}](#get-apihistoryid)
- [GET /api/history/{id}/thumbnail](#get-apihistoryidthumbnail)
- [POST /api/history/{id}/rename](#post-apihistoryidrename)
- [GET /api/history](#get-apihistory)
- [GET /api/history/live](#get-apihistorylive)
- [POST /api/history/start](#post-apihistorystart)
@ -57,8 +55,7 @@ This document describes the available endpoints and their usage. All endpoints r
- [GET /api/player/m3u8/{mode}/{file:.\*}.m3u8](#get-apiplayerm3u8modefilem3u8)
- [GET /api/player/segments/{segment}/{file:.\*}.ts](#get-apiplayersegmentssegmentfilets)
- [GET /api/player/subtitle/{file:.\*}.vtt](#get-apiplayersubtitlefilevtt)
- [GET /api/player/subtitles/manifest/{file:.\*}](#get-apiplayersubtitlesmanifestfile)
- [GET /api/player/subtitles/{source\_format}/{file:.\*}](#get-apiplayersubtitlessource_formatfile)
- [GET /api/thumbnail](#get-apithumbnail)
- [GET /api/file/ffprobe/{file:.\*}](#get-apifileffprobefile)
- [GET /api/file/info/{file:.\*}](#get-apifileinfofile)
- [GET /api/file/browser/{path:.\*}](#get-apifilebrowserpath)
@ -88,8 +85,6 @@ This document describes the available endpoints and their usage. All endpoints r
- [DELETE /api/conditions/{id}](#delete-apiconditionsid)
- [GET /api/logs](#get-apilogs)
- [GET /api/logs/stream](#get-apilogsstream)
- [GET /api/logs/level](#get-apilogslevel)
- [POST /api/logs/level/{level}](#post-apilogslevellevel)
- [GET /api/notifications/](#get-apinotifications)
- [GET /api/notifications/events/](#get-apinotificationsevents)
- [POST /api/notifications/](#post-apinotifications)
@ -101,11 +96,8 @@ This document describes the available endpoints and their usage. All endpoints r
- [POST /api/notifications/test](#post-apinotificationstest)
- [GET /api/yt-dlp/options](#get-apiyt-dlpoptions)
- [GET /api/system/configuration](#get-apisystemconfiguration)
- [GET /api/system/folders](#get-apisystemfolders)
- [GET /api/system/diagnostics](#get-apisystemdiagnostics)
- [GET /api/system/limits](#get-apisystemlimits)
- [POST /api/system/terminal](#post-apisystemterminal)
- [GET /api/system/terminal](#get-apisystemterminal)
- [GET /api/system/terminal/active](#get-apisystemterminalactive)
- [GET /api/system/terminal/{session\_id}](#get-apisystemterminalsession_id)
- [DELETE /api/system/terminal/{session\_id}](#delete-apisystemterminalsession_id)
@ -537,61 +529,6 @@ or an error:
---
### GET /api/history/{id}/thumbnail
**Purpose**: Return thumbnail for a downloaded item.
**Path Parameter**:
- `id` = item ID.
**Behavior**:
- Returns an existing local sidecar or artwork image when available.
- Otherwise generates a representative frame thumbnail with `ffmpeg` and caches it if enabled.
**Response**:
- `200 OK` with an image file response.
- `404 Not Found` if the item, downloaded file, or local thumbnail is not available.
- `400 Bad Request` if `id` is missing.
**Notes**:
- Audio-only files return `404` unless a local image sidecar already exists.
---
### POST /api/history/{id}/rename
**Purpose**: Rename a downloaded history file and its sidecars.
**Path Parameter**:
- `id` = Unique history item ID.
**Body**:
```json
{
"new_name": "renamed_video.mp4"
}
```
**Response**:
```json
{
"_id": "<uuid>",
"title": "Video Title",
"url": "https://youtube.com/watch?v=...",
....
}
```
**Error Responses**:
- `400 Bad Request` if `id` or `new_name` is missing, or the item has no downloaded file.
- `404 Not Found` if the item does not exist.
- `409 Conflict` if the rename destination already exists.
- `500 Internal Server Error` if the filesystem rename fails unexpectedly.
**Notes**:
- Uses the same file rename behavior as the file browser actions.
- Renames matching sidecar files together with the main media file.
---
### GET /api/history
**Purpose**: Returns the download queue and/or download history with optional pagination support.
@ -700,16 +637,10 @@ GET /api/history?type=queue&status=pending&order=ASC
This endpoint returns the current state of active downloads from memory.
**Query Parameters**:
- `limit` (optional): Override the configured queue display limit for this request. `0` means unlimited.
**Response**:
```json
{
"history_count": 0, // total number of completed items in history
"queue_count": 250, // total number of queued items
"queue_loaded": 100, // number of queued items included in this response
"queue_limit": 100, // effective display limit, 0 means unlimited
"queue":{
"id": "abc123",
"url": "https://example.com/video",
@ -1619,48 +1550,14 @@ Binary TS data (`Content-Type: video/mpegts`).
---
### GET /api/player/subtitles/manifest/{file:.*}
**Purpose**: Returns subtitle track metadata for a local media file.
### GET /api/thumbnail
**Purpose**: Proxy/fetch a remote thumbnail image.
**Path Parameter**:
- `file` = Relative path of the media file within the `download_path`.
**Query Parameter**:
- `?url=<remote-thumbnail-url>`
**Response**:
```json
{
"subtitles": [
{
"lang": "en",
"name": "VTT (0) - en",
"source_format": "vtt",
"delivery_format": "vtt",
"renderer": "native",
"url": "/api/player/subtitles/vtt/path/to/video.en.vtt"
},
{
"lang": "en",
"name": "ASS (1) - en",
"source_format": "ass",
"delivery_format": "ass",
"renderer": "assjs",
"url": "/api/player/subtitles/ass/path/to/video.en.ass"
}
]
}
```
---
### GET /api/player/subtitles/{source_format}/{file:.*}
**Purpose**: Delivers a subtitle file using its preferred playback format.
**Path Parameters**:
- `source_format` = `vtt`, `srt`, or `ass`.
- `file` = Relative path of the subtitle file.
**Response**:
- `text/vtt; charset=UTF-8` for `vtt` and `srt` sources.
- `text/x-ssa; charset=UTF-8` for `ass` sources.
**Response**:
Binary image data with the appropriate `Content-Type`.
---
@ -1696,16 +1593,21 @@ Binary TS data (`Content-Type: video/mpegts`).
},
"mimetype": "video/mp4",
"sidecar": {
"subtitle": [
"subtitles": [
{
"file": "filename.xxx.ass",
"file": "filename.xxx.vtt",
"lang": "xxx",
"name": "ASS (0) - xxx"
"name": "VTT 0 - XXX|end",
},
...
}
],
"video": [],
"audio": [],
"image": [],
"text": []
"text": [],
"metadata": [],
...
}
}
```
@ -2323,46 +2225,12 @@ Binary image data with appropriate headers
{
"logs": [
{
"id": "<uuid>",
"datetime": "2026-05-18T12:00:00.000+00:00",
"level": "error",
"levelno": 40,
"logger": "ytptube",
"message": "Failed to download 'Example Video'.",
"exception": {
"type": "ValueError",
"message": "bad",
"file": "/app/library/downloads/queue_manager.py",
"line": 123,
"stack": [
{
"path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py",
"module": "queue_manager",
"function": "start",
"line": 123
}
]
},
"source": {
"path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py",
"module": "queue_manager",
"function": "start",
"line": 123
},
"fields": {
"download": {
"download_id": "abc123",
"media_id": "video-id",
"title": "Example Video",
"url": "https://example.test/video",
"preset": "default",
"status": "error",
"has_cookies": false
}
}
}
"timestamp": "2023-01-01T12:00:00Z",
"level": "INFO",
"message": "...",
...
},
...
],
"offset": 0,
"limit": 100,
@ -2384,45 +2252,9 @@ Binary image data with appropriate headers
**Event Payload**:
```json
{
"id": "<uuid>",
"datetime": "2026-05-18T12:00:00.000+00:00",
"level": "error",
"levelno": 40,
"logger": "ytptube",
"message": "Failed to download 'Example Video'.",
"exception": {
"type": "ValueError",
"message": "bad",
"file": "/app/library/downloads/queue_manager.py",
"line": 123,
"stack": [
{
"path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py",
"module": "queue_manager",
"function": "start",
"line": 123
}
]
},
"source": {
"path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py",
"module": "queue_manager",
"function": "start",
"line": 123
},
"fields": {
"download": {
"download_id": "abc123",
"media_id": "video-id",
"title": "Example Video",
"url": "https://example.test/video",
"preset": "default",
"status": "error",
"has_cookies": false
}
}
"id": "<sha256>",
"line": "<log line>",
"datetime": "2024-01-01T12:00:00.000000+00:00"
}
```
@ -2430,31 +2262,6 @@ Binary image data with appropriate headers
---
### GET /api/logs/level
**Purpose**: Read the active runtime log level.
**Response**:
```json
{
"conf": "info",
"active": "info",
"levels": ["debug", "info", "warning", "error"]
}
```
---
### POST /api/logs/level/{level}
**Purpose**: Change the active runtime log level.
**Path Parameter**:
- `level`: One of `debug`, `info`, `warning`, `error`.
**Response**:
- `204 No Content` on success.
---
### GET /api/notifications/
**Purpose**: Retrieve notification targets with pagination.
@ -2635,86 +2442,55 @@ or an error:
---
### GET /api/system/configuration
**Purpose**: Retrieve system configuration.
**Purpose**: Retrieve comprehensive system configuration including app settings, presets, download fields, queue status, and folder structure.
**Response**:
```json
{
"app": {...},
"presets": [...],
"dl_fields": [...],
"paused": false,
"history_count": 150
}
```
---
### GET /api/system/folders
**Purpose**: List child directories for a given relative path within the download directory.
**Query Parameters**:
- `path=<relative-path>` (optional, default: root) - Relative path within the download directory.
**Response**:
```json
{
"path": "videos",
"folders": ["archive", "shorts", "2024"]
}
```
**Notes**:
- Results are cached server-side for a short time.
- Non-existent paths return an empty folder list.
---
### GET /api/system/diagnostics
**Purpose**: View system information.
**Response**:
```json
{
"status": "error",
"generated_at": 1713000000,
"summary": {
"total": 10,
"pass": 5,
"fail": 2,
"warn": 1,
"skip": 2,
"required_failed": 2
"app": {
"version": "...",
"download_path": "/path/to/downloads",
"base_path": "/",
...
},
"runtime": {
"app_version": "1.0.0",
"app_branch": "main",
"app_commit_sha": "abcdef12",
"app_build_date": "20260526",
"started": 1712999900,
"uptime_seconds": 100,
"platform": "linux",
"platform_release": "6.8.0",
"platform_machine": "x86_64",
"python_version": "3.13.1",
"python_minimum": "3.13",
"is_native": false,
"console_enabled": false
},
"requirements": {
"python": {
"current": "3.13.1",
"required": "3.13",
"supported": true,
"note": ""
"presets": [
{
"id": 1,
"name": "default",
"description": "...",
...
}
},
"checks": []
],
"dl_fields": [
{
"id": 1,
"name": "Title",
"field": "title",
"kind": "text",
...
}
],
"paused": false,
"folders": [
{"name": "folder1", "path": "folder1"},
{"name": "folder2", "path": "folder2"}
],
"history_count": 150,
"queue": [
{
"id": "abc123",
"url": "https://example.com/video",
"status": "pending",
...
}
]
}
```
**Notes**:
- Unexpected collection errors are returned as an `error`.
- This endpoint combines multiple data sources into a single response for efficient initialization
- The `folders` array includes available download folders up to the configured depth limit
- The `queue` array contains active download items
---
@ -2726,6 +2502,7 @@ or an error:
{
"downloads": {
"paused": false,
"live_bypasses_limits": true,
"global": {
"limit": 20,
"active": 3,
@ -2802,36 +2579,6 @@ or an error:
---
### GET /api/system/terminal
**Purpose**: List recent terminal sessions.
**Response**:
```json
{
"items": [
{
"session_id": "3a8c5f7e2d3b4a8f9c0d1e2f3a4b5c6d",
"command": "--help",
"status": "completed",
"created_at": 1713000000.0,
"started_at": 1713000000.0,
"finished_at": 1713000004.0,
"expires_at": 1713086404.0,
"available_until": 1713086404.0,
"exit_code": 0,
"last_sequence": 15
}
]
}
```
**Notes**:
- Expired sessions are removed automatically later.
- `403 Forbidden` if console is disabled.
---
### GET /api/system/terminal/active
**Purpose**: Return the currently active terminal session metadata, or `null` if no session is active.
@ -2859,7 +2606,7 @@ or
---
### GET /api/system/terminal/{session_id}
**Purpose**: Return metadata for a specific terminal session.
**Purpose**: Return metadata for a specific terminal session while it is active or still within the replay/drain window.
**Response**:
```json
@ -2870,7 +2617,7 @@ or
"created_at": 1713000000.0,
"started_at": 1713000000.0,
"finished_at": 1713000004.0,
"expires_at": 1713086404.0,
"expires_at": 1713000034.0,
"exit_code": 0,
"last_sequence": 15
}
@ -2894,7 +2641,8 @@ or
**Notes**:
- This only applies to the currently active session.
- Cancelled sessions are marked as `interrupted`.
- The client should stay attached to the stream to receive the final `close` event and refreshed terminal status.
- Cancelled sessions finalize as `interrupted` and remain replayable until the drain window expires.
- `403 Forbidden` if console is disabled.
- `404 Not Found` if the session does not exist or has already expired.
@ -2926,6 +2674,10 @@ If both `since` and `Last-Event-ID` are present, the larger value is used.
{ "exitcode": 0 }
```
**Notes**:
- Replay/restore works while the session is still running or until the finished session expires.
- Finished sessions are removed lazily after the transcript drain window elapses.
- `403 Forbidden` if console is disabled.
- `400 Bad Request` if the replay cursor is invalid.
- `404 Not Found` if the session does not exist or has already expired.
@ -3468,7 +3220,7 @@ Emitted when a download item is moved between queue and history.
"data": {
"id": "abc123",
"from": "queue",
"to": "history"
"to": "done"
}
}
```

188
FAQ.md
View file

@ -3,9 +3,6 @@
Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line,
or the `environment:` section in `compose.yaml` file.
<details>
<summary>Click to expand</summary>
| Environment Variable | Description | Default |
| ------------------------------- | ------------------------------------------------------------------- | --------------------- |
| TZ | The timezone to use for the application | `(not_set)` |
@ -40,17 +37,16 @@ or the `environment:` section in `compose.yaml` file.
| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` |
| YTP_YTDLP_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` |
| YTP_YTDLP_VERSION | The version of yt-dlp to use. Defaults to latest version | `(not_set)` |
| YTP_BROWSER_URL | Remote browser endpoint for the bundled `browser` extractor | `(not_set)` |
| YTP_FLARESOLVERR_URL | FlareSolverr endpoint URL. | `(not_set)` |
| YTP_FLARESOLVERR_MAX_TIMEOUT | Max FlareSolverr challenge timeout in seconds | `120` |
| YTP_FLARESOLVERR_CLIENT_TIMEOUT | HTTP client timeout (seconds) when calling FlareSolverr | `120` |
| YTP_FLARESOLVERR_CACHE_TTL | The cache TTL (in seconds) for FlareSolverr solutions | `600` |
| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` |
| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` |
| YTP_QUEUE_DISPLAY_LIMIT | Max queued downloads returned to the UI. `0` = unlimited | `100` |
| YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` |
| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` |
| YTP_TEMP_DISABLED | Disable temp files handling. | `false` |
| YTP_DOWNLOAD_PATH_DEPTH | How many subdirectories to show in auto complete. | `1` |
| YTP_ALLOW_INTERNAL_URLS | Allow requests to internal URLs | `false` |
| YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` |
| YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` |
@ -60,23 +56,19 @@ or the `environment:` section in `compose.yaml` file.
| YTP_IGNORE_ARCHIVED_ITEMS | Don't report archived items in the download history. | `false` |
| YTP_CHECK_FOR_UPDATES | Whether to check for application updates. | `true` |
| YTP_EXTRACT_INFO_CONCURRENCY | The number of concurrent extract info operations. | `4` |
| YTP_THUMB_CONCURRENCY | The number of concurrent ffmpeg thumbnail generations allowed. | `2` |
| YTP_THUMB_GENERATE | Enable ffmpeg thumbnail generation when no local thumbnail exists. | `true` |
| YTP_THUMB_SIDECAR | Save generated thumbnails next to media instead of temp cache. | `false` |
| YTP_DISABLE_EXEC | Strip some dangerous yt-dlp options. | `false` |
</details>
> [!NOTE]
> To raise the worker limit for a specific extractor, set an env variable using this format: `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`
> The extractor name must be uppercase. You can find the extractor name in the download logs. This value cannot be
> higher than `YTP_MAX_WORKERS`; higher values are ignored.
>
> `YTP_SIMPLE_MODE=true` only applies when the browser has no saved layout choice yet. Users can still choose a layout in
> WebUI Settings. `/?simple=1` forces and saves Simple for that browser.
>
> `YTP_AUTO_CLEAR_HISTORY_DAYS` `0` days means no automatic clearing of the download history. lowest value that will
> trigger the clearing is `1` day. This setting will **NOT** delete the downloaded files, it will only clear the
> history from the database.
> To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`.
> The extractor name must be in uppercase, to know the extractor name, check the log for the specific extractor used for the download.
> The limit should not exceed the `YTP_MAX_WORKERS` value as it will be ignored.
> [!IMPORTANT]
> The env variable `YTP_SIMPLE_MODE` only control what being displayed for first time visitor, the users can still switch between the two modes via the WebUI settings page.
## Notes about YTP_AUTO_CLEAR_HISTORY_DAYS
- `0` days means no automatic clearing of the download history. lowest value that will trigger the clearing is `1` day.
- This setting will **NOT** delete the downloaded files, it will only clear the history from the database.
# Browser extensions & bookmarklets
@ -123,35 +115,6 @@ As this is a simple basic authentication, if your browser doesn't show the promp
`http://username:password@your_ytptube_url:port`
# Security recommendations
YTPTube is designed for LAN and home-lab use behind a firewall or reverse proxy. The web interface and API are
unauthenticated by default because in a trusted network, auth adds friction without meaningful benefit. However,
if you expose YTPTube to the internet directly or via port forwarding **YOU MUST enable authentication**.
### Without auth, anyone who can reach the API can:
- Download arbitrary content through your IP and server.
- Delete or modify your downloaded files and database.
- Run arbitrary `yt-dlp` options, including `--exec`, which executes shell commands inside the container.
This is not a vulnerability, it's the intended design. The `cli` field passes options directly to `yt-dlp`,
a tool that by design can execute commands. Auth is the mechanism that controls who gets to use that power.
**If you expose YTPTube to untrusted networks**, do one of the following:
1. **Enable authentication** — set both `YTP_AUTH_USERNAME` and `YTP_AUTH_PASSWORD`.
2. **Put it behind a reverse proxy** with its own authentication layer (see [Run behind reverse proxy](#run-behind-reverse-proxy)).
3. **Keep it on a private network** with no public exposure.
YTPTube already gates other powerful features behind explicit opt-in: the built-in terminal, file browser actions and internal
URL requests for example. The `cli` field is no different, its power is by design, and access control is your responsibility.
> [!NOTE]
> If you choose to run without authentication but still want to reduce at least some impact, you can set
> `YTP_DISABLE_EXEC=true`. This strips some dangerous options at run time. However, understand that this is not a
> substitute for auth an unauthenticated API is still fully open for all other operations.
# I cant download anything
If you are receiving errors like:
@ -204,28 +167,6 @@ YTP_YTDLP_VERSION=2025.07.21 or master or nightly
Then restart the container to apply the changes.
# Custom output template placeholders
YTPTube supports custom `ytp_*` placeholders in `yt-dlp` output template via the following syntax `%(ytp_*:<args>)s`.
## Currently available extra placeholders are:
- `ytp_random`: random mixed letters and digits,
- `N` A number is required to specify the length of the random string, for example `%(ytp_random:8)s` will generate a random string of 8 characters.
- if the args followed by `:s` it will generate random letters only, if followed by `:d` it will generate random digits only.
## Examples of the custom placeholders in action:
- Template: `%(title)s [%(ytp_random:8)s].%(ext)s`
- Example result: `My Video [A7k2Pq9Z].mp4`
- Template: `%(uploader)s/%(ytp_random:6:d)s - %(title)s.%(ext)s`
- Example result: `MyChannel/483920 - My Video.mp4`
- Template: `%(playlist)s/%(ytp_random:10:s)s/%(title)s.%(ext)s`
- Example result: `Favorites/QwErTyUiOp/My Video.mp4`
> [!NOTE]
> `%(ytp_` placeholders are a YTPTube extension and not avaliable via console or directly via yt-dlp.
# How can I monitor sites without RSS feeds?
YTPTube includes a **generic task handler** that turns JSON definitions into site-specific scrapers. You can use it
@ -651,108 +592,3 @@ services:
```
For more information please visit [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) project.
# How to use the browser extractor?
YTPTube ships with a bundled `browser` extractor plugin for `yt-dlp`. It opens the target page in a remote browser,
waits for the page network activity, then tries to extract media urls from the captured requests and media elements.
This is mainly useful for sites where the regular extractor or generic page parsing does not expose the final media url,
but a real browser session does.
## Environment variables
You can define the remote browser endpoint globally with the following environment variable.
### Selenium example
```env
YTP_BROWSER_URL=selenium+http://selenium:4444/wd/hub
```
### Playwright example
For a native Playwright server, point to the Playwright websocket endpoint:
```env
YTP_BROWSER_URL=playwright+ws://playwright:3000/
```
### Playwright CDP example
To connect Playwright over the Chrome DevTools Protocol, use either the shorthand `playwright+cdp://` form or an explicit transport form:
```env
YTP_BROWSER_URL=playwright+cdp://chrome:9222/
# same as playwright+cdp+http://chrome:9222/
```
- `YTP_BROWSER_URL` is the full backend selector and endpoint in one value.
- Supported forms are:
- `selenium+http://...`
- `selenium+https://...`
- `playwright+ws://...`
- `playwright+wss://...`
- `playwright+cdp://...` which is treated as `playwright+cdp+http://...`
- `playwright+cdp+http://...`
- `playwright+cdp+https://...`
- `playwright+cdp+ws://...`
- `playwright+cdp+wss://...`
## yt-dlp usage
If you want to set the browser extractor options directly on the yt-dlp side, you can also use `--extractor-args` with `generic:url=...`:
### Selenium example
```bash
--use-extractors "generic" --extractor-args "generic:url=selenium+http://selenium:4444/wd/hub"
```
### Playwright example
```bash
--use-extractors "generic" --extractor-args "generic:url=playwright+ws://playwright:3000/"
```
### Playwright CDP example
```bash
--use-extractors "generic" --extractor-args "generic:url=playwright+cdp://chrome:9222/"
```
The explicit `--extractor-args` value takes priority over `YTP_BROWSER_URL`.
## Example compose setup
```yaml
services:
ytptube:
user: "${UID:-1000}:${UID:-1000}"
image: ghcr.io/arabcoders/ytptube:latest
container_name: ytptube
restart: unless-stopped
environment:
- YTP_BROWSER_URL=selenium+http://selenium:4444/wd/hub # or playwright+ws://playwright:3000/ or playwright+cdp://chrome:9222/
ports:
- "8081:8081"
volumes:
- ./config:/config:rw
- ./downloads:/downloads:rw
selenium:
image: selenium/standalone-chrome:latest
container_name: selenium
restart: unless-stopped
shm_size: 2gb
playwright:
image: mcr.microsoft.com/playwright:v1.59.0-noble
container_name: playwright
restart: unless-stopped
command: /bin/sh -c "npx -y playwright@1.59.0 run-server --port 3000 --host 0.0.0.0"
```
> [!NOTE]
> The browser extractor is slower than the normal extractor flow and should only be used when a site actually needs a real browser session.
> playwright require same version for both the server and the client, so make sure to use the same version in the container and in your local environment if you want to test it locally.

View file

@ -38,35 +38,26 @@ Example of the Simple mode interface.
* Conditions feature to apply custom options based on `yt-dlp` returned info.
* Custom browser extensions, bookmarklets and iOS shortcuts to send links to YTPTube instance.
* A bundled executable version for Windows, macOS and Linux. `MacOS version is untested`.
* Use playwright or selenium for extractors that require a browser. see [related FAQ](FAQ.md#how-to-use-the-browser-extractor).
Please read the [FAQ](FAQ.md) for more information.
# Installation
> [!IMPORTANT]
> By default YTPTube runs without authentication. If you expose it to the internet, **enable auth**. See [security recommendations](FAQ.md#security-recommendations).
## Run using docker command
```bash
mkdir -p ./{config,downloads/{files,tmp}} && docker run -itd --rm --user "${UID}:${UID}" --name ytptube \
mkdir -p ./{config,downloads/files,downloads/tmp} && docker run -itd --rm --user "${UID}:${UID}" --name ytptube \
-e YTP_TEMP_PATH=/downloads/tmp -e YTP_DOWNLOAD_PATH=/downloads/files \
-p 8081:8081 -v ./config:/config:rw -v ./downloads:/downloads:rw \
ghcr.io/arabcoders/ytptube:latest
```
## Run using podman
```bash
mkdir -p ./{config,downloads/{files,tmp}} && podman run -itd --rm --userns=keep-id --name ytptube \
-e YTP_TEMP_PATH=/downloads/tmp -e YTP_DOWNLOAD_PATH=/downloads/files \
-p 8081:8081 -v ./config:/config:rw -v ./downloads:/downloads:rw \
arabcoders/ytptube:latest
```
Then you can access the WebUI at `http://localhost:8081`.
> [!NOTE]
> If you are using `podman` instead of `docker`, you can use the same command, but you need to change the user to `0:0`
> it will appears to be running as root, but it will run as the user who started the container.
## Using compose file
The following is an example of a `compose.yaml` file that can be used to run YTPTube.
@ -75,8 +66,6 @@ The following is an example of a `compose.yaml` file that can be used to run YTP
services:
ytptube:
user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id.
# comment out the above line and uncomment the below line if you are using podman-compose.
#userns_mode: keep-id
image: ghcr.io/arabcoders/ytptube:latest
container_name: ytptube
restart: unless-stopped
@ -91,14 +80,18 @@ services:
```
> [!IMPORTANT]
> Make sure to change the `user` line to match your user id and group id in docker setups, or use `userns_mode: keep-id` in podman setups.
> Make sure to change the `user` line to match your user id and group id.
```bash
mkdir -p ./{config,downloads/{files,tmp}} && docker compose -f compose.yaml up -d
mkdir -p ./{config,downloads/files,downloads/tmp} && docker compose -f compose.yaml up -d
```
Then you can access the WebUI at `http://localhost:8081`.
> [!NOTE]
> you can use podman-compose instead of docker-compose, as it supports the same syntax. However, you should change the
> user to `0:0` it will appears to be running as root, but it will run as the user who started the container.
## Unraid
For `Unraid` users You can install the `Community Applications` plugin, and search for **ytptube** it comes
@ -112,14 +105,17 @@ For simple API documentation, you can refer to the [API documentation](API.md).
This project is not affiliated with yt-dlp or any other service.
This is a personal project designed to make downloading videos from the internet more convenient for me. It is not
intended for piracy or any unlawful use. This project was built primarily for my own use and preferences.
Its a personal project designed to make downloading videos from the internet more convenient. Its not intended for
piracy or any unlawful use.
AI-assisted tools are used in this project. If you are uncomfortable with this, you should not use this project.
AI-based tools may have been used to assist with parts of this project. Regardless of how a change is produced, every
change is reviewed and approved by the human maintainer before it is included.
Contributions are welcome, but I may decline changes that do not interest me or do not align with my vision for this
project. Unsolicited pull requests will be closed. For suggestions or feature requests, please open a discussion or
join the Discord server.
This project was built primarily for my own needs and preferences. The UI might not be the most polished or visually
refined, but Im happy with it as it is. You can, however, create and load your own UI for complete customization.
Contributions are welcome, but I may decline changes that dont align with my vision for the project. Unsolicited pull
requests may be ignored. For suggestions or feature requests, please open a discussion or join the Discord server.
# Social contact

View file

@ -16,10 +16,7 @@ def pytest_configure(config) -> None:
if getattr(config.option, "basetemp", None) is None:
config.option.basetemp = str(get_test_run_root() / "pytest")
os.environ["YTP_FILE_LOGGING"] = "false"
def pytest_unconfigure(config) -> None:
del config
os.environ.pop("YTP_FILE_LOGGING", None)
cleanup_test_run_root()

View file

@ -1,18 +1,18 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
from app.features.conditions.models import ConditionModel
from app.features.core.migration import Migration as FeatureMigration
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.conditions.repository import ConditionsRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -36,11 +36,7 @@ class Migration(FeatureMigration):
try:
items: list[dict] | None = json.loads(self._source_file.read_text())
except Exception as exc:
LOG.exception(
"Failed to read conditions migration file '%s'. Ignoring.",
self._source_file,
extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
await self._move_file(self._source_file)
return
@ -58,11 +54,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert condition '%s'.",
normalized.name,
extra={"condition_name": normalized.name, "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert condition '%s': %s", normalized.name, exc)
LOG.info("Migrated %s condition(s) from %s.", inserted, self._source_file)
await self._move_file(self._source_file)

View file

@ -1,50 +1,31 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from app.features.conditions.migration import Migration
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
from contextlib import AbstractAsyncContextManager
from collections.abc import AsyncGenerator, Iterable
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
from sqlalchemy import delete, func, or_, select
from app.features.conditions.models import ConditionModel
from app.features.core.deps import get_session
from app.library.log import get_logger
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> ConditionModel:
model = ConditionModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for ConditionModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: ConditionModel | dict[str, Any]) -> ConditionModel:
if isinstance(payload, ConditionModel):
return payload
return _model_from_payload(payload)
LOG: logging.Logger = logging.getLogger(__name__)
class ConditionsRepository(metaclass=Singleton):
def __init__(self, session: SessionFactory | None = None) -> None:
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: SessionFactory = session or get_session
self.session: AsyncGenerator[AsyncSession] = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -57,7 +38,7 @@ class ConditionsRepository(metaclass=Singleton):
def get_instance() -> ConditionsRepository:
return ConditionsRepository()
async def all(self) -> list[ConditionModel]:
async def list(self) -> list[ConditionModel]:
async with self.session() as session:
result: Result[tuple[ConditionModel]] = await session.execute(
select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc())
@ -112,11 +93,11 @@ class ConditionsRepository(metaclass=Singleton):
result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: ConditionModel | dict[str, Any]) -> ConditionModel:
async def create(self, payload: ConditionModel | dict) -> ConditionModel:
async with self.session() as session:
model = _coerce_model(payload)
model: ConditionModel = ConditionModel(**payload) if isinstance(payload, dict) else payload
if model.id is not None:
model.id = None # ty: ignore
model.id = None
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Condition with name '{model.name}' already exists."
@ -179,11 +160,13 @@ class ConditionsRepository(metaclass=Singleton):
await session.commit()
return model
async def replace_all(self, items: Iterable[dict[str, Any] | ConditionModel]) -> list[ConditionModel]:
async def replace_all(self, items: Iterable[dict | ConditionModel]) -> list[ConditionModel]:
async with self.session() as session:
try:
await session.execute(delete(ConditionModel))
models: list[ConditionModel] = [_coerce_model(item) for item in items]
models: list[ConditionModel] = [
ConditionModel(**item) if isinstance(item, dict) else item for item in items
]
session.add_all(models)
await session.commit()
except Exception:

View file

@ -1,4 +1,4 @@
import asyncio
import logging
from collections import OrderedDict
from typing import Any
@ -14,11 +14,10 @@ from app.library.cache import Cache
from app.library.config import Config
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.router import route
from app.library.Utils import validate_url
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def _model(model: Any) -> Condition:
@ -86,7 +85,7 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf
return web.json_response({"error": "condition is required."}, status=web.HTTPBadRequest.status_code)
try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
validate_url(url, allow_internal=config.allow_internal_urls)
except ValueError as e:
return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code)
@ -115,44 +114,18 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf
else:
data = cache.get(key)
except Exception as e:
LOG.exception(
"Failed to extract video info for condition check '%s'.",
cond,
extra={
"route": "conditions.match",
"condition": cond,
"url": url,
"preset": preset,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
return web.json_response(
data={"error": f"Failed to extract video info. '{e!s}'"},
status=web.HTTPInternalServerError.status_code,
)
if not isinstance(data, dict):
return web.json_response(
data={"error": "Failed to extract video info."},
status=web.HTTPInternalServerError.status_code,
)
try:
from app.features.ytdlp.mini_filter import match_str
status: bool = match_str(cond, data)
except Exception as e:
LOG.exception(
"Failed to evaluate condition '%s'.",
cond,
extra={
"route": "conditions.match",
"condition": cond,
"url": url,
"preset": preset,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
return web.json_response(
data={"error": str(e)},
status=web.HTTPBadRequest.status_code,

View file

@ -1,19 +1,19 @@
import logging
from collections.abc import Iterable
from numbers import Number
from aiohttp import web
from app.features.conditions.models import ConditionModel
from app.features.conditions.repository import ConditionsRepository
from app.features.conditions.schemas import Condition
from app.features.ytdlp.mini_filter import match_str
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.Singleton import Singleton
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("feature.conditions")
def _ignored_identifiers(ignore_conditions: Iterable[str | int | float] | None) -> tuple[set[str], bool]:
def _ignored_identifiers(ignore_conditions: Iterable[str | Number] | None) -> tuple[set[str], bool]:
ignored: set[str] = set()
ignore_all = False
@ -21,7 +21,7 @@ def _ignored_identifiers(ignore_conditions: Iterable[str | int | float] | None)
return ignored, ignore_all
for value in ignore_conditions:
if isinstance(value, bool) or not isinstance(value, (str, int, float)):
if isinstance(value, bool) or not isinstance(value, (str, Number)):
continue
identifier = str(value).strip()
@ -55,7 +55,7 @@ class Conditions(metaclass=Singleton):
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations")
async def get_all(self) -> list[ConditionModel]:
return await self._repo.all()
return await self._repo.list()
async def save(self, item: ConditionModel | dict) -> ConditionModel:
"""
@ -80,7 +80,7 @@ class Conditions(metaclass=Singleton):
if item.id is None or 0 == item.id:
model = await repo.create(item)
else:
model = await repo.update(item.id, Condition.model_validate(item).model_dump())
model = await repo.update(item.id, item.serialize())
except KeyError as exc:
raise ValueError(str(exc)) from exc
@ -113,15 +113,13 @@ class Conditions(metaclass=Singleton):
repo = self._repo
return await repo.get(identifier)
async def match(
self, info: dict, ignore_conditions: Iterable[str | int | float] | None = None
) -> ConditionModel | None:
async def match(self, info: dict, ignore_conditions: Iterable[str | Number] | None = None) -> ConditionModel | None:
"""
Check if any condition matches the info dict.
Args:
info (dict): The info dict to check.
ignore_conditions (Iterable[str | int | float] | None): Condition ids or names to skip for this match.
ignore_conditions (Iterable[str | Number] | None): Condition ids or names to skip for this match.
Returns:
Condition|None: The condition if found, None otherwise.
@ -135,7 +133,7 @@ class Conditions(metaclass=Singleton):
return None
repo = self._repo
items: list[ConditionModel] = await repo.all()
items: list[ConditionModel] = await repo.list()
if len(items) < 1:
return None
@ -147,29 +145,17 @@ class Conditions(metaclass=Singleton):
continue
if not item.filter:
LOG.error(
"Filter is empty for '%s'.", item.name, extra={"condition_id": item.id, "condition_name": item.name}
)
LOG.error(f"Filter is empty for '{item.name}'.")
continue
try:
if not match_str(item.filter, info):
continue
LOG.debug(
"Matched '%s: %s' with filter '%s'.",
item.id,
item.name,
item.filter,
extra={"condition_id": item.id, "condition_name": item.name, "filter": item.filter},
)
LOG.debug(f"Matched '{item.id}: {item.name}' with filter '{item.filter}'.")
return item
except Exception as e:
LOG.exception(
"Failed to evaluate condition '%s'.",
item.name,
extra={"condition_id": item.id, "condition_name": item.name, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to evaluate '{item.id}: {item.name}'. '{e!s}'.")
continue
return None
@ -189,7 +175,7 @@ class Conditions(metaclass=Singleton):
if not info or not isinstance(info, dict) or len(info) < 1:
return None
if not (item := await self.get(str(identifier))) or not item.enabled or not item.filter:
if not (item := await self.get(identifier)) or not item.enabled or not item.filter:
return None
return item if match_str(item.filter, info) else None

View file

@ -6,7 +6,6 @@ import pytest
import pytest_asyncio
from types import SimpleNamespace
import pytest
from typing import Any
from aiohttp import web
from aiohttp.test_utils import make_mocked_request
@ -40,13 +39,13 @@ async def repo():
def _json_request(path: str, payload: object) -> web.Request:
mock_request: Any = make_mocked_request("POST", path)
request = make_mocked_request("POST", path)
async def _json() -> object:
return payload
mock_request.json = _json
return mock_request
request.json = _json # type: ignore[attr-defined]
return request
class TestAllowInternalUrlsScope:
@ -54,7 +53,7 @@ class TestAllowInternalUrlsScope:
Config._reset_singleton()
@pytest.mark.asyncio
async def test_rejects_internal_url(self) -> None:
async def test_conditions_test_rejects_internal_url_when_disallowed(self) -> None:
config = Config.get_instance()
config.allow_internal_urls = False
encoder = Encoder()
@ -209,7 +208,7 @@ class TestConditionsRepository:
await repo.create({"name": "A", "filter": "test", "priority": 1})
await repo.create({"name": "C", "filter": "test", "priority": 2})
items = await repo.all()
items = await repo.list()
assert items[0].name == "C", "Highest priority should be first"
assert items[1].name == "A", "Same priority sorted alphabetically"
@ -241,6 +240,6 @@ class TestConditionsRepository:
assert len(result) == 2, "Should create 2 new conditions"
all_items = await repo.all()
all_items = await repo.list()
assert len(all_items) == 2, "Should only have new conditions"
assert all_items[0].name in ["New 1", "New 2"], "Should only have new items"

View file

@ -1,16 +1,15 @@
from __future__ import annotations
import abc
import logging
import time
from pathlib import Path
from typing import TYPE_CHECKING
from app.library.log import get_logger
if TYPE_CHECKING:
from app.library.config import Config
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(abc.ABC):
@ -27,11 +26,7 @@ class Migration(abc.ABC):
try:
await self.migrate()
except Exception as exc:
LOG.exception(
"Feature migration '%s' failed.",
self.name,
extra={"feature": self.name, "exception_type": type(exc).__name__},
)
LOG.exception("Feature migration '%s' failed: %s", self.name, exc)
return False
return True

View file

@ -3,7 +3,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import DateTime as SQLADateTime
from sqlalchemy import Dialect, TypeDecorator
from sqlalchemy import TypeDecorator
from sqlalchemy.orm import DeclarativeBase
@ -20,18 +20,16 @@ class UTCDateTime(TypeDecorator):
impl = SQLADateTime
cache_ok = True
def process_bind_param(self, value: datetime | None, dialect: Dialect) -> datetime | None:
def process_bind_param(self, value: datetime | None, _dialect) -> datetime | None:
"""Convert datetime to UTC before storing."""
_ = dialect
if value is not None:
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC).replace(tzinfo=None)
return value
def process_result_value(self, value: datetime | None, dialect: Dialect) -> datetime | None:
def process_result_value(self, value: datetime | None, _dialect) -> datetime | None:
"""Ensure datetime is timezone-aware (UTC) when loading."""
_ = dialect
if value is not None and value.tzinfo is None:
return value.replace(tzinfo=UTC)
return value

View file

@ -1,18 +1,18 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
from app.features.core.migration import Migration as FeatureMigration
from app.features.dl_fields.schemas import DLField
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.dl_fields.repository import DLFieldsRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -36,11 +36,7 @@ class Migration(FeatureMigration):
try:
items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text())
except Exception as exc:
LOG.exception(
"Failed to read download fields migration file '%s'. Ignoring.",
self._source_file,
extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
await self._move_file(self._source_file)
return
@ -58,11 +54,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert download field '%s'.",
normalized["name"],
extra={"field_name": normalized["name"], "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert dl field '%s': %s", normalized["name"], exc)
LOG.info("Migrated %s dl field(s) from %s.", inserted, self._source_file)
await self._move_file(self._source_file)

View file

@ -1,49 +1,30 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any, cast
import logging
from typing import TYPE_CHECKING, Any
from sqlalchemy import delete, func, or_, select
from app.features.core.deps import get_session
from app.features.dl_fields.migration import Migration
from app.features.dl_fields.models import DLFieldModel
from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
from contextlib import AbstractAsyncContextManager
from collections.abc import AsyncGenerator, Iterable
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> DLFieldModel:
model = DLFieldModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for DLFieldModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
if isinstance(payload, DLFieldModel):
return payload
return _model_from_payload(payload)
LOG: logging.Logger = logging.getLogger(__name__)
class DLFieldsRepository(metaclass=Singleton):
def __init__(self, session: SessionFactory | None = None) -> None:
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: SessionFactory = session or get_session
self.session: AsyncGenerator[AsyncSession] = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -56,7 +37,7 @@ class DLFieldsRepository(metaclass=Singleton):
def get_instance() -> DLFieldsRepository:
return DLFieldsRepository()
async def all(self) -> list[DLFieldModel]:
async def list(self) -> list[DLFieldModel]:
async with self.session() as session:
result: Result[tuple[DLFieldModel]] = await session.execute(
select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc())
@ -109,9 +90,9 @@ class DLFieldsRepository(metaclass=Singleton):
result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
async def create(self, payload: DLFieldModel | dict) -> DLFieldModel:
async with self.session() as session:
model = _coerce_model(payload)
model: DLFieldModel = DLFieldModel(**payload) if isinstance(payload, dict) else payload
if await self.get_by_name(name=model.name) is not None:
msg: str = f"DL field with name '{model.name}' already exists."
@ -172,16 +153,13 @@ class DLFieldsRepository(metaclass=Singleton):
await session.commit()
return model
async def replace_all(self, items: Iterable[dict[str, Any] | DLFieldModel]) -> list[DLFieldModel]:
async def replace_all(self, items: Iterable[dict | DLFieldModel]) -> list[DLFieldModel]:
async with self.session() as session:
try:
await session.execute(delete(DLFieldModel))
models: list[DLFieldModel] = []
for item in items:
if isinstance(item, dict):
models.append(DLFieldModel(**cast("dict[str, Any]", item)))
else:
models.append(item)
models: list[DLFieldModel] = [
DLFieldModel(**item) if isinstance(item, dict) else item for item in items
]
session.add_all(models)
await session.commit()
except Exception:

View file

@ -1,3 +1,4 @@
import logging
from typing import Any
from aiohttp import web
@ -10,10 +11,9 @@ from app.features.dl_fields.schemas import DLField, DLFieldList, DLFieldPatch
from app.features.dl_fields.service import DLFields
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.router import route
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def _model(model: Any) -> DLField:

View file

@ -1,18 +1,18 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from app.features.dl_fields.models import DLFieldModel
from app.features.dl_fields.repository import DLFieldsRepository
from app.features.dl_fields.schemas import DLField
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from aiohttp import web
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("feature.dl_fields")
class DLFields(metaclass=Singleton):
@ -33,10 +33,10 @@ class DLFields(metaclass=Singleton):
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations")
async def get_all(self) -> list[DLFieldModel]:
return await self._repo.all()
return await self._repo.list()
async def get_all_serialized(self) -> list[dict[str, Any]]:
items = await self._repo.all()
items = await self._repo.list()
return [DLField.model_validate(item).model_dump() for item in items]
async def save(self, item: DLField | dict) -> DLFieldModel:

View file

@ -204,7 +204,7 @@ class TestDLFieldsRepository:
await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1})
await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0})
items = await repo.all()
items = await repo.list()
assert items[0].name == "c", "Lowest order should be first"
assert items[1].name == "a", "Same order sorted alphabetically"
@ -243,6 +243,6 @@ class TestDLFieldsRepository:
assert len(result) == 2, "Should create 2 new fields"
all_items = await repo.all()
all_items = await repo.list()
assert len(all_items) == 2, "Should only have new fields"
assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items"

View file

@ -1,6 +1,7 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
@ -8,12 +9,11 @@ from app.features.core.migration import Migration as FeatureMigration
from app.features.notifications.schemas import NotificationEvents
from app.features.presets.service import Presets
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.notifications.repository import NotificationsRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -37,11 +37,7 @@ class Migration(FeatureMigration):
try:
items: list[dict] | None = json.loads(self._source_file.read_text())
except Exception as exc:
LOG.exception(
"Failed to read notifications migration file '%s'. Ignoring.",
self._source_file,
extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
await self._move_file(self._source_file)
return
@ -60,11 +56,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert notification target '%s'.",
normalized.get("name"),
extra={"target_name": normalized.get("name"), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert notification '%s': %s", normalized.get("name"), exc)
LOG.info("Migrated %s notification target(s) from %s.", inserted, self._source_file)
await self._move_file(self._source_file)

View file

@ -1,5 +1,6 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from sqlalchemy import func, or_, select
@ -7,43 +8,23 @@ from sqlalchemy import func, or_, select
from app.features.core.deps import get_session
from app.features.notifications.migration import Migration
from app.features.notifications.models import NotificationModel
from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from collections.abc import AsyncGenerator
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> NotificationModel:
model = NotificationModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for NotificationModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: NotificationModel | dict[str, Any]) -> NotificationModel:
if isinstance(payload, NotificationModel):
return payload
return _model_from_payload(payload)
LOG: logging.Logger = logging.getLogger(__name__)
class NotificationsRepository(metaclass=Singleton):
def __init__(self, session: SessionFactory | None = None) -> None:
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: SessionFactory = session or get_session
self.session: AsyncGenerator[AsyncSession] = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -56,7 +37,7 @@ class NotificationsRepository(metaclass=Singleton):
def get_instance() -> NotificationsRepository:
return NotificationsRepository()
async def all(self) -> list[NotificationModel]:
async def list(self) -> list[NotificationModel]:
async with self.session() as session:
result: Result[tuple[NotificationModel]] = await session.execute(
select(NotificationModel).order_by(NotificationModel.name.asc())
@ -111,11 +92,11 @@ class NotificationsRepository(metaclass=Singleton):
result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: NotificationModel | dict[str, Any]) -> NotificationModel:
async def create(self, payload: NotificationModel | dict) -> NotificationModel:
async with self.session() as session:
model = _coerce_model(payload)
model: NotificationModel = NotificationModel(**payload) if isinstance(payload, dict) else payload
if model.id is not None:
model.id = None # ty: ignore
model.id = None
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Notification target with name '{model.name}' already exists."

View file

@ -1,3 +1,4 @@
import logging
from typing import Any
from aiohttp import web
@ -10,10 +11,9 @@ from app.features.notifications.schemas import Notification, NotificationEvents,
from app.features.notifications.service import Notifications
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.router import route
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def _model(model: Any) -> Notification:

View file

@ -1,8 +1,10 @@
from __future__ import annotations
import asyncio
import logging
import traceback
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
from typing import TYPE_CHECKING, Any
from app.features.notifications.models import NotificationModel
from app.features.notifications.repository import NotificationsRepository
@ -21,7 +23,6 @@ from app.library.encoder import Encoder
from app.library.Events import Event, EventBus, Events
from app.library.httpx_client import async_client
from app.library.ItemDTO import Item, ItemDTO
from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
@ -30,7 +31,7 @@ if TYPE_CHECKING:
import httpx
from aiohttp import web
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("feature.notifications")
class Notifications(metaclass=Singleton):
@ -64,10 +65,10 @@ class Notifications(metaclass=Singleton):
await self._repo.run_migrations()
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations")
EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{type(self).__name__}.emit")
EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{__class__.__name__}.emit")
async def all(self) -> list[NotificationModel]:
return await self._repo.all()
async def list(self) -> list[NotificationModel]:
return await self._repo.list()
async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]:
return await self._repo.list_paginated(page, per_page)
@ -92,8 +93,8 @@ class Notifications(metaclass=Singleton):
async def delete(self, identifier: int | str) -> NotificationModel:
return await self._repo.delete(identifier)
async def send(self, ev: Event, wait: bool = True) -> list[dict | Awaitable[dict]]:
targets = await self._repo.all()
async def send(self, ev: Event, wait: bool = True) -> list[dict] | list[Awaitable[dict]]:
targets = await self._repo.list()
if len(targets) < 1:
return []
@ -126,7 +127,7 @@ class Notifications(metaclass=Singleton):
if wait:
return await asyncio.gather(*tasks)
return cast("list[dict | Awaitable[dict]]", tasks)
return tasks
def emit(self, e: Event, _, **__) -> None:
if not NotificationEvents.is_valid(e.event):
@ -271,27 +272,14 @@ class Notifications(metaclass=Singleton):
if not status:
msg = "Apprise failed to send notification."
self._raise_apprise_error(msg)
raise RuntimeError(msg) # noqa: TRY301
except Exception as exc:
LOG.exception(
"Failed to send Apprise notification for event '%s'.",
ev.event,
extra={
"event_id": ev.id,
"event": ev.event,
"target_count": len(targets),
"targets": [t.name for t in targets],
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
LOG.error("Error sending Apprise notification: %s", exc)
return {"error": str(exc), "event": ev.event, "id": ev.id, "targets": [t.name for t in targets]}
return {}
@staticmethod
def _raise_apprise_error(msg: str) -> None:
raise RuntimeError(msg)
async def _send(self, target: Notification, ev: Event) -> dict:
try:
LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name)
@ -347,17 +335,14 @@ class Notifications(metaclass=Singleton):
return resp_data
except Exception as exc:
LOG.exception(
"Failed to send notification event '%s: %s' to '%s'.",
err_msg = str(exc) or type(exc).__name__
tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
LOG.error(
"Error sending notification event '%s: %s' to '%s'. '%s'. %s",
ev.event,
ev.id,
target.name,
extra={
"event_id": ev.id,
"event": ev.event,
"target_name": target.name,
"url": target.request.url,
"exception_type": type(exc).__name__,
},
err_msg,
tb,
)
return {"url": target.request.url, "status": 500, "text": str(ev)}

View file

@ -35,7 +35,7 @@ class TestNotificationsRepository:
@pytest.mark.asyncio
async def test_list_empty(self, repo):
"""List returns empty when no notifications exist."""
notifications = await repo.all()
notifications = await repo.list()
assert notifications == [], "Should return empty list when no notifications exist"
@pytest.mark.asyncio

View file

@ -1,6 +1,7 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any
@ -8,12 +9,11 @@ from app.features.core.migration import Migration as FeatureMigration
from app.features.presets.schemas import Preset
from app.features.presets.utils import preset_name
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.presets.repository import PresetsRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -32,11 +32,7 @@ class Migration(FeatureMigration):
try:
items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text())
except Exception as exc:
LOG.exception(
"Failed to read presets migration file '%s'. Ignoring.",
self._source_file,
extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
await self._move_file(self._source_file)
return
@ -46,7 +42,7 @@ class Migration(FeatureMigration):
return
inserted = 0
seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.all()}
seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.list()}
for index, item in enumerate(items):
if not (normalized := self._normalize(item, index, seen_names)):
@ -55,11 +51,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert preset '%s'.",
normalized["name"],
extra={"preset": normalized["name"], "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert preset '%s': %s", normalized["name"], exc)
LOG.info("Migrated %s preset(s) from %s.", inserted, self._source_file)
await self._move_file(self._source_file)

View file

@ -1,5 +1,6 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from sqlalchemy import func, or_, select
@ -11,7 +12,6 @@ from app.features.presets.models import PresetModel
from app.features.presets.utils import preset_name, seed_defaults
from app.library.config import Config
from app.library.Events import Event, EventBus, Events
from app.library.log import get_logger
from app.library.Services import Services
from app.library.Singleton import Singleton
@ -27,31 +27,7 @@ if TYPE_CHECKING:
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
def _payload_data(payload: PresetModel | dict[str, Any]) -> dict[str, Any]:
if isinstance(payload, dict):
data: dict[str, Any] = {}
for key, value in payload.items():
if not isinstance(key, str):
msg = "Preset payload keys must be strings."
raise TypeError(msg)
data[key] = value
return data
return {
"name": payload.name,
"description": payload.description,
"folder": payload.folder,
"template": payload.template,
"cookies": payload.cookies,
"cli": payload.cli,
"default": payload.default,
"priority": payload.priority,
"created_at": payload.created_at,
"updated_at": payload.updated_at,
}
LOG: logging.Logger = logging.getLogger(__name__)
class PresetsRepository(metaclass=Singleton):
@ -97,15 +73,15 @@ class PresetsRepository(metaclass=Singleton):
LOG.debug("Refreshing presets cache due to configuration update.")
await self._update_cache()
Services.get_instance().add(PresetsRepository.__name__, self)
Services.get_instance().add(__class__.__name__, self)
EventBus.get_instance().subscribe(
Events.STARTED, handle_event, f"{PresetsRepository.__name__}.run_migrations"
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache")
async def _update_cache(self) -> None:
from app.features.presets.service import Presets
await Presets.get_instance().refresh_cache(await self.all())
await Presets.get_instance().refresh_cache(await self.list())
@staticmethod
def get_instance() -> PresetsRepository:
@ -121,7 +97,7 @@ class PresetsRepository(metaclass=Singleton):
LOG.error("Default preset '%s' not found, using 'default' preset.", default_name)
config.default_preset = "default"
async def all(self) -> list[PresetModel]:
async def list(self) -> list[PresetModel]:
async with self.session() as session:
result: Result[tuple[PresetModel]] = await session.execute(
select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc())
@ -262,9 +238,24 @@ class PresetsRepository(metaclass=Singleton):
result: Result[tuple[PresetModel]] = await session.execute(query.limit(1))
return result.scalar_one_or_none()
async def create(self, payload: PresetModel | dict[str, Any]) -> PresetModel:
async def create(self, payload: PresetModel | dict) -> PresetModel:
async with self.session() as session:
data = _payload_data(payload)
data: dict[str, Any]
if isinstance(payload, dict):
data = dict(payload)
else:
data = {
"name": payload.name,
"description": payload.description,
"folder": payload.folder,
"template": payload.template,
"cookies": payload.cookies,
"cli": payload.cli,
"default": payload.default,
"priority": payload.priority,
"created_at": payload.created_at,
"updated_at": payload.updated_at,
}
data.pop("id", None)
model = PresetModel(**data)

View file

@ -12,7 +12,7 @@ class Presets(metaclass=Singleton):
def __init__(self, repo: PresetsRepository | None = None) -> None:
self._repo: PresetsRepository = repo or PresetsRepository.get_instance()
self._cache: list[tuple[int, str, Preset]] = []
Services.get_instance().add(type(self).__name__, self)
Services.get_instance().add(__class__.__name__, self)
@staticmethod
def get_instance() -> Presets:

View file

@ -57,7 +57,7 @@ class TestPresetsRepository:
await repo.create({"name": "A", "priority": 1})
await repo.create({"name": "C", "priority": 2})
items = await repo.all()
items = await repo.list()
assert items[0].name == "c", "Highest priority should be first"
assert items[1].name == "a", "Same priority should sort by name"
@ -99,7 +99,7 @@ class TestPresetsRepository:
assert total_pages == 1, "Should compute pages from the filtered total"
@pytest.mark.asyncio
async def test_list_paginated_multi_sort(self, repo):
async def test_list_paginated_supports_multiple_sort_fields(self, repo):
await repo.create({"name": "Charlie", "priority": 2})
await repo.create({"name": "Alpha", "priority": 1})
await repo.create({"name": "Bravo", "priority": 1})
@ -113,24 +113,24 @@ class TestPresetsRepository:
], "Should support multiple sort fields and directions"
@pytest.mark.asyncio
async def test_list_paginated_bad_sort(self, repo):
async def test_list_paginated_rejects_invalid_sort_field(self, repo):
with pytest.raises(ValueError, match="sort must use supported fields"):
await repo.list_paginated(page=1, per_page=10, sort="cli", order="asc")
@pytest.mark.asyncio
async def test_list_paginated_bad_order(self, repo):
async def test_list_paginated_rejects_invalid_sort_direction(self, repo):
with pytest.raises(ValueError, match="order must be 'asc' or 'desc'"):
await repo.list_paginated(page=1, per_page=10, sort="name", order="sideways")
@pytest.mark.asyncio
async def test_list_paginated_mismatched_order(self, repo):
async def test_list_paginated_rejects_mismatched_sort_and_order_lengths(self, repo):
with pytest.raises(ValueError, match="order must provide one direction or match the number of sort fields"):
await repo.list_paginated(page=1, per_page=10, sort="priority,name", order="asc,desc,asc")
@pytest.mark.asyncio
class TestPresetRoutes:
async def test_list_route_sort(self, repo):
async def test_list_route_supports_sort_params(self, repo):
await repo.create({"name": "Alpha", "priority": 1})
await repo.create({"name": "Bravo", "priority": 1})
await repo.create({"name": "Charlie", "priority": 2})
@ -144,7 +144,7 @@ class TestPresetRoutes:
assert response.status == web.HTTPOk.status_code, "Should return 200 for valid sorting"
assert [item["name"] for item in payload["items"]] == ["bravo", "alpha", "charlie"], "Should sort response"
async def test_list_route_bad_sort(self, repo):
async def test_list_route_rejects_invalid_sort_field(self, repo):
request = MagicMock(spec=Request)
request.query = {"sort": "cli", "order": "asc"}
@ -154,7 +154,7 @@ class TestPresetRoutes:
assert response.status == web.HTTPBadRequest.status_code, "Should reject unsupported sort field"
assert "sort" in payload["error"], "Should explain invalid sort field"
async def test_list_route_bad_order(self, repo):
async def test_list_route_rejects_invalid_sort_direction(self, repo):
request = MagicMock(spec=Request)
request.query = {"sort": "name", "order": "sideways"}
@ -164,7 +164,7 @@ class TestPresetRoutes:
assert response.status == web.HTTPBadRequest.status_code, "Should reject unsupported sort direction"
assert "order" in payload["error"], "Should explain invalid sort direction"
async def test_list_route_exclude_defaults(self, repo):
async def test_list_route_supports_excluding_defaults(self, repo):
await repo.create({"name": "System Default", "default": True, "priority": 10})
await repo.create({"name": "Custom Preset", "priority": 1})

View file

@ -1,11 +1,10 @@
import logging
import re
from datetime import UTC, datetime
from app.library.log import get_logger
NAME_WHITESPACE_PATTERN = re.compile(r"\s+")
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
async def seed_defaults(repo) -> None:
@ -49,11 +48,7 @@ async def seed_defaults(repo) -> None:
await repo.update(existing.id, payload)
except Exception as exc:
LOG.exception(
"Failed to seed default preset '%s'.",
preset.get("name"),
extra={"preset": preset.get("name"), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to seed default preset '%s': %s", preset.get("name"), exc)
def preset_name(value: str) -> str:

View file

@ -5,16 +5,18 @@ Python wrapper for ffprobe command line tool. ffprobe must exist in the path.
import asyncio
import functools
import json
import logging
import operator
import os
import subprocess # qa: ignore
from pathlib import Path
import anyio
from app.features.streaming.types import FFProbeError
from app.library.log import get_logger
from app.library.Utils import timed_lru_cache
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("streaming.ffprobe")
class FFStream:
@ -37,26 +39,21 @@ class FFStream:
def __repr__(self):
if "codec_long_name" not in self.__dict__:
self.__dict__["codec_long_name"] = self.__dict__.get("codec_name", "")
index = self.__dict__.get("index", "?")
codec_type = self.__dict__.get("codec_type", "unknown")
codec_long_name = self.__dict__.get("codec_long_name", "")
self.codec_long_name = self.__dict__.get("codec_name", "")
if self.is_video():
return f"<Stream: #{index} [{codec_type}] {codec_long_name}, {self.__dict__.get('framerate')}, ({self.__dict__.get('width')}x{self.__dict__.get('height')})>"
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, {self.framerate}, ({self.width}x{self.height})>"
if self.is_audio():
return (
f"<Stream: #{index} [{codec_type}] {codec_long_name}, "
f"channels: {self.__dict__.get('channels')} ({self.__dict__.get('channel_layout')}), "
f"{self.__dict__.get('sample_rate')}Hz> "
f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, channels: {self.channels} ({self.channel_layout}), "
"{sample_rate}Hz> "
)
if self.is_subtitle() or self.is_attachment():
return f"<Stream: #{index} [{codec_type}] {codec_long_name}>"
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}>"
return f"<Stream: #{index} [{codec_type}]>"
return f"<Stream: #{self.index} [{self.codec_type}]>"
def is_audio(self):
"""
@ -255,14 +252,14 @@ async def ffprobe(file: Path | str) -> FFProbeResult:
raise OSError(msg)
try:
proc = await asyncio.create_subprocess_exec(
"ffprobe",
"-h",
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
)
await proc.wait()
async with await anyio.open_file(os.devnull, "w") as tempf:
await asyncio.create_subprocess_exec(
"ffprobe",
"-h",
stdout=tempf,
stderr=tempf,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
)
except FileNotFoundError as e:
msg = "ffprobe not found."
raise OSError(msg) from e

View file

@ -39,7 +39,7 @@ class M3u8:
m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
segmentSize: str = f"{self.duration:.6f}"
segmentSize: float = f"{self.duration:.6f}"
splits: int = math.ceil(duration / self.duration)
segmentParams: dict = {}

View file

@ -14,7 +14,7 @@ class Playlist:
"The path where files are downloaded."
async def make(self, file: Path) -> str:
ref: Path = Path(str(file.relative_to(self.download_path)).strip("/"))
ref: str = Path(str(file.relative_to(self.download_path)).strip("/"))
try:
ff = await ffprobe(file)

View file

@ -1,7 +1,7 @@
# flake8: noqa: ARG002
from __future__ import annotations
import os
import shutil
import subprocess
import sys
from functools import lru_cache
@ -16,13 +16,9 @@ def detect_qsv_capabilities() -> dict[str, dict[str, bool]]:
Returns a dict where keys are codec names (e.g. "h264", "hevc", "vp9") and
values are dicts with keys "full" and "lp" booleans.
"""
vainfo = shutil.which("vainfo")
if not vainfo:
return {}
try:
result: subprocess.CompletedProcess[str] = subprocess.run(
[vainfo],
["vainfo"], # noqa: S607
capture_output=True,
text=True,
check=False,
@ -91,13 +87,9 @@ def ffmpeg_encoders() -> set[str]:
"""
from app.library.config import SUPPORTED_CODECS
ffmpeg = shutil.which("ffmpeg")
if not ffmpeg:
return set()
try:
result: subprocess.CompletedProcess[str] = subprocess.run(
[ffmpeg, "-hide_banner", "-loglevel", "error", "-encoders"],
["ffmpeg", "-hide_banner", "-loglevel", "error", "-encoders"], # noqa: S607
capture_output=True,
text=True,
check=False,
@ -157,11 +149,9 @@ class _BaseBuilder:
codec_name: str
def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
_ = ctx
return []
def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
_ = ctx
return [*args, "-codec:v", self.codec_name]
@ -322,4 +312,4 @@ def encoder_fallback_chain(codec: str) -> tuple[str, ...]:
"libx264": [],
}
return tuple(chains.get(codec, chains["libx264"]))
return chains.get(codec, chains["libx264"])

View file

@ -1,6 +1,7 @@
import asyncio
import logging
import os
import subprocess
import subprocess # type: ignore
import sys
import tempfile
from pathlib import Path
@ -18,7 +19,6 @@ from app.features.streaming.library.segment_encoders import (
select_encoder,
)
from app.library.config import SUPPORTED_CODECS, Config
from app.library.log import get_logger
if TYPE_CHECKING:
from asyncio.subprocess import Process
@ -26,7 +26,7 @@ if TYPE_CHECKING:
from .ffprobe import FFProbeResult
from .segment_encoders import EncoderBuilder
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("player.segments")
class Segments:
@ -170,15 +170,9 @@ class Segments:
stderr_task: asyncio.Task[None] = asyncio.create_task(_drain_stderr())
LOG.debug(
"Streaming segment %s for '%s'.",
self.index,
file,
extra={"file": str(file), "segment_index": self.index, "ffmpeg_args": args},
)
LOG.debug(f"Streaming '{file}' segment '{self.index}'. ffmpeg: {' '.join(args)}")
try:
assert proc.stdout is not None
while True:
chunk: bytes = await proc.stdout.read(1024 * 64)
if not chunk:
@ -197,10 +191,7 @@ class Segments:
try:
await asyncio.wait_for(proc.wait(), timeout=5)
except TimeoutError:
LOG.warning(
"Segment stream for '%s' did not stop ffmpeg in time after the client disconnected; killing it.",
file,
)
LOG.error("ffmpeg process did not terminate in time. Killing it.")
proc.kill()
raise
except ConnectionResetError:
@ -213,10 +204,7 @@ class Segments:
try:
rc: int = await asyncio.wait_for(proc.wait(), timeout=5)
except TimeoutError:
LOG.warning(
"Segment stream for '%s' did not stop ffmpeg in time after the client disconnected; killing it.",
file,
)
LOG.error("ffmpeg process did not terminate in time after disconnect. Killing it.")
proc.kill()
try:
rc = await asyncio.wait_for(proc.wait(), timeout=5)
@ -227,7 +215,7 @@ class Segments:
try:
rc = await asyncio.wait_for(proc.wait(), timeout=30)
except TimeoutError:
LOG.warning("Segment stream for '%s' did not stop ffmpeg in time; killing the process.", file)
LOG.error("ffmpeg process wait timed out. Killing it.")
proc.kill()
try:
rc = await asyncio.wait_for(proc.wait(), timeout=5)
@ -241,13 +229,11 @@ class Segments:
return (wrote_any, rc, client_disconnected_local, stderr_buf.decode("utf-8", errors="ignore").strip())
async def stream(self, file: Path, resp: web.StreamResponse) -> None:
codec: str = self.vcodec
if Segments._cache_initialized and Segments._cached_vcodec:
codec = Segments._cached_vcodec
codec: str = Segments._cached_vcodec if Segments._cache_initialized else self.vcodec
if not codec or codec not in SUPPORTED_CODECS:
codec: str = select_encoder(self.vcodec or "")
LOG.debug("Selected video codec '%s' for segment streaming.", codec, extra={"codec": codec})
LOG.debug(f"Selected video codec '{codec}' for segment streaming.")
if Segments._cached_vcodec and Segments._cache_initialized:
codecs: list[str] = [Segments._cached_vcodec]
@ -273,14 +259,7 @@ class Segments:
if 0 != rc:
err: str = stderr_text[:500] if stderr_text else "no error output"
LOG.warning(
"Retrying segment %s for '%s' because hardware encoder '%s' failed: %s.",
self.index,
file,
s_codec,
err,
extra={"ffmpeg_args": ffmpeg_args, "returncode": rc, "stderr": err, "codec": s_codec},
)
LOG.warning(f"transcoding has failed (cmd={ffmpeg_args}) (rc={rc}): {err}. Trying fallbacks.")
self.attempted.add(s_codec)
finally:
if stream_input.is_symlink():

View file

@ -1,45 +1,14 @@
from __future__ import annotations
from dataclasses import dataclass
import logging
from pathlib import Path
from typing import Any
import anyio
import pysubs2
from pysubs2.formats.substation import SubstationFormat
from pysubs2.time import ms_to_times
from app.library.log import get_logger
from app.library.Utils import ALLOWED_SUBS_EXTENSIONS, get_file_sidecar
from app.library.Utils import ALLOWED_SUBS_EXTENSIONS
LOG = get_logger()
SOURCE_FORMATS: tuple[str, ...] = ("vtt", "srt", "ass")
DELIVERY_FORMATS: dict[str, str] = {
"vtt": "vtt",
"srt": "vtt",
"ass": "ass",
}
RENDERERS: dict[str, str] = {
"vtt": "native",
"srt": "native",
"ass": "assjs",
}
MEDIA_TYPES: dict[str, str] = {
"vtt": "text/vtt; charset=UTF-8",
"ass": "text/x-ssa; charset=UTF-8",
}
TEXT_ENCODINGS: tuple[str, ...] = ("utf-8-sig", "utf-16", "utf-16-le", "utf-16-be", "cp1252")
@dataclass(frozen=True, slots=True)
class SubtitleTrack:
file: Path
lang: str
name: str
source_format: str
delivery_format: str
renderer: str
LOG: logging.Logger = logging.getLogger("player.subtitle")
def ms_to_timestamp(ms: int) -> str:
@ -49,51 +18,18 @@ def ms_to_timestamp(ms: int) -> str:
return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}"
_substation_format: Any = SubstationFormat
_substation_format.ms_to_timestamp = ms_to_timestamp
SubstationFormat.ms_to_timestamp = ms_to_timestamp
class Subtitle:
@staticmethod
def normalize_format(source_format: str) -> str | None:
fmt = source_format.strip().lower().removeprefix(".")
if fmt not in SOURCE_FORMATS:
return None
return fmt
async def read_text(self, file: Path) -> str:
async with await anyio.open_file(file, "rb") as f:
subtitle_bytes = await f.read()
return self.decode_bytes(subtitle_bytes)
@staticmethod
def decode_bytes(subtitle_bytes: bytes) -> str:
for encoding in TEXT_ENCODINGS:
try:
return subtitle_bytes.decode(encoding)
except UnicodeDecodeError:
continue
return subtitle_bytes.decode("utf-8", errors="replace")
def media_type(self, file: Path) -> str:
fmt = self.normalize_format(file.suffix)
if fmt is None:
msg = f"File '{file}' subtitle type is not supported."
raise Exception(msg)
return MEDIA_TYPES[DELIVERY_FORMATS[fmt]]
async def make(self, file: Path) -> str:
fmt = self.normalize_format(file.suffix)
if fmt is None or file.suffix not in ALLOWED_SUBS_EXTENSIONS:
if file.suffix not in ALLOWED_SUBS_EXTENSIONS:
msg: str = f"File '{file}' subtitle type is not supported."
raise Exception(msg)
if fmt == "vtt":
return await self.read_text(file)
if file.suffix == ".vtt":
async with await anyio.open_file(file) as f:
return await f.read()
subs: pysubs2.SSAFile = pysubs2.load(path=str(file))
@ -111,45 +47,3 @@ class Subtitle:
pass
return subs.to_string("vtt")
async def make_delivery(self, file: Path) -> tuple[str, str]:
fmt = self.normalize_format(file.suffix)
if fmt is None or file.suffix not in ALLOWED_SUBS_EXTENSIONS:
msg: str = f"File '{file}' subtitle type is not supported."
raise Exception(msg)
if fmt == "ass":
return await self.read_text(file), self.media_type(file)
return await self.make(file), self.media_type(file)
def get_subtitle_tracks(file: Path) -> list[SubtitleTrack]:
sidecars = get_file_sidecar(file).get("subtitle", [])
indexed_tracks: list[tuple[int, SubtitleTrack]] = []
for index, item in enumerate(sidecars):
track_file = item.get("file")
if not isinstance(track_file, Path):
continue
fmt = Subtitle.normalize_format(track_file.suffix)
if fmt is None:
continue
indexed_tracks.append(
(
index,
SubtitleTrack(
file=track_file,
lang=str(item.get("lang") or "und"),
name=str(item.get("name") or track_file.name),
source_format=fmt,
delivery_format=DELIVERY_FORMATS[fmt],
renderer=RENDERERS[fmt],
),
)
)
indexed_tracks.sort(key=lambda item: (SOURCE_FORMATS.index(item[1].source_format), item[0]))
return [track for _, track in indexed_tracks]

View file

@ -1,284 +0,0 @@
from __future__ import annotations
import asyncio
import os
import subprocess
from pathlib import Path
from app.features.streaming.library.ffprobe import ffprobe
from app.library.cache import Cache
from app.library.config import Config
from app.library.log import get_logger
from app.library.Utils import FILES_TYPE, get_file_sidecar
LOG = get_logger()
IMAGE_TYPES: tuple[str, ...] = (".jpg", ".jpeg", ".png", ".webp")
FOLDER_IMAGE_ORDER: tuple[str, ...] = ("thumbnail", "poster", "artwork", "cover", "fanart")
THUMBNAIL_SAMPLE_FRAMES = 200
THUMBNAIL_DEFAULT_SEEK_SECONDS = 3.0
THUMBNAIL_MIN_SEEK_SECONDS = 1.0
THUMBNAIL_MAX_SEEK_SECONDS = 15.0
THUMBNAIL_END_MARGIN_SECONDS = 0.1
THUMBNAIL_MISS_TTL = 3600.0
_LOCK = asyncio.Lock()
_IN_PROCESS: dict[str, asyncio.Task[Path | None]] = {}
_SEM: asyncio.Semaphore | None = None
_SEM_LIMIT: int | None = None
def _get_semaphore() -> asyncio.Semaphore:
global _SEM, _SEM_LIMIT # noqa: PLW0603
limit: int = max(1, int(Config.get_instance().thumb_concurrency))
if _SEM is None or _SEM_LIMIT != limit:
_SEM = asyncio.Semaphore(limit)
_SEM_LIMIT = limit
LOG.info("Configured thumbnail generation to run %s job(s) at a time.", limit, extra={"limit": limit})
return _SEM
def _is_same_stem_image(media_file: Path, image_file: Path) -> bool:
if image_file.parent != media_file.parent:
return False
for image_type in FILES_TYPE:
if "image" != image_type.get("type"):
continue
rx = image_type.get("rx")
if rx and rx.search(image_file.name):
break
else:
return False
return image_file.stem == media_file.stem
def pick_local_thumb(media_file: Path) -> Path | None:
sidecar: dict[str, list[dict[str, str]]] = get_file_sidecar(media_file)
images: list[dict[str, str]] = sidecar.get("image", [])
local_images: list[Path] = [Path(item["file"]) for item in images if isinstance(item.get("file"), Path)]
for image_file in local_images:
if image_file.suffix.lower() in IMAGE_TYPES and _is_same_stem_image(media_file, image_file):
return image_file
by_name: dict[str, Path] = {
image_file.stem.lower(): image_file for image_file in local_images if image_file.suffix.lower() in IMAGE_TYPES
}
for name in FOLDER_IMAGE_ORDER:
if image_file := by_name.get(name):
return image_file
return None
def _seek_seconds(ff_info) -> float | None:
duration: float = 0.0
try:
duration = float(ff_info.metadata.get("duration") or 0.0)
except (TypeError, ValueError):
duration = 0.0
if duration <= 0:
return THUMBNAIL_DEFAULT_SEEK_SECONDS
if duration <= THUMBNAIL_END_MARGIN_SECONDS:
return None
return min(
max(duration * 0.10, THUMBNAIL_MIN_SEEK_SECONDS),
THUMBNAIL_MAX_SEEK_SECONDS,
duration - THUMBNAIL_END_MARGIN_SECONDS,
)
def _build_ffmpeg_args(media_file: Path, output_file: Path, *, seek_seconds: float | None) -> list[str]:
args: list[str] = [
"ffmpeg",
"-nostdin",
"-hide_banner",
"-loglevel",
"error",
"-y",
]
if seek_seconds is not None and seek_seconds > 0:
args.extend(["-ss", f"{seek_seconds:.3f}"])
args.extend(
[
"-i",
str(media_file),
"-vf",
f"thumbnail={THUMBNAIL_SAMPLE_FRAMES},scale=1280:-1:force_original_aspect_ratio=decrease",
"-frames:v",
"1",
"-q:v",
"3",
"-an",
str(output_file),
]
)
return args
async def _run_ffmpeg(media_file: Path, output_file: Path) -> Path | None:
ff_info = await ffprobe(media_file)
if not ff_info.has_video():
LOG.debug(
"Skipping thumbnail generation for '%s' because no video stream exists.",
media_file,
extra={"media_file": str(media_file)},
)
return None
output_file.parent.mkdir(parents=True, exist_ok=True)
temp_file = output_file.with_suffix(".tmp.jpg")
if temp_file.exists():
temp_file.unlink(missing_ok=True)
seek_seconds: float | None = _seek_seconds(ff_info)
attempts: list[float | None] = [seek_seconds]
if seek_seconds is not None and seek_seconds > 0:
attempts.append(None)
sem = _get_semaphore()
if sem.locked():
limit = _SEM_LIMIT or 1
LOG.debug(
"Waiting for a thumbnail generation slot for '%s' (limit=%s).",
media_file,
limit,
extra={"media_file": str(media_file), "limit": limit},
)
async with sem:
last_error: str = "ffmpeg produced an empty thumbnail file"
for idx, attempt_seek in enumerate(attempts, start=1):
temp_file.unlink(missing_ok=True)
args: list[str] = _build_ffmpeg_args(media_file, temp_file, seek_seconds=attempt_seek)
LOG.debug(
"Generating thumbnail for '%s'. attempt=%s/%s seek=%s",
media_file,
idx,
len(attempts),
"none" if attempt_seek is None else f"{attempt_seek:.3f}s",
extra={
"media_file": str(media_file),
"attempt": idx,
"attempt_count": len(attempts),
"seek": "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
},
)
try:
proc = await asyncio.create_subprocess_exec(
*args,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
)
except FileNotFoundError as exc:
msg = "ffmpeg not found."
raise OSError(msg) from exc
stdout, stderr = await proc.communicate()
if 0 == proc.returncode and temp_file.exists() and temp_file.stat().st_size > 0:
temp_file.replace(output_file)
LOG.info(
"Generated thumbnail '%s' for '%s'.",
output_file,
media_file,
extra={"media_file": str(media_file), "output_file": str(output_file)},
)
return output_file
if 0 != proc.returncode:
last_error: str = (
f"ffmpeg thumbnail generation failed (rc={proc.returncode}).\n"
f"stdout:\n{stdout.decode('utf-8', errors='replace')}\n"
f"stderr:\n{stderr.decode('utf-8', errors='replace')}"
)
else:
last_error: str = "ffmpeg produced an empty thumbnail file"
LOG.debug(
"Thumbnail generation attempt failed for '%s'. seek=%s",
media_file,
"none" if attempt_seek is None else f"{attempt_seek:.3f}s",
extra={
"media_file": str(media_file),
"seek": "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
},
)
temp_file.unlink(missing_ok=True)
raise OSError(last_error)
async def ensure_thumb(media_file: Path, cache_root: Path, item_id: str | None = None) -> Path | None:
config: Config = Config.get_instance()
cache: Cache = Cache.get_instance()
if bool(config.thumb_generate) is not True:
return None
sidecar: bool = bool(config.thumb_sidecar)
if not sidecar and not item_id:
msg = "item_id is required when thumbnail sidecar is disabled."
raise ValueError(msg)
thumb_id: str = str(media_file.resolve()) if sidecar else str(item_id)
cache_file: Path = media_file.with_name(f"{media_file.name}.jpg") if sidecar else cache_root / f"{item_id}.jpg"
miss_key: str = f"thumbnail-miss:{thumb_id}"
if cache_file.exists() and cache_file.stat().st_size > 0:
cache.delete(miss_key)
return cache_file
if cache.has(miss_key):
return None
async with _LOCK:
if cache_file.exists() and cache_file.stat().st_size > 0:
cache.delete(miss_key)
return cache_file
if cache.has(miss_key):
return None
task = _IN_PROCESS.get(thumb_id)
if task is not None and not task.done():
LOG.debug("Waiting for thumbnail generation for '%s'.", media_file, extra={"media_file": str(media_file)})
else:
task = asyncio.create_task(_run_ffmpeg(media_file, cache_file), name=f"thumb-{item_id or media_file.stem}")
_IN_PROCESS[thumb_id] = task
LOG.debug(
"Starting thumbnail generation for '%s' -> '%s'.",
media_file,
cache_file,
extra={"media_file": str(media_file), "cache_file": str(cache_file)},
)
try:
result: Path | None = await task
if result is None:
cache.set(miss_key, value=True, ttl=THUMBNAIL_MISS_TTL)
else:
cache.delete(miss_key)
return result
except OSError:
cache.set(miss_key, value=True, ttl=THUMBNAIL_MISS_TTL)
raise
finally:
async with _LOCK:
current = _IN_PROCESS.get(thumb_id)
if current is task and task.done():
_IN_PROCESS.pop(thumb_id, None)

View file

@ -1,22 +1,21 @@
import logging
import time
from datetime import UTC, datetime
from pathlib import Path
from aiohttp import web
from aiohttp.web import Request, Response
from aiohttp.web_response import StreamResponse
from app.features.streaming.library.m3u8 import M3u8
from app.features.streaming.library.playlist import Playlist
from app.features.streaming.library.segments import Segments
from app.features.streaming.library.subtitle import Subtitle, get_subtitle_tracks
from app.features.streaming.library.subtitle import Subtitle
from app.features.streaming.types import StreamingError
from app.library.config import Config
from app.library.log import get_logger
from app.library.router import route
from app.library.Utils import get_file
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("streaming")
@route("GET", "api/player/playlist/{file:.*}.m3u8", "playlist_create")
@ -33,7 +32,7 @@ async def playlist_create(request: Request, config: Config, app: web.Application
Response: The response object.
"""
file: str | None = request.match_info.get("file")
file: str = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@ -84,8 +83,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
Response: The response object.
"""
file: str | None = request.match_info.get("file")
mode: str | None = request.match_info.get("mode")
file: str = request.match_info.get("file")
mode: str = request.match_info.get("mode")
if mode not in ["video", "subtitle"]:
return web.json_response(
@ -95,14 +94,13 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
duration: float | None = None
duration_arg = request.query.get("duration", None)
duration = request.query.get("duration", None)
if "subtitle" in mode:
if not duration_arg:
if not duration:
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
duration = float(duration_arg)
duration = float(duration)
base_path: str = config.base_path.rstrip("/")
@ -126,19 +124,11 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
if "subtitle" in mode:
if duration is None:
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
text = await cls.make_subtitle(file=realFile, duration=duration)
else:
text = await cls.make_stream(file=realFile)
except StreamingError as e:
LOG.exception(
"Failed to create %s streaming playlist for '%s': %s.",
mode,
file,
e,
extra={"route": "streaming.playlist", "file_path": file, "mode": mode, "exception_type": type(e).__name__},
)
LOG.exception(e)
return web.json_response(data={"error": str(e)}, status=web.HTTPNotFound.status_code)
return web.Response(
@ -153,7 +143,7 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
@route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream")
async def segments_stream(request: Request, config: Config, app: web.Application) -> StreamResponse:
async def segments_stream(request: Request, config: Config, app: web.Application) -> Response:
"""
Get the segments.
@ -166,9 +156,9 @@ async def segments_stream(request: Request, config: Config, app: web.Application
Response: The response object.
"""
file: str | None = request.match_info.get("file")
segment: str | None = request.match_info.get("segment")
sd: str | None = request.query.get("sd")
file: str = request.match_info.get("file")
segment: int = request.match_info.get("segment")
sd: int = request.query.get("sd")
vc: int = int(request.query.get("vc", 0))
ac: int = int(request.query.get("ac", 0))
@ -245,7 +235,7 @@ async def subtitles_get(request: Request, config: Config, app: web.Application)
Response: The response object.
"""
file: str | None = request.match_info.get("file")
file: str = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@ -287,134 +277,3 @@ async def subtitles_get(request: Request, config: Config, app: web.Application)
},
status=web.HTTPOk.status_code,
)
@route("GET", "api/player/subtitles/manifest/{file:.*}", "subtitles_manifest_get")
async def subtitles_manifest_get(request: Request, config: Config, app: web.Application) -> Response:
"""
Get subtitle track metadata for a media file.
Args:
request (Request): The request object.
config (Config): The configuration instance.
app (web.Application): The aiohttp application instance.
Returns:
Response: The response object.
"""
file: str | None = request.match_info.get("file")
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
realFile, status = get_file(download_path=config.download_path, file=file)
if web.HTTPFound.status_code == status:
return Response(
status=status,
headers={
"Location": str(
app.router["subtitles_manifest_get"].url_for(
file=str(realFile).replace(config.download_path, "").strip("/")
)
),
},
)
if web.HTTPNotFound.status_code == status:
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
tracks = [
{
"lang": track.lang,
"name": track.name,
"source_format": track.source_format,
"delivery_format": track.delivery_format,
"renderer": track.renderer,
"url": str(
app.router["subtitles_track_get"].url_for(
source_format=track.source_format,
file=str(track.file).replace(config.download_path, "").strip("/"),
)
),
}
for track in get_subtitle_tracks(realFile)
]
return web.json_response(data={"subtitles": tracks}, status=web.HTTPOk.status_code)
@route("GET", "api/player/subtitles/{source_format}/{file:.*}", "subtitles_track_get")
async def subtitles_track_get(request: Request, config: Config, app: web.Application) -> Response:
"""
Get a subtitle file using its preferred delivery format.
Args:
request (Request): The request object.
config (Config): The configuration instance.
app (web.Application): The aiohttp application instance.
Returns:
Response: The response object.
"""
file: str | None = request.match_info.get("file")
source_format: str | None = request.match_info.get("source_format")
if not file:
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
fmt = Subtitle.normalize_format(source_format or "")
if fmt is None:
return web.json_response(
data={"error": "Only vtt, srt, and ass subtitle formats are supported."},
status=web.HTTPBadRequest.status_code,
)
realFile, status = get_file(download_path=config.download_path, file=file)
if web.HTTPFound.status_code == status:
return Response(
status=status,
headers={
"Location": str(
app.router["subtitles_track_get"].url_for(
source_format=fmt,
file=str(realFile).replace(config.download_path, "").strip("/"),
)
),
},
)
if web.HTTPNotFound.status_code == status:
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
if Subtitle.normalize_format(realFile.suffix) != fmt:
return web.json_response(
data={"error": f"Subtitle file '{file}' does not match requested source format '{fmt}'."},
status=web.HTTPBadRequest.status_code,
)
mtime = realFile.stat().st_mtime
if request.if_modified_since and request.if_modified_since.timestamp() == mtime:
lastMod = time.strftime("%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple())
return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod})
body, content_type = await Subtitle().make_delivery(file=realFile)
return web.Response(
body=body,
headers={
"Content-Type": content_type,
"X-Accel-Buffering": "no",
"Access-Control-Allow-Origin": "*",
"Pragma": "public",
"Cache-Control": f"public, max-age={time.time() + 31536000}",
"Last-Modified": time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple()
),
"Expires": time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple()
),
},
status=web.HTTPOk.status_code,
)

View file

@ -84,11 +84,13 @@ class TestFFProbe:
# First call should execute the function
result1 = await ffprobe(str(self.test_file))
assert result1 is not None
assert isinstance(result1.metadata, dict)
first_call_count = call_count
# Second call with same argument should use cached result
result2 = await ffprobe(str(self.test_file))
assert result2 is not None
assert isinstance(result2.metadata, dict)
assert call_count == first_call_count, (
@ -117,7 +119,7 @@ class TestFFProbe:
# Test with Path object
result = await ffprobe(self.test_file)
assert result.metadata == {"duration": "10.0"}
assert result is not None
def test_ffprobe_result_properties(self):
"""Test FFProbeResult object properties."""

View file

@ -64,7 +64,7 @@ async def test_make_stream_basic_ok_codecs(tmp_path: Path, monkeypatch: pytest.M
@pytest.mark.asyncio
async def test_make_stream_transcode_flags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
async def test_make_stream_transcode_flags_and_remainder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
base = tmp_path / "dl"
base.mkdir()
media = base / "v.mp4"
@ -107,7 +107,7 @@ async def test_make_stream_transcode_flags(tmp_path: Path, monkeypatch: pytest.M
@pytest.mark.asyncio
async def test_make_stream_no_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
async def test_make_stream_raises_without_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
base = tmp_path / "dl"
base.mkdir()
media = base / "v.mp4"

View file

@ -99,7 +99,7 @@ async def test_make_playlist_with_subtitles(tmp_path: Path, monkeypatch: pytest.
@pytest.mark.asyncio
async def test_make_playlist_no_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
async def test_make_playlist_raises_without_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
base = tmp_path / "downloads"
base.mkdir()
media = base / "file.mp4"

View file

@ -4,7 +4,6 @@ from pathlib import Path
from typing import Any
import pytest
from aiohttp import web
from app.features.streaming.library.segments import Segments
from app.tests.helpers import get_test_system_temp_root
@ -149,18 +148,18 @@ class _FakeProc:
self.killed = True
class _FakeResp(web.StreamResponse):
def __init__(self, fail_with: BaseException | None = None) -> None:
class _FakeResp:
def __init__(self, fail_with: Exception | None = None) -> None:
self.data: bytearray = bytearray()
self.eof = False
self._exc = fail_with
async def write(self, data: bytes | bytearray | memoryview, *_args: Any, **_kwargs: Any) -> None:
async def write(self, data: bytes) -> None:
if self._exc:
raise self._exc
self.data.extend(data)
async def write_eof(self, *_args: Any, **_kwargs: Any) -> None:
async def write_eof(self) -> None:
self.eof = True
@ -173,7 +172,7 @@ class _FakeProcFail(_FakeProc):
@pytest.mark.asyncio
async def test_build_ffmpeg_args_no_dri(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
async def test_build_ffmpeg_args_no_dri_falls_back_to_software(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
media = tmp_path / "file.mp4"
media.write_bytes(b"data")
@ -220,7 +219,7 @@ async def test_build_ffmpeg_args_no_dri(tmp_path: Path, monkeypatch: pytest.Monk
@pytest.mark.asyncio
async def test_stream_gpu_fallback(
async def test_stream_gpu_failure_falls_back_to_software(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
async def fake_ffprobe(_file: Path):
@ -256,14 +255,14 @@ async def test_stream_gpu_fallback(
# Encourage GPU preference
seg.vcodec = "" # empty -> try GPUs first
resp = _FakeResp()
with caplog.at_level(logging.WARNING, logger="ytptube"):
with caplog.at_level(logging.WARNING, logger="player.segments"):
await seg.stream(tmp_path / "file.mp4", resp)
# Ensure fallback path streamed data
assert len(resp.data) > 0
# Ensure we logged the reason for GPU failure (message text may vary)
assert any(
("hardware encoder" in r.message.lower()) or ("transcoding has failed" in r.message) for r in caplog.records
("Hardware encoder failed" in r.message) or ("transcoding has failed" in r.message) for r in caplog.records
)
assert any("nvenc failure" in r.message for r in caplog.records)
# Verify second invocation switched codec to a safe fallback (software)

View file

@ -4,7 +4,7 @@ from unittest.mock import patch
import pytest
from app.features.streaming.library.subtitle import Subtitle, get_subtitle_tracks, ms_to_timestamp
from app.features.streaming.library.subtitle import Subtitle, ms_to_timestamp
class TestMsToTimestamp:
@ -24,12 +24,12 @@ class TestMsToTimestamp:
@pytest.mark.asyncio
async def test_make_unsupported_extension(tmp_path: Path) -> None:
file = tmp_path / "sub.txt"
file.write_text("not a subtitle")
srt = tmp_path / "sub.txt"
srt.write_text("not a subtitle")
subtitle = Subtitle()
sub = Subtitle()
with pytest.raises(Exception, match="subtitle type is not supported"):
await subtitle.make(file)
await sub.make(srt)
@pytest.mark.asyncio
@ -38,51 +38,11 @@ async def test_make_vtt_reads_file(tmp_path: Path) -> None:
content = "WEBVTT\n\n00:00:00.00 --> 00:00:01.00\nHello"
vtt.write_text(content)
subtitle = Subtitle()
out = await subtitle.make(vtt)
sub = Subtitle()
out = await sub.make(vtt)
assert out == content
@pytest.mark.asyncio
async def test_make_delivery_ass(tmp_path: Path) -> None:
ass = tmp_path / "file.ass"
content = "[Script Info]\nTitle: Demo\n"
ass.write_text(content, encoding="utf-8")
subtitle = Subtitle()
out, media_type = await subtitle.make_delivery(ass)
assert out == content
assert media_type == "text/x-ssa; charset=UTF-8"
def test_tracks_order(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
media = tmp_path / "video.mkv"
media.write_text("x", encoding="utf-8")
ass_file = tmp_path / "video.ass"
ass_file.write_text("ass", encoding="utf-8")
vtt_file = tmp_path / "video.vtt"
vtt_file.write_text("WEBVTT\n\n", encoding="utf-8")
srt_file = tmp_path / "video.en.srt"
srt_file.write_text("1\n00:00:00,000 --> 00:00:01,000\nHello\n", encoding="utf-8")
monkeypatch.setattr(
"app.features.streaming.library.subtitle.get_file_sidecar",
lambda _file: {
"subtitle": [
{"file": ass_file, "lang": "en", "name": "ASS"},
{"file": srt_file, "lang": "en", "name": "SRT"},
{"file": vtt_file, "lang": "en", "name": "VTT"},
]
},
)
tracks = get_subtitle_tracks(media)
assert [track.source_format for track in tracks] == ["vtt", "srt", "ass"]
assert [track.delivery_format for track in tracks] == ["vtt", "vtt", "ass"]
assert [track.renderer for track in tracks] == ["native", "native", "assjs"]
class _DummySubs:
def __init__(self, events):
self.events = events
@ -101,13 +61,13 @@ async def test_make_no_events_raises(tmp_path: Path) -> None:
with patch("app.features.streaming.library.subtitle.pysubs2.load") as mock_load:
mock_load.return_value = _DummySubs(events=[])
subtitle = Subtitle()
sub = Subtitle()
with pytest.raises(Exception, match="No subtitle events were found"):
await subtitle.make(srt)
await sub.make(srt)
@pytest.mark.asyncio
async def test_make_single_vtt(tmp_path: Path) -> None:
async def test_make_single_event_returns_vtt(tmp_path: Path) -> None:
srt = tmp_path / "sub.ass"
srt.write_text("dummy")
@ -115,14 +75,14 @@ async def test_make_single_vtt(tmp_path: Path) -> None:
d = _DummySubs(events=[single])
with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
subtitle = Subtitle()
out = await subtitle.make(srt)
sub = Subtitle()
out = await sub.make(srt)
assert out == "OUT"
assert d.snapshot == [1000], "Snapshot should contain the single event"
@pytest.mark.asyncio
async def test_make_two_events_same_end(tmp_path: Path) -> None:
async def test_make_two_events_pop_first_when_ends_equal(tmp_path: Path) -> None:
srt = tmp_path / "sub.srt"
srt.write_text("dummy")
@ -131,14 +91,14 @@ async def test_make_two_events_same_end(tmp_path: Path) -> None:
d = _DummySubs(events=[e1, e2])
with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
subtitle = Subtitle()
out = await subtitle.make(srt)
sub = Subtitle()
out = await sub.make(srt)
assert out == "OUT"
assert d.snapshot == [5000], "Since ends are equal, first should be popped => only last remains"
@pytest.mark.asyncio
async def test_make_two_events_keep_both(tmp_path: Path) -> None:
async def test_make_two_events_no_pop_when_different(tmp_path: Path) -> None:
srt = tmp_path / "sub.srt"
srt.write_text("dummy")
@ -147,7 +107,7 @@ async def test_make_two_events_keep_both(tmp_path: Path) -> None:
d = _DummySubs(events=[e1, e2])
with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
subtitle = Subtitle()
out = await subtitle.make(srt)
sub = Subtitle()
out = await sub.make(srt)
assert out == "OUT"
assert d.snapshot == [5000, 6000], "Both remain since ends differ"

View file

@ -1,132 +0,0 @@
from pathlib import Path
from types import SimpleNamespace
import pytest
from aiohttp import web
from aiohttp.test_utils import make_mocked_request
from app.features.streaming import router
from app.library.config import Config
class _DummyRoute:
def __init__(self, pattern: str) -> None:
self.pattern = pattern
def url_for(self, **params: str) -> str:
path = self.pattern
for key, value in params.items():
path = path.replace(f"{{{key}:.*}}", value)
path = path.replace(f"{{{key}}}", value)
return path
class _DummyRouter(dict[str, _DummyRoute]):
def __init__(self) -> None:
super().__init__(
{
"subtitles_manifest_get": _DummyRoute("/api/player/subtitles/manifest/{file:.*}"),
"subtitles_track_get": _DummyRoute("/api/player/subtitles/{source_format}/{file:.*}"),
"subtitles_get": _DummyRoute("/api/player/subtitle/{file:.*}.vtt"),
}
)
def _make_request(path: str, *, match_info: dict[str, str]) -> web.Request:
return make_mocked_request("GET", path, app=SimpleNamespace(router=_DummyRouter()), match_info=match_info)
@pytest.mark.asyncio
async def test_subtitles_manifest_order(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
config = Config.get_instance()
config.download_path = str(tmp_path)
media = tmp_path / "video.mp4"
media.write_text("x", encoding="utf-8")
monkeypatch.setattr(
"app.features.streaming.router.get_file",
lambda **_kwargs: (media, web.HTTPOk.status_code),
)
monkeypatch.setattr(
"app.features.streaming.router.get_subtitle_tracks",
lambda _file: [
SimpleNamespace(
lang="en",
name="English VTT",
source_format="vtt",
delivery_format="vtt",
renderer="native",
file=tmp_path / "video.vtt",
),
SimpleNamespace(
lang="en",
name="English ASS",
source_format="ass",
delivery_format="ass",
renderer="assjs",
file=tmp_path / "video.ass",
),
],
)
req = _make_request(
"/api/player/subtitles/manifest/video.mp4",
match_info={"file": "video.mp4"},
)
response = await router.subtitles_manifest_get(req, config, req.app)
assert response.status == web.HTTPOk.status_code
body = response.text
assert '"source_format": "vtt"' in body
assert '"renderer": "native"' in body
assert '"source_format": "ass"' in body
assert '"renderer": "assjs"' in body
assert body.index('"source_format": "vtt"') < body.index('"source_format": "ass"')
@pytest.mark.asyncio
async def test_subtitles_track_get_ass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
config = Config.get_instance()
config.download_path = str(tmp_path)
subtitle = tmp_path / "video.ass"
subtitle.write_text("[Script Info]\nTitle: Demo\n", encoding="utf-8")
monkeypatch.setattr(
"app.features.streaming.router.get_file",
lambda **_kwargs: (subtitle, web.HTTPOk.status_code),
)
req = _make_request(
"/api/player/subtitles/ass/video.ass",
match_info={"source_format": "ass", "file": "video.ass"},
)
response = await router.subtitles_track_get(req, config, req.app)
assert response.status == web.HTTPOk.status_code
assert response.text == "[Script Info]\nTitle: Demo\n"
assert response.headers["Content-Type"] == "text/x-ssa; charset=UTF-8"
@pytest.mark.asyncio
async def test_subtitles_track_bad_format(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
config = Config.get_instance()
config.download_path = str(tmp_path)
subtitle = tmp_path / "video.ass"
subtitle.write_text("[Script Info]\nTitle: Demo\n", encoding="utf-8")
monkeypatch.setattr(
"app.features.streaming.router.get_file",
lambda **_kwargs: (subtitle, web.HTTPOk.status_code),
)
req = _make_request(
"/api/player/subtitles/vtt/video.ass",
match_info={"source_format": "vtt", "file": "video.ass"},
)
response = await router.subtitles_track_get(req, config, req.app)
assert response.status == web.HTTPBadRequest.status_code
assert b"does not match requested source format" in response.body

View file

@ -1,358 +0,0 @@
from __future__ import annotations
import asyncio
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock
import pytest
from app.tests.helpers import temporary_test_dir
@pytest.mark.asyncio
async def test_pick_same_stem() -> None:
from app.features.streaming.library.thumbnail import pick_local_thumb
with temporary_test_dir("thumb-local") as temp_dir:
media = temp_dir / "video.mp4"
poster = temp_dir / "poster.jpg"
thumb = temp_dir / "video.jpg"
media.write_text("video")
poster.write_text("poster")
thumb.write_text("thumb")
assert pick_local_thumb(media) == thumb
@pytest.mark.asyncio
async def test_singleflight(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-singleflight") as temp_dir:
media = temp_dir / "video.mp4"
cache_root = temp_dir / "cache"
media.write_text("video")
item_id = "item-1"
calls = {"count": 0}
async def fake_run_ffmpeg(_file: Path, output_file: Path) -> Path:
calls["count"] += 1
await asyncio.sleep(0.01)
output_file.parent.mkdir(parents=True, exist_ok=True)
output_file.write_text("image")
return output_file
monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
first, second = await asyncio.gather(
thumbnail.ensure_thumb(media, cache_root, item_id=item_id),
thumbnail.ensure_thumb(media, cache_root, item_id=item_id),
)
assert first == second
assert first is not None
assert calls["count"] == 1
assert thumbnail._IN_PROCESS == {}
@pytest.mark.asyncio
async def test_cache_hit(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-cache") as temp_dir:
media = temp_dir / "video.mp4"
cache_root = temp_dir / "cache"
media.write_text("video")
item_id = "item-1"
cache_file = cache_root / f"{item_id}.jpg"
cache_file.parent.mkdir(parents=True, exist_ok=True)
cache_file.write_text("image")
async def fake_run_ffmpeg(_file: Path, _output_file: Path) -> Path:
raise AssertionError("ffmpeg should not run when cache exists")
monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
result = await thumbnail.ensure_thumb(media, cache_root, item_id=item_id)
assert result == cache_file
@pytest.mark.asyncio
async def test_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-disabled") as temp_dir:
media = temp_dir / "video.mp4"
media.write_text("video")
monkeypatch.setattr(
thumbnail.Config,
"get_instance",
staticmethod(lambda: type("Cfg", (), {"thumb_generate": False, "thumb_sidecar": False})()),
)
result = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id="item-1")
assert result is None
@pytest.mark.asyncio
async def test_sidecar_mode(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail._SEM = None
thumbnail._SEM_LIMIT = None
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-sidecar") as temp_dir:
media = temp_dir / "video.mp4"
media.write_text("video")
sidecar = media.with_name(f"{media.name}.jpg")
monkeypatch.setattr(
thumbnail.Config,
"get_instance",
staticmethod(
lambda: type(
"Cfg",
(),
{"thumb_generate": True, "thumb_sidecar": True, "thumb_concurrency": 1},
)()
),
)
async def fake_run_ffmpeg(_file: Path, out_file: Path) -> Path:
out_file.write_text("image")
return out_file
monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
result = await thumbnail.ensure_thumb(media, temp_dir / "cache")
assert result == sidecar
assert sidecar.exists()
@pytest.mark.asyncio
async def test_miss_cache(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-miss") as temp_dir:
media = temp_dir / "video.mp4"
media.write_text("video")
item_id = "item-1"
monkeypatch.setattr(
thumbnail.Config,
"get_instance",
staticmethod(
lambda: type(
"Cfg",
(),
{"thumb_generate": True, "thumb_sidecar": False, "thumb_concurrency": 1},
)()
),
)
calls = {"count": 0}
async def fake_run_ffmpeg(_file: Path, _out_file: Path) -> Path | None:
calls["count"] += 1
return None
monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
first = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id=item_id)
second = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id=item_id)
assert first is None
assert second is None
assert calls["count"] == 1
def test_sem_limit(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._SEM = None
thumbnail._SEM_LIMIT = None
monkeypatch.setattr(
thumbnail.Config,
"get_instance",
staticmethod(
lambda: type("Cfg", (), {"thumb_concurrency": 3, "thumb_generate": True, "thumb_sidecar": False})()
),
)
sem = thumbnail._get_semaphore()
assert sem._value == 3
assert thumbnail._SEM_LIMIT == 3
def test_seek_bounds() -> None:
from app.features.streaming.library.ffprobe import FFProbeResult
from app.features.streaming.library.thumbnail import _seek_seconds
ff_info = FFProbeResult()
ff_info.metadata = {"duration": "120.0"}
assert _seek_seconds(ff_info) == 12.0
ff_info.metadata = {"duration": "2.0"}
assert _seek_seconds(ff_info) == 1.0
ff_info.metadata = {"duration": "0.05"}
assert _seek_seconds(ff_info) is None
ff_info.metadata = {}
assert _seek_seconds(ff_info) == 3.0
@pytest.mark.asyncio
async def test_cache_name_uses_item_id(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
thumbnail._IN_PROCESS.clear()
thumbnail.Cache.get_instance().clear()
with temporary_test_dir("thumb-id-name") as temp_dir:
media = temp_dir / "video.mp4"
cache_root = temp_dir / "cache"
media.write_text("video")
async def fake_run_ffmpeg(_file: Path, output_file: Path) -> Path:
output_file.parent.mkdir(parents=True, exist_ok=True)
output_file.write_text("image")
return output_file
monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
result = await thumbnail.ensure_thumb(media, cache_root, item_id="item-1")
assert result == cache_root / "item-1.jpg"
@pytest.mark.asyncio
async def test_retry_no_seek(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library.ffprobe import FFProbeResult
from app.features.streaming.library import thumbnail
with temporary_test_dir("thumb-retry") as temp_dir:
media = temp_dir / "video.mp4"
output = temp_dir / ".jpg"
media.write_text("video")
ff_info: Any = FFProbeResult()
ff_info.metadata = {"duration": "120.0"}
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))
calls: list[list[str]] = []
class DummyProc:
def __init__(self, attempt: int, out_path: Path) -> None:
self.returncode = 1 if attempt == 1 else 0
self._out_path = out_path
async def communicate(self) -> tuple[bytes, bytes]:
if self.returncode == 0:
self._out_path.write_text("image")
return b"", b""
return b"", b"seek failed"
async def fake_create_subprocess_exec(*args, **kwargs):
del kwargs
calls.append([str(arg) for arg in args])
return DummyProc(len(calls), Path(str(args[-1])))
monkeypatch.setattr(thumbnail.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
result = await thumbnail._run_ffmpeg(media, output)
assert result == output
assert len(calls) == 2
assert "-ss" in calls[0]
assert calls[0][calls[0].index("-ss") + 1] == "12.000"
assert "-ss" not in calls[1]
assert calls[0][calls[0].index("-vf") + 1] == "thumbnail=200,scale=1280:-1:force_original_aspect_ratio=decrease"
assert calls[0][-1].endswith(".tmp.jpg")
@pytest.mark.asyncio
async def test_limit_wait(monkeypatch: pytest.MonkeyPatch) -> None:
from app.features.streaming.library import thumbnail
from app.features.streaming.library.ffprobe import FFProbeResult
thumbnail._SEM = None
thumbnail._SEM_LIMIT = None
thumbnail._IN_PROCESS.clear()
monkeypatch.setattr(
thumbnail.Config,
"get_instance",
staticmethod(
lambda: type("Cfg", (), {"thumb_concurrency": 1, "thumb_generate": True, "thumb_sidecar": False})()
),
)
with temporary_test_dir("thumb-limit") as temp_dir:
media1 = temp_dir / "video1.mp4"
media2 = temp_dir / "video2.mp4"
out1 = temp_dir / "out1.jpg"
out2 = temp_dir / "out2.jpg"
media1.write_text("video")
media2.write_text("video")
ff_info: Any = FFProbeResult()
ff_info.metadata = {"duration": "60.0"}
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))
active = {"count": 0, "max": 0}
class DummyProc:
def __init__(self, out_path: Path) -> None:
self.returncode = 0
self._out_path = out_path
async def communicate(self) -> tuple[bytes, bytes]:
active["count"] += 1
active["max"] = max(active["max"], active["count"])
await asyncio.sleep(0.01)
self._out_path.write_text("image")
active["count"] -= 1
return b"", b""
async def fake_create_subprocess_exec(*args, **kwargs):
del kwargs
return DummyProc(Path(str(args[-1])))
monkeypatch.setattr(thumbnail.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
first, second = await asyncio.gather(
thumbnail._run_ffmpeg(media1, out1),
thumbnail._run_ffmpeg(media2, out2),
)
assert first == out1
assert second == out2
assert active["max"] == 1

View file

@ -1,3 +1,4 @@
# flake8: noqa: ARG004
from typing import TYPE_CHECKING, Any
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult
@ -11,7 +12,6 @@ if TYPE_CHECKING:
class BaseHandler:
@staticmethod
async def can_handle(task: HandleTask) -> bool:
_ = task
return False
@staticmethod
@ -24,7 +24,6 @@ class BaseHandler:
@staticmethod
def parse(url: str) -> Any | None:
_ = url
return None
@staticmethod

View file

@ -6,12 +6,12 @@ import asyncio
import fnmatch
import hashlib
import json
import logging
import re
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin
import httpx
import jmespath
from parsel import Selector
@ -25,16 +25,16 @@ from app.features.ytdlp.utils import get_archive_id
from app.library.cache import Cache
from app.library.config import Config
from app.library.httpx_client import Globals, build_request_headers, get_async_client, resolve_curl_transport
from app.library.log import get_logger
from ._base_handler import BaseHandler
if TYPE_CHECKING:
from pathlib import Path
import httpx
from parsel.selector import SelectorList
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("handlers.generic")
CACHE: Cache = Cache()
@ -64,15 +64,12 @@ class GenericTaskHandler(BaseHandler):
from app.features.tasks.definitions.utils import model_to_schema
repo = TaskDefinitionsRepository.get_instance()
models = await repo.all()
models = await repo.list()
cls._definitions = [model_to_schema(model) for model in models]
return cls._definitions
except Exception as exc:
LOG.exception(
"Failed to load generic task definitions.",
extra={"error": str(exc), "exception_type": type(exc).__name__},
)
LOG.error(f"Failed to load task definitions from database: {exc}")
return []
@classmethod
@ -105,15 +102,7 @@ class GenericTaskHandler(BaseHandler):
if pattern_str and re.match(pattern_str, url):
return definition
except Exception as exc:
LOG.exception(
"Failed to match a generic task definition.",
extra={
"definition": definition.name,
"url": url,
"error": str(exc),
"exception_type": type(exc).__name__,
},
)
LOG.error(f"Error while matching definition '{definition.name}': {exc}")
return None
@ -131,18 +120,13 @@ class GenericTaskHandler(BaseHandler):
"""
definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
if definition:
LOG.debug(
"Task '%s' matched a generic task definition.",
task.name,
extra={"task_name": task.name, "url": task.url, "definition": definition.name},
)
LOG.debug(f"'{task.name}': Matched generic task definition '{definition.name}'.")
return True
return False
@staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004
definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
if not definition:
return TaskFailure(message="No generic task definition matched the provided URL.")
@ -150,34 +134,14 @@ class GenericTaskHandler(BaseHandler):
ytdlp_opts: dict[str, Any] = task.get_ytdlp_opts().get_all()
target_url: str = definition.definition.request.url or task.url
LOG.debug(
"Fetching content for task '%s'.",
task.name,
extra={
"task_name": task.name,
"definition": definition.name,
"url": target_url,
"engine": definition.definition.engine.type,
},
)
LOG.debug(f"{task.name!r}: Fetching '{target_url}' using engine '{definition.definition.engine.type}'.")
try:
body_text, json_data = await GenericTaskHandler._fetch_content(
url=target_url, definition=definition, ytdlp_opts=ytdlp_opts
)
except httpx.HTTPError as exc:
return TaskFailure(message="Failed to fetch target URL.", error=str(exc))
except Exception as exc:
LOG.exception(
"Failed to fetch content for task '%s'.",
task.name,
extra={
"task_name": task.name,
"definition": definition.name,
"url": target_url,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return TaskFailure(message="Failed to fetch target URL.", error=str(exc))
if "json" == definition.definition.response.type and json_data is None:
@ -215,9 +179,7 @@ class GenericTaskHandler(BaseHandler):
continue
else:
LOG.warning(
"Task '%s' could not generate a static archive ID. Fetching it with yt-dlp.",
task.name,
extra={"definition": definition.name, "task_name": task.name, "url": url},
f"[{definition.name}]: '{task.name}': Unable to generate static archive id for '{url}' in feed. Doing real request to fetch yt-dlp archive id."
)
(info, _) = await fetch_info(
@ -225,23 +187,18 @@ class GenericTaskHandler(BaseHandler):
url=url,
no_archive=True,
no_log=True,
budget_sleep=True,
)
if not info:
LOG.error(
"Task '%s' failed to extract info to generate an archive ID. Skipping item.",
task.name,
extra={"definition": definition.name, "task_name": task.name, "url": url},
f"[{definition.name}]: '{task.name}': Failed to extract info for URL '{url}' to generate archive ID. Skipping."
)
CACHE.set(cache_key, None)
continue
if not info.get("id") or not info.get("extractor_key"):
LOG.error(
"Task '%s' returned incomplete info while generating an archive ID. Skipping item.",
task.name,
extra={"definition": definition.name, "task_name": task.name, "url": url},
f"[{definition.name}]: '{task.name}': Incomplete info extracted for URL '{url}' to generate archive ID. Skipping."
)
CACHE.set(cache_key, None)
continue
@ -337,17 +294,7 @@ class GenericTaskHandler(BaseHandler):
try:
json_data: dict[str, Any] = response.json()
except Exception as exc:
LOG.exception(
"Task definition '%s' returned invalid JSON for '%s'.",
definition.name,
url,
extra={
"definition": definition.name,
"url": url,
"error": str(exc),
"exception_type": type(exc).__name__,
},
)
LOG.error(f"Failed to decode JSON response from '{url}': {exc}")
return response.text, None
return response.text, json_data
@ -377,11 +324,7 @@ class GenericTaskHandler(BaseHandler):
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
except ImportError as exc:
LOG.exception(
"Task definition '%s' requested Selenium, but Selenium is not installed.",
definition.name,
extra={"definition": definition.name, "error": str(exc), "exception_type": type(exc).__name__},
)
LOG.error(f"Selenium engine requested but selenium is not installed: {exc!s}.")
return (None, None)
options_map: dict[str, Any] = definition.definition.engine.options
@ -394,12 +337,7 @@ class GenericTaskHandler(BaseHandler):
browser: str = str(options_map.get("browser", "chrome")).lower()
if "chrome" != browser:
LOG.error(
"Task definition '%s' requested unsupported Selenium browser '%s'.",
definition.name,
browser,
extra={"definition": definition.name, "browser": browser},
)
LOG.error(f"Unsupported selenium browser '{browser}'. Only 'chrome' is supported.")
return (None, None)
arguments: list[str] | str = options_map.get("arguments", ["--headless", "--disable-gpu"])
@ -481,9 +419,7 @@ class GenericTaskHandler(BaseHandler):
link_values: list[str] = extracted.get("link", [])
if not link_values:
LOG.debug(
"Definition '%s' produced no link values.", definition.name, extra={"definition": definition.name}
)
LOG.debug(f"Definition '{definition.name}' produced no link values.")
return []
total_items: int = len(link_values)
@ -518,11 +454,7 @@ class GenericTaskHandler(BaseHandler):
base_url: str,
) -> list[dict[str, str]]:
if json_data is None:
LOG.debug(
"Definition '%s' expects JSON but no data was parsed.",
definition.name,
extra={"definition": definition.name},
)
LOG.debug(f"Definition '{definition.name}' expects JSON but no data was parsed.")
return []
if definition.definition.parse.get("items"):
@ -561,11 +493,7 @@ class GenericTaskHandler(BaseHandler):
container_type = container.get("type", "css")
container_selector = container.get("selector") or container.get("expression") or ""
if not container_selector:
LOG.error(
"Task definition '%s' is missing an item container selector.",
definition.name,
extra={"definition": definition.name},
)
LOG.error(f"Container missing selector/expression. Definition '{definition.name}'.")
return []
container_fields = container.get("fields", {})
@ -619,11 +547,7 @@ class GenericTaskHandler(BaseHandler):
container_fields = container.get("fields", {})
if "jsonpath" != container_type:
LOG.error(
"JSON response requires container selector type 'jsonpath'. Definition '%s'.",
definition.name,
extra={"definition": definition.name, "container_type": container_type},
)
LOG.error(f"JSON response requires container selector type 'jsonpath'. Definition '{definition.name}'.")
return []
nodes: Any = GenericTaskHandler._json_search(json_data, container_selector)
@ -679,16 +603,7 @@ class GenericTaskHandler(BaseHandler):
try:
pattern: re.Pattern[str] = re.compile(rule.expression, re.MULTILINE | re.DOTALL)
except re.error as exc:
LOG.exception(
"Invalid regex expression '%s'.",
rule.expression,
extra={
"field": field,
"expression": rule.expression,
"error": str(exc),
"exception_type": type(exc).__name__,
},
)
LOG.error(f"Invalid regex expression '{rule.expression}': {exc}")
return values
for match in pattern.finditer(target):
@ -699,12 +614,7 @@ class GenericTaskHandler(BaseHandler):
return values
LOG.error(
"Unsupported extraction type '%s' for JSON data in field '%s'.",
rule.type,
field,
extra={"field": field, "rule_type": rule.type},
)
LOG.error(f"Unsupported extraction type '{rule.type}' for JSON data in field '{field}'.")
return values
@staticmethod
@ -712,11 +622,7 @@ class GenericTaskHandler(BaseHandler):
try:
return jmespath.search(expression, data)
except Exception as exc:
LOG.exception(
"JSONPath search failed for expression '%s'.",
expression,
extra={"expression": expression, "error": str(exc), "exception_type": type(exc).__name__},
)
LOG.error(f"JSONPath search failed for expression '{expression}': {exc}")
return None
@staticmethod
@ -751,16 +657,7 @@ class GenericTaskHandler(BaseHandler):
try:
pattern: re.Pattern[str] = re.compile(rule.expression, re.MULTILINE | re.DOTALL)
except re.error as exc:
LOG.exception(
"Invalid regex expression '%s'.",
rule.expression,
extra={
"field": field,
"expression": rule.expression,
"error": str(exc),
"exception_type": type(exc).__name__,
},
)
LOG.error(f"Invalid regex expression '{rule.expression}': {exc}")
return values
for match in pattern.finditer(html):
@ -772,7 +669,7 @@ class GenericTaskHandler(BaseHandler):
return values
if "jsonpath" == rule.type:
LOG.error("Field '%s' uses 'jsonpath' on a non-JSON response.", field, extra={"field": field})
LOG.error("Extraction type 'jsonpath' is only valid for JSON responses.")
return values
selection: SelectorList[Selector] = (
@ -804,12 +701,7 @@ class GenericTaskHandler(BaseHandler):
try:
return match.group(attribute)
except (IndexError, KeyError):
LOG.debug(
"Regex group '%s' not found in pattern '%s'.",
attribute,
match.re.pattern,
extra={"attribute": attribute, "pattern": match.re.pattern},
)
LOG.debug(f"Regex group '{attribute}' not found in pattern '{match.re.pattern}'.")
return None
if match.groupdict():

View file

@ -1,23 +1,20 @@
import hashlib
import logging
import re
from typing import TYPE_CHECKING, Any
from xml.etree.ElementTree import Element
import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.extractor import fetch_info
from app.features.ytdlp.utils import get_archive_id
from app.library.cache import Cache
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
if TYPE_CHECKING:
from xml.etree.ElementTree import Element
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("handlers.rss")
CACHE: Cache = Cache()
@ -29,11 +26,7 @@ class RssGenericHandler(BaseHandler):
@staticmethod
async def can_handle(task: HandleTask) -> bool:
LOG.debug(
"Checking if task '%s' uses a parsable RSS feed.",
task.name,
extra={"task_name": task.name, "url": task.url},
)
LOG.debug(f"'{task.name}': Checking if task URL is parsable RSS feed: {task.url}")
return RssGenericHandler.parse(task.url) is not None
@staticmethod
@ -57,11 +50,7 @@ class RssGenericHandler(BaseHandler):
from defusedxml.ElementTree import fromstring
feed_url: str = parsed["url"]
LOG.debug(
"Fetching RSS/Atom feed for task '%s'.",
task.name,
extra={"task_name": task.name, "feed_url": feed_url},
)
LOG.debug(f"'{task.name}': Fetching RSS/Atom feed from {feed_url}")
response = await RssGenericHandler.request(url=feed_url, ytdlp_opts=params)
response.raise_for_status()
@ -82,12 +71,7 @@ class RssGenericHandler(BaseHandler):
# Try to parse as Atom feed first
entries = root.findall("atom:entry", ns)
if entries:
LOG.debug(
"'%s': Detected Atom feed format with %s entries",
task.name,
len(entries),
extra={"task_name": task.name, "feed_url": feed_url, "entry_count": len(entries)},
)
LOG.debug(f"'{task.name}': Detected Atom feed format with {len(entries)} entries")
for entry in entries:
link_elem: Element | None = entry.find("atom:link[@rel='alternate']", ns)
if link_elem is None:
@ -98,11 +82,7 @@ class RssGenericHandler(BaseHandler):
url = link_elem.get("href", "")
if not url:
LOG.warning(
"'%s': Atom entry missing URL. Skipping.",
task.name,
extra={"task_name": task.name, "feed_url": feed_url},
)
LOG.warning(f"'{task.name}': Atom entry missing URL. Skipping.")
continue
title_elem: Element | None = entry.find("atom:title", ns)
@ -116,12 +96,7 @@ class RssGenericHandler(BaseHandler):
else:
# Try to parse as RSS feed
rss_items = root.findall(".//item")
LOG.debug(
"'%s': Detected RSS feed format with %s items",
task.name,
len(rss_items),
extra={"task_name": task.name, "feed_url": feed_url, "entry_count": len(rss_items)},
)
LOG.debug(f"'{task.name}': Detected RSS feed format with {len(rss_items)} items")
for item in rss_items:
# Try different link element names (link, url, media:content)
@ -142,11 +117,7 @@ class RssGenericHandler(BaseHandler):
url = enclosure_elem.get("url", "")
if not url:
LOG.warning(
"'%s': RSS item missing URL. Skipping.",
task.name,
extra={"task_name": task.name, "feed_url": feed_url},
)
LOG.warning(f"'{task.name}': RSS item missing URL. Skipping.")
continue
title_elem = item.find("title")
@ -161,14 +132,12 @@ class RssGenericHandler(BaseHandler):
return feed_url, items, real_count
@staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
"""
Extract items from an RSS/Atom feed.
Args:
task (Task): The task containing the feed URL.
config (Config | None): Optional handler configuration.
Returns:
TaskResult | TaskFailure: Extraction result with parsed items or failure information.
@ -182,19 +151,8 @@ class RssGenericHandler(BaseHandler):
try:
feed_url, items, real_count = await RssGenericHandler._get(task, params, parsed)
except httpx.HTTPError as exc:
return TaskFailure(message="Failed to fetch RSS/Atom feed.", error=str(exc))
except Exception as exc:
LOG.exception(
"Failed to fetch RSS/Atom feed for task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return TaskFailure(message="Failed to fetch RSS/Atom feed.", error=str(exc))
task_items: list[TaskItem] = []
@ -214,17 +172,12 @@ class RssGenericHandler(BaseHandler):
if CACHE.has(cache_key):
archive_id = CACHE.get(cache_key)
if not archive_id:
LOG.debug(
"Task '%s' has a cached archive ID lookup failure. Skipping item.",
task.name,
extra={"task_name": task.name, "url": url},
)
LOG.debug(f"'{task.name}': Cached failure for URL '{url}'. Skipping.")
continue
else:
LOG.warning(
"Task '%s' could not generate a static archive ID. Fetching it with yt-dlp.",
task.name,
extra={"task_name": task.name, "url": url},
f"'{task.name}': Unable to generate static archive ID for '{url}' in feed. "
"Doing real request to fetch yt-dlp archive ID."
)
(info, _) = await fetch_info(
@ -232,23 +185,18 @@ class RssGenericHandler(BaseHandler):
url=url,
no_archive=True,
no_log=True,
budget_sleep=True,
)
if not info:
LOG.error(
"Task '%s' failed to extract info to generate an archive ID. Skipping item.",
task.name,
extra={"task_name": task.name, "url": url},
f"'{task.name}': Failed to extract info for URL '{url}' to generate archive ID. Skipping."
)
CACHE.set(cache_key, None)
continue
if not info.get("id") or not info.get("extractor_key"):
LOG.error(
"Task '%s' returned incomplete info while generating an archive ID. Skipping item.",
task.name,
extra={"task_name": task.name, "url": url},
f"'{task.name}': Incomplete info extracted for URL '{url}' to generate archive ID. Skipping."
)
CACHE.set(cache_key, None)
continue

View file

@ -1,15 +1,12 @@
import logging
import re
import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("handlers.tver")
class TverHandler(BaseHandler):
@ -25,11 +22,7 @@ class TverHandler(BaseHandler):
@staticmethod
async def can_handle(task: HandleTask) -> bool:
LOG.debug(
"Checking if task '%s' uses a parsable Tver series URL.",
task.name,
extra={"task_name": task.name, "url": task.url},
)
LOG.debug(f"Checking if task '{task.name}' is using parsable Tver series URL: {task.url}")
return TverHandler.parse(task.url) is not None
@staticmethod
@ -60,11 +53,7 @@ class TverHandler(BaseHandler):
return {"platform_uid": platform_uid, "platform_token": platform_token}
except Exception as exc:
LOG.warning(
"Failed to create Tver session.",
extra={"error": str(exc), "exception_type": type(exc).__name__},
exc_info=True,
)
LOG.warning(f"Failed to create tver session: {exc}")
return None
@staticmethod
@ -92,11 +81,7 @@ class TverHandler(BaseHandler):
feed_url = TverHandler.SERIES_API.format(id=series_id)
LOG.debug(
"Fetching Tver episodes for task '%s'.",
task.name,
extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
)
LOG.debug(f"Fetching '{task.name}' episodes from tver series {series_id}.")
response = await TverHandler.request(
url=feed_url,
@ -118,11 +103,7 @@ class TverHandler(BaseHandler):
try:
contents = data.get("result", {}).get("contents", [])
if not contents:
LOG.warning(
"No contents found in Tver series response for '%s'.",
task.name,
extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
)
LOG.warning(f"No contents found in tver series response for '{task.name}'.")
return feed_url, items, has_items
season_block = contents[0] if contents else {}
@ -133,13 +114,9 @@ class TverHandler(BaseHandler):
continue
content = episode_data.get("content", {})
episode_id = content.pop("id", None)
episode_id = content.pop("id")
if not episode_id:
LOG.warning(
"Episode missing ID in '%s' feed. Skipping.",
task.name,
extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
)
LOG.warning(f"Episode missing ID in '{task.name}' feed. Skipping.")
continue
url = f"https://tver.jp/episodes/{episode_id}"
@ -150,9 +127,7 @@ class TverHandler(BaseHandler):
archive_id = id_dict.get("archive_id")
if not archive_id:
LOG.warning(
"Task '%s' could not compute an archive ID for an episode. Skipping item.",
task.name,
extra={"task_name": task.name, "series_id": series_id, "episode_id": episode_id, "url": url},
f"Could not compute archive ID for episode '{episode_id}' in '{task.name}' feed. Skipping."
)
continue
@ -162,24 +137,12 @@ class TverHandler(BaseHandler):
)
except Exception as exc:
LOG.warning(
"Failed to parse Tver episodes for task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"series_id": series_id,
"error": str(exc),
"exception_type": type(exc).__name__,
},
exc_info=True,
)
LOG.warning(f"Error parsing tver episodes for '{task.name}': {exc}")
return feed_url, items, has_items
@staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
series_id: str | None = TverHandler.parse(task.url)
if not series_id:
return TaskFailure(message="Unrecognized Tver series URL.")
@ -188,20 +151,8 @@ class TverHandler(BaseHandler):
try:
feed_url, items, has_items = await TverHandler._collect_feed(task, params, series_id)
except httpx.HTTPError as exc:
return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc))
except Exception as exc:
LOG.exception(
"Failed to fetch Tver feed for task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"series_id": series_id,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc))
task_items: list[TaskItem] = []
@ -210,11 +161,10 @@ class TverHandler(BaseHandler):
if not (url := entry.get("url")):
continue
archive_id: str | None = entry.get("archive_id")
metadata = entry.get("metadata", {})
if not isinstance(metadata, dict):
metadata = {}
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
archive_id: str = entry.get("archive_id")
task_items.append(
TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=entry.get("metadata", {}))
)
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})

View file

@ -1,20 +1,17 @@
import logging
import re
from typing import TYPE_CHECKING
from xml.etree.ElementTree import Element
import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
if TYPE_CHECKING:
from xml.etree.ElementTree import Element
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("handlers.twitch")
class TwitchHandler(BaseHandler):
@ -24,11 +21,7 @@ class TwitchHandler(BaseHandler):
@staticmethod
async def can_handle(task: HandleTask) -> bool:
LOG.debug(
"Checking if task '%s' uses a parsable Twitch URL.",
task.name,
extra={"task_name": task.name, "url": task.url},
)
LOG.debug(f"Checking if task '{task.name}' is using parsable Twitch URL: {task.url}")
return TwitchHandler.parse(task.url) is not None
@staticmethod
@ -41,7 +34,7 @@ class TwitchHandler(BaseHandler):
feed_url: str = TwitchHandler.FEED.format(handle=handle_name)
LOG.debug("Fetching '%s' feed.", task.name, extra={"task_name": task.name, "feed_url": feed_url})
LOG.debug(f"Fetching '{task.name}' feed.")
response = await TwitchHandler.request(url=feed_url, ytdlp_opts=params)
response.raise_for_status()
@ -53,22 +46,14 @@ class TwitchHandler(BaseHandler):
link_elem: Element[str] | None = entry.find("link")
url: str = link_elem.text.strip() if link_elem is not None and link_elem.text else ""
if not url:
LOG.warning(
"Entry in '%s' feed is missing URL. Skipping entry.",
task.name,
extra={"task_name": task.name, "feed_url": feed_url},
)
LOG.warning(f"Entry in '{task.name}' feed is missing URL. Skipping entry.")
continue
match: re.Match[str] | None = re.search(
r"^https?://(?:www\.)?twitch\.tv/videos/(?P<id>\d+)(?:[/?].*)?$", url
)
if not match:
LOG.warning(
"Task '%s' produced a feed entry that does not look like a Twitch VOD link. Skipping entry.",
task.name,
extra={"task_name": task.name, "url": url},
)
LOG.warning(f"URL in '{task.name}' feed does not look like a VOD link: {url}")
continue
vid: str = match.group("id")
@ -81,11 +66,7 @@ class TwitchHandler(BaseHandler):
id_dict = get_archive_id(url)
archive_id: str | None = id_dict.get("archive_id")
if not archive_id:
LOG.warning(
"Task '%s' could not compute an archive ID for a Twitch video. Skipping entry.",
task.name,
extra={"task_name": task.name, "video_id": vid, "url": url},
)
LOG.warning(f"Could not compute archive ID for video '{vid}' in '{task.name}' feed. Skipping entry.")
continue
items.append({"id": vid, "url": url, "title": title, "archive_id": archive_id})
@ -93,8 +74,7 @@ class TwitchHandler(BaseHandler):
return feed_url, items, has_items
@staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
handle_name: str | None = TwitchHandler.parse(task.url)
if not handle_name:
return TaskFailure(message="Unrecognized Twitch channel URL.")
@ -103,19 +83,8 @@ class TwitchHandler(BaseHandler):
try:
feed_url, items, has_items = await TwitchHandler._collect_feed(task, params, handle_name)
except httpx.HTTPError as exc:
return TaskFailure(message="Failed to fetch Twitch feed.", error=str(exc))
except Exception as exc:
LOG.exception(
"Failed to fetch Twitch feed for task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return TaskFailure(message="Failed to fetch Twitch feed.", error=str(exc))
task_items: list[TaskItem] = []
@ -124,7 +93,7 @@ class TwitchHandler(BaseHandler):
if not (url := entry.get("url")):
continue
archive_id: str | None = entry.get("archive_id")
archive_id: str = entry.get("archive_id")
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id))
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})

View file

@ -1,20 +1,17 @@
import logging
import re
from typing import TYPE_CHECKING, Any
from xml.etree.ElementTree import Element
import httpx
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.ytdlp.utils import get_archive_id
from app.library.config import Config
from app.library.log import get_logger
from ._base_handler import BaseHandler
if TYPE_CHECKING:
from xml.etree.ElementTree import Element
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("handlers.youtube")
class YoutubeHandler(BaseHandler):
@ -30,11 +27,7 @@ class YoutubeHandler(BaseHandler):
@staticmethod
async def can_handle(task: HandleTask) -> bool:
LOG.debug(
"Checking if task '%s' uses a parsable YouTube URL.",
task.name,
extra={"task_name": task.name, "url": task.url},
)
LOG.debug(f"'{task.name}': Checking if task URL is parsable YouTube URL: {task.url}")
return YoutubeHandler.parse(task.url) is not None
@staticmethod
@ -54,7 +47,7 @@ class YoutubeHandler(BaseHandler):
from defusedxml.ElementTree import fromstring
feed_url: str = YoutubeHandler.FEED.format(type=parsed["type"], id=parsed["id"])
LOG.debug("'%s': Fetching feed.", task.name, extra={"task_name": task.name, "feed_url": feed_url})
LOG.debug(f"'{task.name}': Fetching feed.")
response = await YoutubeHandler.request(url=feed_url, ytdlp_opts=params)
response.raise_for_status()
@ -72,11 +65,7 @@ class YoutubeHandler(BaseHandler):
vid_elem: Element[str] | None = entry.find("yt:videoId", ns)
vid: str = vid_elem.text if vid_elem is not None and vid_elem.text else ""
if not vid:
LOG.warning(
"'%s': Entry in the feed is missing a video ID. Skipping.",
task.name,
extra={"task_name": task.name, "feed_url": feed_url},
)
LOG.warning(f"'{task.name}': Entry in the feed is missing a video ID. Skipping.")
continue
url: str = f"https://www.youtube.com/watch?v={vid}"
@ -84,11 +73,7 @@ class YoutubeHandler(BaseHandler):
id_dict: dict[str, str | None] = get_archive_id(url)
archive_id: str | None = id_dict.get("archive_id")
if not archive_id:
LOG.warning(
"Task '%s' could not compute an archive ID for a YouTube video. Skipping item.",
task.name,
extra={"task_name": task.name, "video_id": vid, "url": url},
)
LOG.warning(f"'{task.name}': Could not compute archive ID for video '{vid}' in feed. Skipping.")
continue
title_elem: Element[str] | None = entry.find("atom:title", ns)
@ -104,8 +89,7 @@ class YoutubeHandler(BaseHandler):
return feed_url, items, real_count
@staticmethod
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
_ = config
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
if not parsed:
return TaskFailure(message="Unrecognized YouTube channel or playlist URL.")
@ -114,19 +98,8 @@ class YoutubeHandler(BaseHandler):
try:
feed_url, items, real_count = await YoutubeHandler._get(task, params, parsed)
except httpx.HTTPError as exc:
return TaskFailure(message="Failed to fetch YouTube feed.", error=str(exc))
except Exception as exc:
LOG.exception(
"Failed to fetch YouTube feed for task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return TaskFailure(message="Failed to fetch YouTube feed.", error=str(exc))
task_items: list[TaskItem] = []
@ -135,7 +108,7 @@ class YoutubeHandler(BaseHandler):
if not (url := entry.get("url")):
continue
archive_id: str | None = entry.get("archive_id")
archive_id: str = entry.get("archive_id")
metadata: dict[str, Any] = {"published": entry.get("published")}
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))

View file

@ -1,17 +1,17 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any
from app.features.core.migration import Migration as FeatureMigration
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.tasks.definitions.repository import TaskDefinitionsRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -48,11 +48,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert task definition '%s'.",
normalized.get("name"),
extra={"definition": normalized.get("name"), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert task definition '%s': %s", normalized.get("name"), exc)
finally:
await self._move_file(path)
@ -66,21 +62,13 @@ class Migration(FeatureMigration):
try:
content = path.read_text(encoding="utf-8")
except Exception as exc:
LOG.exception(
"Failed to read task definition '%s'.",
path,
extra={"path": str(path), "exception_type": type(exc).__name__},
)
LOG.error("Failed to read task definition '%s': %s", path, exc)
return None
try:
payload = json.loads(content)
except Exception as exc:
LOG.exception(
"Failed to parse JSON for task definition '%s'.",
path,
extra={"path": str(path), "exception_type": type(exc).__name__},
)
LOG.error("Failed to parse JSON for '%s': %s", path, exc)
return None
if not isinstance(payload, dict):

View file

@ -1,5 +1,6 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from sqlalchemy import func, or_, select
@ -9,28 +10,24 @@ from app.features.core.schemas import CEFeature, ConfigEvent
from app.features.tasks.definitions.migration import Migration
from app.features.tasks.definitions.models import TaskDefinitionModel
from app.library.Events import Event, EventBus, Events
from app.library.log import get_logger
from app.library.Services import Services
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from collections.abc import AsyncGenerator
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class TaskDefinitionsRepository(metaclass=Singleton):
def __init__(self, session: SessionFactory | None = None) -> None:
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: SessionFactory = session or get_session
self.session: AsyncGenerator[AsyncSession] = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -54,12 +51,12 @@ class TaskDefinitionsRepository(metaclass=Singleton):
LOG.debug("Refreshing task definitions due to configuration update.")
await GenericTaskHandler.refresh_definitions(force=True)
Services.get_instance().add(TaskDefinitionsRepository.__name__, self)
Services.get_instance().add(__class__.__name__, self)
EventBus.get_instance().subscribe(
Events.STARTED, handle_event, f"{TaskDefinitionsRepository.__name__}.run_migrations"
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions")
async def all(self) -> list[TaskDefinitionModel]:
async def list(self) -> list[TaskDefinitionModel]:
async with self.session() as session:
result: Result[tuple[TaskDefinitionModel]] = await session.execute(
select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc())
@ -118,9 +115,9 @@ class TaskDefinitionsRepository(metaclass=Singleton):
async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel:
async with self.session() as session:
model: TaskDefinitionModel = TaskDefinitionModel(**payload)
model: TaskDefinitionModel = TaskDefinitionModel(**payload) if isinstance(payload, dict) else payload
if model.id is not None:
model.id = None # ty: ignore
model.id = None
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Task definition with name '{model.name}' already exists."

View file

@ -48,13 +48,10 @@ class HandleTask(TaskSchema):
if isinstance(ret, tuple):
return ret
archive_file = ret.get("file")
items = ret.get("items", set())
if not isinstance(archive_file, Path) or not isinstance(items, set):
return (False, "Failed to get archive information.")
archive_items = [item for item in items if isinstance(item, str)]
archive_file: Path = ret.get("file")
items: set[str] = ret.get("items", set())
if len(archive_items) < 1 or not archive_add(archive_file, archive_items):
if len(items) < 1 or not archive_add(archive_file, list(items)):
return (True, "No new items to mark as downloaded.")
return (True, f"Task '{self.name}' items marked as downloaded.")
@ -73,13 +70,10 @@ class HandleTask(TaskSchema):
if isinstance(ret, tuple):
return ret
archive_file = ret.get("file")
items = ret.get("items", set())
if not isinstance(archive_file, Path) or not isinstance(items, set):
return (False, "Failed to get archive information.")
archive_items = [item for item in items if isinstance(item, str)]
archive_file: Path = ret.get("file")
items: set[str] = ret.get("items", set())
if len(archive_items) < 1 or not archive_delete(archive_file, archive_items):
if len(items) < 1 or not archive_delete(archive_file, list(items)):
return (True, "No items to remove from archive file.")
return (True, f"Removed '{self.name}' items from archive file.")
@ -113,7 +107,6 @@ class HandleTask(TaskSchema):
no_archive=True,
follow_redirect=False,
sanitize_info=True,
budget_sleep=True,
)
if not ie_info or not isinstance(ie_info, dict):
@ -140,7 +133,7 @@ class HandleTask(TaskSchema):
archive_file: Path = Path(archive_file)
(ie_info, _) = await fetch_info(params, self.url, no_archive=True, follow_redirect=True, budget_sleep=True)
(ie_info, _) = await fetch_info(params, self.url, no_archive=True, follow_redirect=True)
if not ie_info or not isinstance(ie_info, dict):
return (False, "Failed to extract information from URL.")

View file

@ -1,3 +1,4 @@
import logging
from typing import Any
from aiohttp import web
@ -15,10 +16,9 @@ from app.features.tasks.definitions.schemas import (
from app.features.tasks.definitions.utils import model_to_schema, schema_to_payload
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.router import route
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
@route("GET", "api/tasks/definitions/", "task_definitions")
@ -95,11 +95,7 @@ async def task_definitions_create(request: Request, encoder: Encoder, notify: Ev
except ValueError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
except Exception as exc:
LOG.exception(
"Failed to create task definition '%s'.",
getattr(definition_input, "name", None),
extra={"definition": getattr(definition_input, "name", None), "exception_type": type(exc).__name__},
)
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to create task definition."},
status=web.HTTPInternalServerError.status_code,
@ -148,15 +144,7 @@ async def task_definitions_update(request: Request, encoder: Encoder, notify: Ev
except ValueError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
except Exception as exc:
LOG.exception(
"Failed to update task definition '%s'.",
definition_input.name,
extra={
"definition_id": identifier,
"definition": definition_input.name,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to update task definition."},
status=web.HTTPInternalServerError.status_code,
@ -214,11 +202,7 @@ async def task_definitions_patch(request: Request, encoder: Encoder, notify: Eve
except ValueError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
except Exception as exc:
LOG.exception(
"Failed to patch task definition '%s'.",
identifier,
extra={"definition_id": identifier, "exception_type": type(exc).__name__},
)
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to patch task definition."},
status=web.HTTPInternalServerError.status_code,
@ -244,11 +228,7 @@ async def task_definitions_delete(request: Request, encoder: Encoder, notify: Ev
except KeyError as exc:
return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code)
except Exception as exc:
LOG.exception(
"Failed to delete task definition '%s'.",
identifier,
extra={"definition_id": identifier, "exception_type": type(exc).__name__},
)
LOG.exception(exc)
return web.json_response(
data={"error": "Failed to delete task definition."},
status=web.HTTPInternalServerError.status_code,

View file

@ -3,19 +3,18 @@ from __future__ import annotations
import asyncio
import importlib
import inspect
import logging
import pkgutil
import random
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from app.features.tasks.definitions.handlers._base_handler import BaseHandler
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
from app.features.tasks.models import TaskModel
from app.features.ytdlp.utils import archive_read
from app.library.downloads.queue_manager import DownloadQueue
from app.library.Events import EventBus, Events
from app.library.ItemDTO import Item, ItemDTO
from app.library.log import get_logger
from app.library.Services import Services
if TYPE_CHECKING:
@ -23,12 +22,12 @@ if TYPE_CHECKING:
from app.library.config import Config
from app.library.Scheduler import Scheduler
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("definitions.service")
class TaskHandle:
def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None:
self._handlers: list[type[BaseHandler]] = []
self._handlers: list[type] = []
"The available handlers."
self._repo: TasksRepository = tasks
"The tasks manager."
@ -36,7 +35,7 @@ class TaskHandle:
"The scheduler."
self._config: Config = config
"The configuration."
self._task_name: str = f"{TaskHandle.__name__}._dispatcher"
self._task_name: str = f"{__class__.__name__}._dispatcher"
"The task name for the scheduler."
self._queued: dict[str, set[str]] = {}
"Queued archive IDs per handler."
@ -46,11 +45,11 @@ class TaskHandle:
EventBus.get_instance().subscribe(
Events.ITEM_ERROR,
self._handle_item_error,
f"{TaskHandle.__name__}.item_error",
f"{__class__.__name__}.item_error",
)
def load(self) -> None:
self._handlers: list[type[BaseHandler]] = self._discover()
self._handlers: list[type] = self._discover()
timer: str = self._config.tasks_handler_timer
try:
@ -59,30 +58,20 @@ class TaskHandle:
CronSim(timer, datetime.now(UTC))
except Exception as e:
timer = "15 */1 * * *"
LOG.error(
"Invalid task handler timer '%s'; using default '%s'.",
self._config.tasks_handler_timer,
timer,
extra={
"timer": self._config.tasks_handler_timer,
"default_timer": timer,
"exception_type": type(e).__name__,
},
)
LOG.error(f"Invalid timer format. '{e!s}'. Defaulting to '{timer}'.")
self._scheduler.add(
timer=timer,
func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"),
id=f"{TaskHandle.__name__}._dispatcher",
id=f"{__class__.__name__}._dispatcher",
)
async def _dispatcher(self):
s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []}
handler_groups: dict[str, list[tuple[HandleTask, type[BaseHandler]]]] = {}
dispatches: list[tuple[HandleTask, type[BaseHandler], asyncio.Task[TaskResult | TaskFailure | None]]] = []
handler_groups: dict[str, list[tuple[HandleTask, type]]] = {}
tasks: list[TaskModel] = await self._repo.all()
tasks: list[TaskModel] = await self._repo.list()
for task_model in tasks:
task: HandleTask = HandleTask.model_validate(task_model)
@ -92,16 +81,12 @@ class TaskHandle:
continue
if not task.get_ytdlp_opts().get_all().get("download_archive"):
LOG.debug(
"Task '%s' does not have an archive file configured.",
task.name,
extra={"task_id": task.id, "task_name": task.name},
)
LOG.debug(f"Task '{task.name}' does not have an archive file configured.")
s["f"].append(task.name)
continue
try:
handler: type[BaseHandler] | None = await self._find_handler(task)
handler: type | None = await self._find_handler(task)
if handler is None:
s["u"].append(task.name)
continue
@ -110,12 +95,9 @@ class TaskHandle:
if handler_name not in handler_groups:
handler_groups[handler_name] = []
handler_groups[handler_name].append((task, handler))
s["h"].append(task.name)
except Exception as e:
LOG.exception(
"Failed to find handler for task '%s'.",
task.name,
extra={"task_id": task.id, "task_name": task.name, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to handle task '{task.name}'. '{e!s}'.")
s["f"].append(task.name)
for tasks_with_handlers in handler_groups.values():
@ -130,61 +112,15 @@ class TaskHandle:
name=f"taskHandler-{task.id}",
)
t.add_done_callback(lambda fut, t=task: self._handle_exception(fut, t))
dispatches.append((task, handler, t))
except Exception as e:
LOG.exception(
"Failed to schedule handler '%s' for task '%s'.",
handler.__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"exception_type": type(e).__name__,
},
)
s["f"].append(task.name)
if dispatches:
results = await asyncio.gather(*(t for _, _, t in dispatches), return_exceptions=True)
for (task, handler, _), result in zip(dispatches, results, strict=True):
if isinstance(result, TaskResult):
s["h"].append(task.name)
continue
if isinstance(result, TaskFailure):
s["f"].append(task.name)
continue
if result is None:
LOG.error(
"Handler '%s' returned no result for task '%s'.",
handler.__name__,
task.name,
extra={"task_id": task.id, "task_name": task.name, "handler_name": handler.__name__},
)
s["f"].append(task.name)
LOG.error(f"Failed to dispatch task '{task.name}'. '{e!s}'.")
if len(tasks) > 0:
LOG.info(
"Task dispatch finished: %s handled, %s unhandled, %s disabled, %s failed.",
len(s["h"]),
len(s["u"]),
len(s["d"]),
len(s["f"]),
extra={
"handled_count": len(s["h"]),
"unhandled_count": len(s["u"]),
"disabled_count": len(s["d"]),
"failed_count": len(s["f"]),
},
f"Tasks handler summary: Handled: {len(s['h'])}, Unhandled: {len(s['u'])}, Disabled: {len(s['d'])}, Failed: {len(s['f'])}."
)
async def _dispatch(
self, task: HandleTask, handler: type[BaseHandler], delay: float
) -> TaskResult | TaskFailure | None:
async def _dispatch(self, task: HandleTask, handler: type, delay: float) -> TaskResult | TaskFailure | None:
"""
Dispatch a task after a random delay to avoid rate limiting.
@ -198,12 +134,7 @@ class TaskHandle:
"""
if delay > 0:
LOG.debug(
"Delaying dispatch of task '%s' by %.1f seconds.",
task.name,
delay,
extra={"task_id": task.id, "task_name": task.name, "delay_s": round(delay, 1)},
)
LOG.debug(f"Delaying dispatch of task '{task.name}' by {delay:.1f} seconds.")
await asyncio.sleep(delay)
return await self.dispatch(task, handler=handler)
@ -212,29 +143,15 @@ class TaskHandle:
return
if exc := fut.exception():
LOG.exception(
"Task handler raised after dispatch.",
extra={"task_id": task.id, "task_name": task.name, "exception_type": type(exc).__name__},
exc_info=(type(exc), exc, exc.__traceback__),
)
LOG.error(f"Exception while handling task '{task.name}': {exc}")
async def _find_handler(self, task: HandleTask) -> type[BaseHandler] | None:
async def _find_handler(self, task: HandleTask) -> type | None:
for cls in self._handlers:
try:
if await Services.get_instance().handle_async(handler=cls.can_handle, task=task):
return cls
except Exception as e:
LOG.exception(
"Handler '%s' capability check failed for task '%s'.",
cls.__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": cls.__name__,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
continue
return None
@ -242,10 +159,9 @@ class TaskHandle:
async def dispatch(
self,
task: HandleTask,
handler: type[BaseHandler] | None = None,
**kwargs,
handler: type | None = None,
**kwargs, # noqa: ARG002
) -> TaskResult | TaskFailure | None:
_ = kwargs
"""
Dispatch a task to the appropriate handler.
@ -269,59 +185,20 @@ class TaskHandle:
extraction: TaskResult | TaskFailure = await services.handle_async(
handler=handler.extract, task=task, config=self._config
)
except NotImplementedError as exc:
LOG.exception(
"Task handler '%s' does not implement extraction for task '%s'.",
handler.__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"exception_type": type(exc).__name__,
},
)
except NotImplementedError:
LOG.error(f"Handler '{handler.__name__}' does not implement extract().")
return TaskFailure(message="Handler does not support extraction.")
except Exception as exc:
LOG.exception(
"Handler '%s' extraction failed for task '%s'.",
handler.__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"exception_type": type(exc).__name__,
},
)
LOG.exception(exc)
raise
if isinstance(extraction, TaskFailure):
msg: str = extraction.message
if extraction.error and extraction.error != extraction.message:
msg = f"{msg} {extraction.error}"
LOG.error(
"Handler '%s' failed to extract items for task '%s' because %s.",
handler.__name__,
task.name,
msg,
extra={"task_id": task.id, "task_name": task.name, "handler_name": handler.__name__, "error": msg},
)
LOG.error(f"Handler '{handler.__name__}' failed to extract items: {extraction.message}")
return extraction
if not isinstance(extraction, TaskResult):
LOG.error(
"Handler '%s' returned unexpected result type '%s' for task '%s'.",
handler.__name__,
type(extraction).__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"result_type": type(extraction).__name__,
},
f"Handler '{handler.__name__}' returned unexpected result type '{type(extraction).__name__}'.",
)
return TaskFailure(
message="Handler returned invalid result type.", metadata={"type": type(extraction).__name__}
@ -349,18 +226,7 @@ class TaskHandle:
for item in raw_items:
if not isinstance(item, TaskItem):
LOG.warning(
"Handler '%s' returned unexpected item type '%s' for task '%s'.",
handler.__name__,
type(item).__name__,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"item_type": type(item).__name__,
},
)
LOG.warning("Handler '{handler.__name__}' produced non-TaskItem entry: {item!r}")
continue
url: str = item.url
@ -369,13 +235,7 @@ class TaskHandle:
archive_id: str | None = item.archive_id
if not archive_id:
LOG.warning(
"Handler '%s' skipped '%s' for task '%s' because it has no archive ID.",
handler.__name__,
url,
task.name,
extra={"task_id": task.id, "task_name": task.name, "handler_name": handler.__name__, "url": url},
)
LOG.warning(f"'{task.name}': Item with URL '{url}' is missing an archive ID. Skipping.")
continue
if archive_id in queued:
@ -404,31 +264,12 @@ class TaskHandle:
if not filtered:
if raw_items:
LOG.debug(
"Handler '%s' found %s item(s) for task '%s', but none were queued.",
handler.__name__,
len(raw_items),
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"raw_count": len(raw_items),
},
f"Handler '{handler.__name__}' produced '{len(raw_items)}' for '{task.name}' items, none queued after filtering."
)
return TaskResult(items=[], metadata=metadata)
LOG.info(
"Handler '%s' found %s new item(s) for task '%s'.",
handler.__name__,
len(filtered),
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"handler_name": handler.__name__,
"item_count": len(filtered),
"raw_count": len(raw_items),
},
f"Handler '{handler.__name__}' Found '{len(filtered)}' new items for '{task.name}' (raw={len(raw_items)})."
)
base_item = Item.format(
@ -503,12 +344,7 @@ class TaskHandle:
try:
matched = await services.handle_async(handler=handler_cls.can_handle, task=task)
except Exception as exc: # pragma: no cover - defensive
LOG.exception(
"Handler '%s' inspection capability check failed for '%s'.",
handler_cls.__name__,
url,
extra={"handler_name": handler_cls.__name__, "url": url, "exception_type": type(exc).__name__},
)
LOG.exception(exc)
message = str(exc)
return TaskFailure(
message=message,
@ -546,12 +382,7 @@ class TaskHandle:
metadata={**base_metadata, "supported": False},
)
except Exception as exc:
LOG.exception(
"Handler '%s' manual inspection failed for '%s'.",
handler_cls.__name__,
url,
extra={"handler_name": handler_cls.__name__, "url": url, "exception_type": type(exc).__name__},
)
LOG.exception(exc)
message = str(exc)
return TaskFailure(
message=message,
@ -572,11 +403,7 @@ class TaskHandle:
if not isinstance(extraction, TaskResult):
LOG.error(
"Handler '%s' returned unexpected result type '%s' while inspecting '%s'.",
handler_cls.__name__,
type(extraction).__name__,
url,
extra={"handler_name": handler_cls.__name__, "url": url, "result_type": type(extraction).__name__},
f"Handler '{handler_cls.__name__}' returned unexpected result type '{type(extraction).__name__}' during inspection.",
)
extraction = TaskResult()
@ -586,11 +413,11 @@ class TaskHandle:
return TaskResult(items=list(extraction.items), metadata=combined_metadata)
def _discover(self) -> list[type[BaseHandler]]:
def _discover(self) -> list[type]:
"""Discover all available task handlers."""
import app.features.tasks.definitions.handlers as handlers_pkg
handlers: list[type[BaseHandler]] = []
handlers: list[type] = []
for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__):
if module_name.startswith("_"):

View file

@ -8,7 +8,6 @@ from app.features.tasks.definitions.results import TaskFailure, TaskResult
from app.features.tasks.definitions.schemas import (
Definition,
EngineConfig,
Parse,
RequestConfig,
ResponseConfig,
TaskDefinition,
@ -22,7 +21,7 @@ def reset_generic_handler(monkeypatch):
monkeypatch.setattr(GenericTaskHandler, "_sources_mtime", {})
def test_build_def_payload():
def test_build_task_definition_parses_valid_payload():
definition = TaskDefinition(
id=1,
name="example",
@ -31,24 +30,26 @@ def test_build_def_payload():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
}
),
parse={
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
),
)
assert definition.name == "example"
assert definition.match_url == ["https://example.com/articles/*"]
assert definition.definition.parse["link"]["expression"] == ".article a.link::attr(href)"
assert definition is not None, "TaskDefinition should be created"
assert "example" == definition.name, "Name should match"
assert "https://example.com/articles/*" in definition.match_url, "Match URL should be in list"
assert "link" in definition.definition.parse, "Parse should contain link field"
assert ".article a.link::attr(href)" == definition.definition.parse["link"]["expression"], (
"Link expression should match"
)
def test_build_def_container():
def test_build_task_definition_handles_container():
definition = TaskDefinition(
id=2,
name="container",
@ -57,28 +58,28 @@ def test_build_def_container():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"selector": ".cards .card",
"fields": {
"link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
},
}
parse={
"items": {
"selector": ".cards .card",
"fields": {
"link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
),
)
assert definition.definition.parse["items"]["selector"] == ".cards .card"
assert definition.definition.parse["items"]["fields"]["link"]["attribute"] == "href"
assert definition is not None, "TaskDefinition should be created"
assert "items" in definition.definition.parse, "Parse should contain items container"
assert ".cards .card" == definition.definition.parse["items"]["selector"], "Items selector should match"
assert "link" in definition.definition.parse["items"]["fields"], "Items fields should contain link"
def test_build_def_json():
def test_build_task_definition_handles_json():
definition = TaskDefinition(
id=3,
name="json-def",
@ -87,30 +88,32 @@ def test_build_def_json():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
parse={
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
),
)
assert definition.definition.response.type == "json"
assert definition.definition.parse["items"]["type"] == "jsonpath"
assert definition.definition.parse["items"]["fields"]["link"]["type"] == "jsonpath"
assert definition is not None, "TaskDefinition should be created"
assert "json" == definition.definition.response.type, "Response type should be json"
assert "items" in definition.definition.parse, "Parse should contain items container"
assert "jsonpath" == definition.definition.parse["items"]["type"], "Items type should be jsonpath"
assert "jsonpath" == definition.definition.parse["items"]["fields"]["link"]["type"], (
"Link field type should be jsonpath"
)
def test_parse_items_basic():
def test_parse_items_extracts_values():
definition = TaskDefinition(
id=4,
name="example",
@ -119,13 +122,11 @@ def test_parse_items_basic():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
"id": {"type": "css", "expression": ".article", "attribute": "data-id"},
}
),
parse={
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
"id": {"type": "css", "expression": ".article", "attribute": "data-id"},
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
@ -145,16 +146,14 @@ def test_parse_items_basic():
items = GenericTaskHandler._parse_items(definition, html, "https://example.com/base/")
assert len(items) == 2
assert items[0] == {
"link": "https://example.com/article-101",
"title": "First Title",
"id": "101",
}
assert items[1]["link"] == "https://example.com/article-102"
assert 2 == len(items), "Should extract 2 items"
assert "https://example.com/article-101" == items[0]["link"], "First item link should be absolute URL"
assert "First Title" == items[0]["title"], "First item title should match"
assert "101" == items[0]["id"], "First item id should match"
assert "https://example.com/article-102" == items[1]["link"], "Second item link should match"
def test_parse_items_cards():
def test_parse_items_handles_nested_card_layout():
definition = TaskDefinition(
id=5,
name="nested",
@ -163,36 +162,34 @@ def test_parse_items_cards():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"type": "css",
"selector": ".columns .card",
"fields": {
"link": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "href",
},
"title": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "text",
},
"poet": {
"type": "css",
"expression": "footer .card-footer-item:first-child a",
"attribute": "text",
},
"category": {
"type": "css",
"expression": "footer .card-footer-item:nth-child(2) a",
"attribute": "text",
},
parse={
"items": {
"type": "css",
"selector": ".columns .card",
"fields": {
"link": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "href",
},
}
"title": {
"type": "css",
"expression": ".card-header a[href]",
"attribute": "text",
},
"poet": {
"type": "css",
"expression": "footer .card-footer-item:first-child a",
"attribute": "text",
},
"category": {
"type": "css",
"expression": "footer .card-footer-item:nth-child(2) a",
"attribute": "text",
},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(),
@ -237,21 +234,19 @@ def test_parse_items_cards():
items = GenericTaskHandler._parse_items(definition, html, "https://example.com")
assert len(items) == 2
assert items[0] == {
"link": "https://example.com/poems/view/111",
"title": "First Poem",
"poet": "Poet Alpha",
"category": "Category One",
}
assert items[1] == {
"link": "https://example.com/poems/view/222",
"title": "Second Poem",
"poet": "Poet Beta",
}
assert 2 == len(items), "Should extract 2 items"
assert "https://example.com/poems/view/111" == items[0]["link"], "First item link should match"
assert "First Poem" == items[0]["title"], "First item title should match"
assert "Poet Alpha" == items[0]["poet"], "First item poet should match"
assert "Category One" == items[0]["category"], "First item category should match"
assert "https://example.com/poems/view/222" == items[1]["link"], "Second item link should match"
assert "Second Poem" == items[1]["title"], "Second item title should match"
assert "Poet Beta" == items[1]["poet"], "Second item poet should match"
assert "category" not in items[1], "Second item should not have category"
def test_parse_items_json():
def test_parse_items_handles_json_container():
definition = TaskDefinition(
id=6,
name="json",
@ -260,19 +255,17 @@ def test_parse_items_json():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "entries",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
"id": {"type": "jsonpath", "expression": "id"},
},
}
parse={
"items": {
"type": "jsonpath",
"selector": "entries",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
"id": {"type": "jsonpath", "expression": "id"},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
@ -294,10 +287,14 @@ def test_parse_items_json():
json_data=payload,
)
assert items == [
{"link": "https://example.com/video/1", "title": "First", "id": "1"},
{"link": "https://example.com/video/2", "title": "Second", "id": "2"},
]
assert 2 == len(items), "Should extract 2 items (third missing link)"
assert "https://example.com/video/1" == items[0]["link"], "First item link should be absolute"
assert "First" == items[0]["title"], "First item title should match"
assert "1" == items[0]["id"], "First item id should match"
assert "https://example.com/video/2" == items[1]["link"], "Second item link should match"
assert "Second" == items[1]["title"], "Second item title should match"
assert "2" == items[1]["id"], "Second item id should match"
@pytest.mark.asyncio
@ -310,18 +307,16 @@ async def test_generic_task_handler_inspect(monkeypatch):
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
parse={
"items": {
"type": "jsonpath",
"selector": "items",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
@ -350,14 +345,14 @@ async def test_generic_task_handler_inspect(monkeypatch):
task = HandleTask(id=1, name="Inspect", url="https://example.com/api")
result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task)
assert isinstance(result, TaskResult)
assert len(result.items) == 1
assert isinstance(result, TaskResult), "Result should be TaskResult"
assert 1 == len(result.items), "Should have 1 item"
item = result.items[0]
assert item.url == "https://example.com/video/1"
assert item.title == "First"
assert "https://example.com/video/1" == item.url, "Item URL should match"
assert "First" == item.title, "Item title should match"
def test_parse_items_json_list():
def test_parse_items_handles_json_top_level_list():
definition = TaskDefinition(
id=8,
name="json-list",
@ -366,18 +361,16 @@ def test_parse_items_json_list():
created_at=datetime.now(),
updated_at=datetime.now(),
definition=Definition(
parse=Parse.model_validate(
{
"items": {
"type": "jsonpath",
"selector": "[]",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
parse={
"items": {
"type": "jsonpath",
"selector": "[]",
"fields": {
"link": {"type": "jsonpath", "expression": "url"},
"title": {"type": "jsonpath", "expression": "title"},
},
}
),
},
engine=EngineConfig(),
request=RequestConfig(),
response=ResponseConfig(type="json"),
@ -396,7 +389,8 @@ def test_parse_items_json_list():
json_data=payload,
)
assert items == [
{"link": "https://example.com/video/1", "title": "First"},
{"link": "https://example.com/video/2", "title": "Second"},
]
assert 2 == len(items), "Should extract 2 items"
assert "https://example.com/video/1" == items[0]["link"], "First item link should match"
assert "First" == items[0]["title"], "First item title should match"
assert "https://example.com/video/2" == items[1]["link"], "Second item link should match"
assert "Second" == items[1]["title"], "Second item title should match"

View file

@ -1,5 +1,3 @@
from pathlib import Path
import pytest
from app.features.tasks.definitions.handlers.rss import RssGenericHandler
@ -23,10 +21,6 @@ class DummyOpts:
return self._data
def _opts(tmp_path: Path) -> DummyOpts:
return DummyOpts({"download_archive": str(tmp_path / "archive.txt")})
class TestRssHandlerParsing:
"""Test URL parsing for RSS/Atom feeds using the tests() method."""
@ -53,7 +47,7 @@ class TestRssHandlerExtraction:
"""Test RSS feed extraction and parsing."""
@pytest.mark.asyncio
async def test_rss_atom_feed_extraction(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_rss_atom_feed_extraction(self, monkeypatch):
"""Test extraction from Atom feed."""
atom_feed = """<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
@ -75,7 +69,7 @@ class TestRssHandlerExtraction:
return DummyResponse(atom_feed)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,
@ -95,7 +89,7 @@ class TestRssHandlerExtraction:
assert result.metadata["entry_count"] == 2
@pytest.mark.asyncio
async def test_rss_feed_extraction(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_rss_feed_extraction(self, monkeypatch):
"""Test extraction from RSS feed."""
rss_feed = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
@ -119,7 +113,7 @@ class TestRssHandlerExtraction:
return DummyResponse(rss_feed)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,
@ -139,7 +133,7 @@ class TestRssHandlerExtraction:
assert result.metadata["entry_count"] == 2
@pytest.mark.asyncio
async def test_can_handle(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_can_handle(self, monkeypatch):
"""Test can_handle method."""
atom_feed = """<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
@ -153,7 +147,7 @@ class TestRssHandlerExtraction:
return DummyResponse(atom_feed)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,
@ -178,7 +172,7 @@ class TestRssHandlerEdgeCases:
"""Test edge cases in RSS handling."""
@pytest.mark.asyncio
async def test_empty_feed(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_empty_feed(self, monkeypatch):
"""Test handling of empty feed."""
empty_feed = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
@ -192,7 +186,7 @@ class TestRssHandlerEdgeCases:
return DummyResponse(empty_feed)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,
@ -208,7 +202,7 @@ class TestRssHandlerEdgeCases:
assert result.metadata["entry_count"] == 0
@pytest.mark.asyncio
async def test_invalid_feed_url(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_invalid_feed_url(self, monkeypatch):
"""Test handling of invalid feed URL."""
from app.features.tasks.definitions.results import TaskFailure
@ -217,7 +211,7 @@ class TestRssHandlerEdgeCases:
raise Exception(msg)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,
@ -231,7 +225,7 @@ class TestRssHandlerEdgeCases:
assert isinstance(result, TaskFailure)
@pytest.mark.asyncio
async def test_missing_urls_in_feed(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_missing_urls_in_feed(self, monkeypatch):
"""Test handling of entries missing URLs."""
feed = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
@ -251,7 +245,7 @@ class TestRssHandlerEdgeCases:
return DummyResponse(feed)
monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path)) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,

View file

@ -69,7 +69,7 @@ class TestTaskDefinitionsRepository:
await repo.create(_sample_definition("Alpha", priority=2))
await repo.create(_sample_definition("Beta", priority=1))
items = await repo.all()
items = await repo.list()
assert len(items) == 2, "Should return two task definitions"
assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name"

View file

@ -1,5 +1,3 @@
from pathlib import Path
import pytest
from app.features.tasks.definitions.handlers.tver import TverHandler
@ -27,7 +25,7 @@ class DummyOpts:
@pytest.mark.asyncio
async def test_tver_handler_extract(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_tver_handler_extract(monkeypatch):
"""Test tver handler extraction of episodes from series."""
session_response = {
"result": {
@ -120,10 +118,8 @@ async def test_tver_handler_extract(monkeypatch: pytest.MonkeyPatch, tmp_path: P
msg = f"Unexpected URL: {url}"
raise RuntimeError(msg)
archive_path = tmp_path / "archive.txt"
monkeypatch.setattr(TverHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": str(archive_path)}))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": "/tmp/archive"}))
task = HandleTask(id=1, name="Test Tver Series", url="https://tver.jp/series/sr8sb9pnhc", preset="default")
@ -146,7 +142,7 @@ async def test_tver_handler_extract(monkeypatch: pytest.MonkeyPatch, tmp_path: P
@pytest.mark.parametrize(("url", "should_match"), TverHandler.tests())
def test_parse(url: str, should_match: bool) -> None:
def test_tver_handler_parse(url: str, should_match: bool):
"""Test tver URL parsing."""
result = TverHandler.parse(url)
if should_match:
@ -157,7 +153,7 @@ def test_parse(url: str, should_match: bool) -> None:
@pytest.mark.asyncio
async def test_can_handle() -> None:
async def test_tver_handler_can_handle():
"""Test tver handler can_handle method."""
task_valid = HandleTask(id=1, name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default")
task_invalid = HandleTask(id=2, name="Test", url="https://youtube.com/watch?v=123", preset="default")

View file

@ -1,5 +1,3 @@
from pathlib import Path
import pytest
from app.features.tasks.definitions.handlers.twitch import TwitchHandler
@ -24,7 +22,7 @@ class DummyOpts:
@pytest.mark.asyncio
async def test_twitch_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_twitch_handler_inspect(monkeypatch):
feed = """
<rss><channel>
<item>
@ -41,10 +39,8 @@ async def test_twitch_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path:
async def fake_request(**kwargs): # noqa: ARG001
return DummyResponse(feed)
archive_path = tmp_path / "archive.txt"
monkeypatch.setattr(TwitchHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": str(archive_path)})) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,

View file

@ -1,5 +1,3 @@
from pathlib import Path
import pytest
from app.features.tasks.definitions.handlers.youtube import YoutubeHandler
@ -24,7 +22,7 @@ class DummyOpts:
@pytest.mark.asyncio
async def test_youtube_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_youtube_handler_inspect(monkeypatch):
feed = """
<feed xmlns='http://www.w3.org/2005/Atom' xmlns:yt='http://www.youtube.com/xml/schemas/2015'>
<entry>
@ -43,10 +41,8 @@ async def test_youtube_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path
async def fake_request(**kwargs): # noqa: ARG001
return DummyResponse(feed)
archive_path = tmp_path / "archive.txt"
monkeypatch.setattr(YoutubeHandler, "request", staticmethod(fake_request))
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": str(archive_path)})) # noqa: ARG005
monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"})) # noqa: ARG005
task = HandleTask(
id=1,

View file

@ -1,17 +1,9 @@
from typing import Any, Literal, overload
from typing import Any
from app.features.tasks.definitions.models import TaskDefinitionModel
from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary
@overload
def model_to_schema(model: TaskDefinitionModel, summary: Literal[False] = False) -> TaskDefinition: ...
@overload
def model_to_schema(model: TaskDefinitionModel, summary: Literal[True]) -> TaskDefinitionSummary: ...
def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary:
"""
Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema.

View file

@ -1,18 +1,18 @@
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any
from app.features.core.migration import Migration as FeatureMigration
from app.features.tasks.schemas import Task
from app.library.config import Config
from app.library.log import get_logger
if TYPE_CHECKING:
from app.features.tasks.repository import TasksRepository
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
class Migration(FeatureMigration):
@ -36,11 +36,7 @@ class Migration(FeatureMigration):
try:
items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text())
except Exception as exc:
LOG.exception(
"Failed to read tasks migration file '%s'. Ignoring.",
self._source_file,
extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
)
LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
await self._move_file(self._source_file)
return
@ -58,11 +54,7 @@ class Migration(FeatureMigration):
await self._repo.create(normalized)
inserted += 1
except Exception as exc:
LOG.exception(
"Failed to insert task '%s'.",
normalized["name"],
extra={"task_name": normalized["name"], "exception_type": type(exc).__name__},
)
LOG.exception("Failed to insert task '%s': %s", normalized["name"], exc)
LOG.info("Migrated %s task(s) from %s.", inserted, self._source_file)
await self._move_file(self._source_file)

View file

@ -1,48 +1,29 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from sqlalchemy import func, or_, select
from app.features.core.deps import get_session
from app.features.tasks.models import TaskModel
from app.library.log import get_logger
from app.library.Singleton import Singleton
if TYPE_CHECKING:
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from collections.abc import AsyncGenerator
from sqlalchemy.engine.result import Result
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
LOG = get_logger()
def _model_from_payload(payload: dict[str, Any]) -> TaskModel:
model = TaskModel()
for key, value in payload.items():
if not hasattr(model, key):
msg = f"'{key}' is an invalid keyword argument for TaskModel"
raise TypeError(msg)
setattr(model, key, value)
return model
def _coerce_model(payload: TaskModel | dict[str, Any]) -> TaskModel:
if isinstance(payload, TaskModel):
return payload
return _model_from_payload(payload)
LOG: logging.Logger = logging.getLogger(__name__)
class TasksRepository(metaclass=Singleton):
def __init__(self, session: SessionFactory | None = None) -> None:
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
self._migrated = False
self.session: SessionFactory = session or get_session
self.session = session or get_session
async def run_migrations(self) -> None:
if self._migrated:
@ -57,7 +38,7 @@ class TasksRepository(metaclass=Singleton):
def get_instance() -> TasksRepository:
return TasksRepository()
async def all(self) -> list[TaskModel]:
async def list(self) -> list[TaskModel]:
async with self.session() as session:
result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc()))
return list(result.scalars().all())
@ -109,7 +90,7 @@ class TasksRepository(metaclass=Singleton):
"""Get all enabled tasks."""
async with self.session() as session:
result: Result[tuple[TaskModel]] = await session.execute(
select(TaskModel).where(TaskModel.enabled).order_by(TaskModel.name.asc())
select(TaskModel).where(TaskModel.enabled == True).order_by(TaskModel.name.asc()) # noqa: E712
)
return list(result.scalars().all())
@ -121,11 +102,11 @@ class TasksRepository(metaclass=Singleton):
)
return list(result.scalars().all())
async def create(self, payload: TaskModel | dict[str, Any]) -> TaskModel:
async def create(self, payload: TaskModel | dict) -> TaskModel:
async with self.session() as session:
model = _coerce_model(payload)
model: TaskModel = TaskModel(**payload) if isinstance(payload, dict) else payload
if model.id is not None:
model.id = None # ty: ignore
model.id = None
if await self.get_by_name(name=model.name) is not None:
msg: str = f"Task with name '{model.name}' already exists."

View file

@ -1,4 +1,4 @@
import asyncio
import logging
from typing import TYPE_CHECKING, Any
from aiohttp import web
@ -17,7 +17,6 @@ from app.library.ag_utils import ag
from app.library.config import Config
from app.library.encoder import Encoder
from app.library.Events import EventBus, Events
from app.library.log import get_logger
from app.library.router import route
from app.library.Utils import get_channel_images, get_file, validate_url
@ -25,7 +24,7 @@ if TYPE_CHECKING:
from pathlib import Path
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
TIMER_SLOTS_PER_HOUR: int = 12
@ -94,11 +93,7 @@ def _offset_timer(timer: str, index: int) -> str:
return f"{minute} {hour} {dom} {month} {dow}"
except Exception as e:
LOG.warning(
"Failed to offset task timer.",
extra={"timer": timer, "index": index, "error": str(e), "exception_type": type(e).__name__},
exc_info=True,
)
LOG.warning(f"Failed to offset timer '{timer}': {e}")
return timer
@ -141,15 +136,10 @@ async def _get_info(url: str, preset: str) -> tuple[str | None, str | None]:
return (name, converted_url)
except TimeoutError:
LOG.debug("Timeout while inferring name from '%s'.", url, extra={"url": url, "preset": preset})
LOG.debug(f"Timeout while inferring name from '{url}'")
return (None, None)
except Exception as e:
LOG.debug(
"Failed to infer a task name from '%s' because %s.",
url,
e,
extra={"url": url, "preset": preset, "error": str(e)},
)
LOG.debug(f"Failed to infer name from '{url}': {e}")
return (None, None)
@ -161,22 +151,6 @@ def _serialize(model: Any) -> dict:
return _model(model).model_dump()
async def _require_timer_or_handler(task: Task, handler: TaskHandle) -> None:
if task.timer:
return
if task.handler_enabled is False:
msg = "Task requires a timer when the handler is disabled."
raise ValueError(msg)
result = await handler.inspect(url=task.url, preset=task.preset, static_only=True)
if isinstance(result, TaskResult) and result.metadata.get("matched") is True:
return
msg = "Task requires a timer when no supported handler matches the URL."
raise ValueError(msg)
@route("GET", "api/tasks/", "tasks_list")
async def tasks_list(request: Request, repo: TasksRepository, encoder: Encoder) -> Response:
page, per_page = normalize_pagination(request)
@ -192,13 +166,7 @@ async def tasks_list(request: Request, repo: TasksRepository, encoder: Encoder)
@route("POST", "api/tasks/", "tasks_add")
async def tasks_add(
request: Request,
repo: TasksRepository,
encoder: Encoder,
notify: EventBus,
handler: TaskHandle,
) -> Response:
async def tasks_add(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response:
data = await request.json()
if isinstance(data, dict):
@ -237,18 +205,17 @@ async def tasks_add(
status=web.HTTPConflict.status_code,
)
pending_tasks: list[dict[str, Any]] = []
created_tasks: list[dict[str, Any]] = []
base_settings = first_task.model_dump(exclude={"id", "name", "url", "created_at", "updated_at"})
for idx, item in enumerate(data):
if not isinstance(item, dict):
LOG.warning("Skipping item %s: not a dict.", idx, extra={"index": idx})
LOG.warning(f"Skipping item {idx}: not a dict")
continue
url = str(item.get("url", "")).strip()
if not url:
LOG.debug("Skipping item %s: empty URL.", idx, extra={"index": idx})
LOG.debug(f"Skipping item {idx}: empty URL")
continue
inferred_name, converted_url = await _get_info(url, item.get("preset", first_task.preset))
@ -276,23 +243,11 @@ async def tasks_add(
try:
validated = Task.model_validate(task_dict)
task_data = validated.model_dump()
except ValidationError as exc:
return web.json_response(
data={"error": "Failed to validate task.", "detail": format_validation_errors(exc)},
status=web.HTTPBadRequest.status_code,
)
LOG.warning(f"Skipping task {idx}: validation failed - {exc}")
continue
try:
await _require_timer_or_handler(validated, handler)
except ValueError as exc:
return web.json_response(
data={"error": str(exc)},
status=web.HTTPBadRequest.status_code,
)
pending_tasks.append(validated.model_dump())
for idx, task_data in enumerate(pending_tasks):
try:
created = await repo.create(task_data)
saved = _serialize(created)
@ -302,11 +257,7 @@ async def tasks_add(
data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.CREATE, data=saved),
)
except ValueError as exc:
LOG.warning(
"Failed to create task from request item %s.",
idx,
extra={"index": idx, "error": str(exc), "exception_type": type(exc).__name__},
)
LOG.warning(f"Failed to create task {idx}: {exc}")
continue
if len(created_tasks) == 0:
@ -353,13 +304,7 @@ async def tasks_delete(request: Request, repo: TasksRepository, encoder: Encoder
@route("PATCH", r"api/tasks/{id:\d+}", "tasks_patch")
async def tasks_patch(
request: Request,
repo: TasksRepository,
encoder: Encoder,
notify: EventBus,
handler: TaskHandle,
) -> Response:
async def tasks_patch(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
@ -389,16 +334,6 @@ async def tasks_patch(
status=web.HTTPConflict.status_code,
)
merged = _model(model).model_copy(update=validated.model_dump(exclude_unset=True))
try:
await _require_timer_or_handler(merged, handler)
except ValueError as exc:
return web.json_response(
data={"error": str(exc)},
status=web.HTTPBadRequest.status_code,
)
updated = _serialize(await repo.update(model.id, validated.model_dump(exclude_unset=True)))
notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.UPDATE, data=updated))
return web.json_response(
@ -409,13 +344,7 @@ async def tasks_patch(
@route("PUT", r"api/tasks/{id:\d+}", "tasks_update")
async def tasks_update(
request: Request,
repo: TasksRepository,
encoder: Encoder,
notify: EventBus,
handler: TaskHandle,
) -> Response:
async def tasks_update(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response:
if not (identifier := request.match_info.get("id")):
return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
@ -445,14 +374,6 @@ async def tasks_update(
status=web.HTTPConflict.status_code,
)
try:
await _require_timer_or_handler(validated, handler)
except ValueError as exc:
return web.json_response(
data={"error": str(exc)},
status=web.HTTPBadRequest.status_code,
)
updated = _serialize(await repo.update(model.id, validated.model_dump(exclude_unset=True)))
notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.UPDATE, data=updated))
@ -483,7 +404,7 @@ async def task_handler_inspect(request: Request, handler: TaskHandle, encoder: E
static_only: bool = data.get("static_only", False) if isinstance(data, dict) else False
if not static_only:
try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
validate_url(url, allow_internal=config.allow_internal_urls)
except ValueError as e:
return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code)
@ -495,16 +416,7 @@ async def task_handler_inspect(request: Request, handler: TaskHandle, encoder: E
url=url, preset=preset, handler_name=handler_name, static_only=static_only
)
except Exception as e:
LOG.exception(
"Failed to inspect task handler for '%s'.",
url,
extra={
"handler_name": handler_name,
"url": url,
"static_only": static_only,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
return web.json_response(
{"error": "Failed to inspect handler.", "message": str(e)},
status=web.HTTPInternalServerError.status_code,
@ -653,12 +565,7 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
if not save_path.exists():
save_path.mkdir(parents=True, exist_ok=True)
except Exception as e:
LOG.warning(
"Failed to resolve the metadata output path for task '%s'.",
task.name,
extra={"task_id": task.id, "task_name": task.name, "error": str(e)},
exc_info=True,
)
LOG.warning(f"Failed to resolve final path from outtmpl. '{e!s}'")
info = {
"id": ag(metadata, ["id", "channel_id"]),
@ -676,12 +583,7 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
data={"error": "Failed to get title from metadata."}, status=web.HTTPBadRequest.status_code
)
LOG.info(
"Generating metadata for task '%s' in '%s'.",
task.name,
save_path,
extra={"task_id": task.id, "task_name": task.name, "save_path": str(save_path)},
)
LOG.info(f"Generating metadata for task '{task.name}' in '{save_path!s}'")
from yt_dlp.utils import sanitize_filename
@ -705,9 +607,8 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
xml_content += f' <uniqueid type="{NFOMakerPP._escape_text(info.get("id_type"))}" default="true">{NFOMakerPP._escape_text(info.get("id"))}</uniqueid>\n'
if info.get("uploader"):
xml_content += f" <studio>{NFOMakerPP._escape_text(info.get('uploader'))}</studio>\n"
tags = info.get("tags", [])
if isinstance(tags, list):
for tag in tags:
if info.get("tags", []):
for tag in info.get("tags", []):
xml_content += f" <tag>{NFOMakerPP._escape_text(tag)}</tag>\n"
if info.get("year"):
xml_content += f" <year>{info.get('year')}</year>\n"
@ -739,25 +640,14 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
url: str | None = None
try:
url = thumbnails.get(key)
LOG.info(
"Fetching '%s' thumbnail for task '%s'.",
key,
task.name,
extra={"task_id": task.id, "task_name": task.name, "thumbnail": key, "url": url},
)
LOG.info(f"Fetching thumbnail '{key}' from '{url}'")
if not url:
continue
try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
except ValueError as exc:
LOG.warning(
"Task '%s' has an invalid '%s' thumbnail URL because %s.",
task.name,
key,
exc,
extra={"task_id": task.id, "task_name": task.name, "thumbnail": key, "url": url},
)
validate_url(url, allow_internal=config.allow_internal_urls)
except ValueError:
LOG.warning(f"Invalid thumbnail url '{url}'")
continue
resp = await client.request(
@ -771,27 +661,11 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
img_file.write_bytes(resp.content)
thumbnails[key] = str(img_file.relative_to(config.download_path))
except Exception as e:
LOG.warning(
"Failed to fetch '%s' thumbnail for task '%s'.",
key,
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"thumbnail": key,
"url": url,
"error": str(e),
},
exc_info=True,
)
url_log = url or "unknown"
LOG.warning(f"Failed to fetch thumbnail '{key}' from '{url_log}'. '{e!s}'")
continue
except Exception as e:
LOG.warning(
"Failed to fetch metadata thumbnails for task '%s'.",
task.name,
extra={"task_id": task.id, "task_name": task.name, "error": str(e)},
exc_info=True,
)
LOG.warning(f"Failed to fetch thumbnails. '{e!s}'")
return web.json_response(data=info, status=web.HTTPOk.status_code, dumps=encoder.encode)
except ValueError as e:

View file

@ -1,12 +1,12 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from app.features.core.schemas import CEAction, CEFeature, ConfigEvent
from app.features.tasks.models import TaskModel
from app.features.tasks.utils import cron_time
from app.library.Events import Event, EventBus, Events
from app.library.log import get_logger
from app.library.Scheduler import Scheduler
from app.library.Services import Services
from app.library.Singleton import Singleton
@ -14,7 +14,7 @@ from app.library.Singleton import Singleton
if TYPE_CHECKING:
from aiohttp import web
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("tasks.service")
class Tasks(metaclass=Singleton):
@ -51,7 +51,7 @@ class Tasks(metaclass=Singleton):
pass
async def _load_tasks(self) -> None:
tasks = await self._repo.all()
tasks = await self._repo.list()
for task in tasks:
if not task.timer or not task.enabled:
@ -59,23 +59,10 @@ class Tasks(metaclass=Singleton):
try:
self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=f"task-cronjob-{task.id}")
LOG.info(
"Queued task '%s' to run at '%s'.",
task.name,
cron_time(task.timer),
extra={"task_id": task.id, "task_name": task.name, "timer": task.timer},
)
LOG.info(f"Task '{task.id}: {task.name}' queued to be executed '{cron_time(task.timer)}'.")
except Exception as e:
LOG.exception(
"Failed to queue task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"timer": task.timer,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
LOG.error(f"Failed to queue task '{task.name}'. '{e!s}'.")
async def _init_handlers_service(self, scheduler) -> None:
"""Initialize the handlers service after migrations."""
@ -108,12 +95,7 @@ class Tasks(metaclass=Singleton):
if task.timer and task.enabled:
self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=task_id)
LOG.info(
"Queued task '%s' to run at '%s'.",
task.name,
cron_time(task.timer),
extra={"task_id": task.id, "task_name": task.name, "timer": task.timer},
)
LOG.info(f"Task '{task.id}: {task.name}' queued to be executed '{cron_time(task.timer)}'.")
async def _runner(self, task: TaskModel) -> None:
"""
@ -131,29 +113,17 @@ class Tasks(metaclass=Singleton):
from app.library.ItemDTO import Item
timeNow: str = datetime.now(UTC).isoformat()
task_id = task.id
task_name = task.name
try:
current_task = await self._repo.get(task_id)
if not current_task:
LOG.info("Task '%s' no longer exists.", task_name, extra={"task_id": task_id, "task_name": task_name})
if not (task := await self._repo.get(task.id)):
LOG.info(f"Task '{task.name}' no longer exists.")
return
task = current_task
if not task.enabled:
LOG.debug(
"Task '%s' is disabled. Skipping execution.",
task.name,
extra={"task_id": task.id, "task_name": task.name},
)
LOG.debug(f"Task '{task.name}' is disabled. Skipping execution.")
return
if not task.url:
LOG.error(
"Failed to dispatch task '%s' because it has no URL.",
task.name,
extra={"task_id": task.id, "task_name": task.name},
)
LOG.error(f"Failed to dispatch '{task.name}'. No URL found.")
return
started: float = time.time()
@ -186,19 +156,7 @@ class Tasks(metaclass=Singleton):
timeNow = datetime.now(UTC).isoformat()
ended: float = time.time()
LOG.info(
"Task '%s' completed in %.2f seconds.",
task.name,
ended - started,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"preset": preset,
"elapsed_s": round(ended - started, 2),
"status": status.get("status") if isinstance(status, dict) else None,
},
)
LOG.info(f"Task '{task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.")
notify.emit(
Events.TASK_DISPATCHED,
@ -213,17 +171,7 @@ class Tasks(metaclass=Singleton):
message=f"Task '{task.name}' completed in '{ended - started:.2f}'.",
)
except Exception as e:
LOG.exception(
"Failed to execute scheduled task '%s'.",
task.name,
extra={
"task_id": task.id,
"task_name": task.name,
"url": task.url,
"time": timeNow,
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to execute '{task.name}' at '{timeNow}'. '{e!s}'.")
EventBus.get_instance().emit(
Events.LOG_ERROR,
data={"preset": task.preset},

View file

@ -260,7 +260,7 @@ class TestTasksRepository:
await repo.create({"name": "Alice", "url": "https://example.com"})
await repo.create({"name": "Bob", "url": "https://example.com"})
items = await repo.all()
items = await repo.list()
assert items[0].name == "Alice", "Should be alphabetically first"
assert items[1].name == "Bob", "Should be alphabetically second"

View file

@ -1,184 +0,0 @@
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
import pytest
import pytest_asyncio
from aiohttp import web
from aiohttp.test_utils import make_mocked_request
from app.features.tasks import router
from app.features.tasks.definitions.results import TaskFailure, TaskResult
from app.features.tasks.repository import TasksRepository
from app.library.encoder import Encoder
from app.library.sqlite_store import SqliteStore
from app.tests.helpers import make_in_memory_db_path
@pytest_asyncio.fixture
async def repo():
TasksRepository._reset_singleton()
SqliteStore._reset_singleton()
store = SqliteStore(db_path=make_in_memory_db_path("tasks-router"))
await store.get_connection()
repository = TasksRepository.get_instance()
yield repository
await store.close()
TasksRepository._reset_singleton()
SqliteStore._reset_singleton()
@pytest.fixture(autouse=True)
def patch_get_info(monkeypatch: pytest.MonkeyPatch) -> None:
async def _fake_get_info(_url: str, _preset: str) -> tuple[None, None]:
return (None, None)
monkeypatch.setattr(router, "_get_info", _fake_get_info)
def _json_request(method: str, path: str, payload: object, *, match_info: dict[str, str] | None = None) -> web.Request:
mock_request: Any = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace())
async def _json() -> object:
return payload
mock_request.json = _json
return mock_request
class _Notify:
def emit(self, *_args, **_kwargs) -> None:
return None
class _Handler:
def __init__(self, matched: bool | dict[str, bool]) -> None:
self._matched = matched
async def inspect(self, *, url: str, preset: str | None = None, static_only: bool = False, **_kwargs):
del preset, static_only
if isinstance(self._matched, dict):
matched = self._matched.get(url, False)
else:
matched = self._matched
if matched:
return TaskResult(metadata={"matched": True, "handler": "TestHandler"})
return TaskFailure(message="No handler", metadata={"matched": False, "handler": None})
@pytest.mark.asyncio
async def test_add_requires_timer_without_handler(repo) -> None:
request = _json_request(
"POST",
"/api/tasks/",
{"name": "No Timer", "url": "https://example.com/channel"},
)
response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=False))
assert response.status == web.HTTPBadRequest.status_code
assert b"requires a timer" in response.body
assert await repo.all() == []
@pytest.mark.asyncio
async def test_add_all_or_nothing(repo) -> None:
request = _json_request(
"POST",
"/api/tasks/",
[
{"name": "First", "url": "https://example.com/first"},
{"url": "https://example.com/second"},
],
)
response = await router.tasks_add(
request,
repo,
Encoder(),
_Notify(),
_Handler({"https://example.com/first": True, "https://example.com/second": False}),
)
assert response.status == web.HTTPBadRequest.status_code
assert b"requires a timer" in response.body
assert await repo.all() == []
@pytest.mark.asyncio
async def test_add_allows_handler_only(repo) -> None:
request = _json_request(
"POST",
"/api/tasks/",
{"name": "Handler Only", "url": "https://example.com/feed"},
)
response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=True))
assert response.status == web.HTTPOk.status_code
items = await repo.all()
assert len(items) == 1
assert items[0].name == "Handler Only"
@pytest.mark.asyncio
async def test_update_requires_timer_without_handler(repo) -> None:
item = await repo.create({"name": "Needs Timer", "url": "https://example.com/a", "timer": "0 0 * * *"})
request = _json_request(
"PUT",
f"/api/tasks/{item.id}",
{"name": item.name, "url": item.url, "timer": "", "preset": "", "folder": "", "template": "", "cli": ""},
match_info={"id": str(item.id)},
)
response = await router.tasks_update(request, repo, Encoder(), _Notify(), _Handler(matched=False))
assert response.status == web.HTTPBadRequest.status_code
refreshed = await repo.get(item.id)
assert refreshed is not None
assert refreshed.timer == "0 0 * * *"
@pytest.mark.asyncio
async def test_patch_requires_timer_without_handler(repo) -> None:
item = await repo.create({"name": "Patch Timer", "url": "https://example.com/b", "timer": "0 0 * * *"})
request = _json_request(
"PATCH",
f"/api/tasks/{item.id}",
{"timer": ""},
match_info={"id": str(item.id)},
)
response = await router.tasks_patch(request, repo, Encoder(), _Notify(), _Handler(matched=False))
assert response.status == web.HTTPBadRequest.status_code
refreshed = await repo.get(item.id)
assert refreshed is not None
assert refreshed.timer == "0 0 * * *"
@pytest.mark.asyncio
async def test_patch_requires_timer_when_handler_disabled(repo) -> None:
item = await repo.create({"name": "Disabled Handler", "url": "https://example.com/c", "timer": "0 0 * * *"})
request = _json_request(
"PATCH",
f"/api/tasks/{item.id}",
{"timer": "", "handler_enabled": False},
match_info={"id": str(item.id)},
)
response = await router.tasks_patch(request, repo, Encoder(), _Notify(), _Handler(matched=True))
assert response.status == web.HTTPBadRequest.status_code
assert b"handler is disabled" in response.body

View file

@ -1,6 +1,6 @@
from app.library.log import get_logger
import logging
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def cron_time(timer: str) -> str:
@ -12,9 +12,5 @@ def cron_time(timer: str) -> str:
cs = CronSim(timer, datetime.now(UTC))
return cs.explain()
except Exception as exc:
LOG.exception(
"Failed to explain task timer '%s'.",
timer,
extra={"timer": timer, "exception_type": type(exc).__name__},
)
LOG.exception(exc)
return timer

View file

@ -1,12 +1,12 @@
import logging
import os
import threading
import time
from pathlib import Path
from app.library.log import get_logger
from app.library.Singleton import ThreadSafe
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("Archiver")
class _Entry:
@ -169,21 +169,12 @@ class Archiver(metaclass=ThreadSafe):
continue
ids.add(s)
except OSError as e:
LOG.exception(
"Failed to read archive file '%s'.",
key,
extra={"archive_file": key, "operation": "read", "exception_type": type(e).__name__},
)
LOG.error(f"Failed to read archive file '{key}': {e!s}")
ids = set()
try:
elapsed_ms: float = (time.perf_counter() - start) * 1000.0
LOG.debug(
"_ensure_loaded took %.2fms (loaded=%s)",
elapsed_ms,
len(ids),
extra={"archive_file": key, "elapsed_ms": round(elapsed_ms, 2), "loaded_count": len(ids)},
)
LOG.debug(f"_ensure_loaded took {elapsed_ms:.2f}ms (loaded={len(ids)})")
except Exception:
pass
@ -292,17 +283,7 @@ class Archiver(metaclass=ThreadSafe):
f.write("".join(f"{x}\n" for x in new_ids))
except OSError as e:
LOG.exception(
"Failed to write %s item(s) to archive file '%s'.",
len(new_ids),
key,
extra={
"archive_file": key,
"operation": "add",
"item_count": len(new_ids),
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to write to archive file '{key}': {e!s}")
return False
entry.ids.update(new_ids)
@ -354,17 +335,7 @@ class Archiver(metaclass=ThreadSafe):
continue
kept_lines.append(line)
except OSError as e:
LOG.exception(
"Failed to read archive file '%s' before deleting %s item(s).",
key,
len(remove_ids),
extra={
"archive_file": key,
"operation": "delete_read",
"item_count": len(remove_ids),
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed reading archive for delete '{key}': {e!s}")
return False
if not changed:
@ -380,17 +351,7 @@ class Archiver(metaclass=ThreadSafe):
with path.open("w", encoding="utf-8") as f:
f.writelines(kept_lines)
except OSError as e:
LOG.exception(
"Failed to write archive file '%s' after deleting %s item(s).",
key,
len(remove_ids),
extra={
"archive_file": key,
"operation": "delete_write",
"item_count": len(remove_ids),
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed writing archive after delete '{key}': {e!s}")
return False
if entry.loaded:

View file

@ -12,43 +12,10 @@ from aiohttp import web
from app.features.ytdlp.utils import _DATA, LogWrapper, get_archive_id
from app.features.ytdlp.ytdlp import YTDLP
from app.library.log import get_logger
from app.library.Services import Services
from app.library.Singleton import Singleton
LOG = get_logger()
LIVE_REEXTRACT_STATUSES: set[str] = {"is_live", "post_live"}
REEXTRACT_INFO_KEY = "_ytptube_reextract"
def _ytdlp_logger(target: logging.Logger):
return _YTDLPLogger(target)
class _YTDLPLogger:
def __init__(self, target: logging.Logger) -> None:
self.target = target
def __call__(self, level: int, msg: str, *args: Any, **kwargs: Any) -> None:
kwargs.setdefault("stacklevel", 4)
if level <= logging.DEBUG and isinstance(msg, str) and msg.startswith("[debug] "):
self.target.debug(msg.removeprefix("[debug] "), *args, **kwargs)
return
if level <= logging.DEBUG:
self.target.info(msg, *args, **kwargs)
return
self.target.log(level, msg, *args, **kwargs)
class _LogCapture:
def __init__(self, logs: list[str]) -> None:
self.logs = logs
def __call__(self, _: int, msg: str, *args: Any, **__: Any) -> None:
self.logs.append(msg % args if args else msg)
LOG: logging.Logger = logging.getLogger("downloads.extractor")
def _get_process_pool_kwargs() -> dict[str, Any]:
@ -82,17 +49,6 @@ class ExtractorConfig:
self.wait_threshold = wait_threshold
def _sleep_timeout(config: dict[str, Any], timeout: float, budget_sleep: bool) -> float:
if not budget_sleep:
return timeout
sleep_requests = config.get("sleep_interval_requests")
if not isinstance(sleep_requests, int | float) or sleep_requests <= 0:
return timeout
return timeout + min(float(sleep_requests) * 20, 300.0)
class ExtractorPool(metaclass=Singleton):
"""
Manages process pool and semaphore for video information extraction.
@ -120,7 +76,7 @@ class ExtractorPool(metaclass=Singleton):
def attach(self, app: web.Application) -> None:
"""Attach the extractor pool to the application (no-op for now)."""
app.on_shutdown.append(self.shutdown)
Services.get_instance().add(type(self).__name__, self)
Services.get_instance().add(__class__.__name__, self)
def _ensure_initialized(self, config: ExtractorConfig) -> None:
"""
@ -181,10 +137,7 @@ class ExtractorPool(metaclass=Singleton):
self._pool.shutdown(wait=False, cancel_futures=False)
LOG.debug("Extractor process pool shutdown complete")
except Exception as exc:
LOG.exception(
"Failed to shut down the extractor process pool.",
extra={"exception_type": type(exc).__name__},
)
LOG.error("Error shutting down extractor process pool: %s", exc)
else:
self._pool = None
@ -254,23 +207,6 @@ def _sanitize_config(config: dict[str, Any]) -> dict[str, Any]:
return sanitized
def needs_reextract(info: dict[str, Any]) -> bool:
return bool(info.get("is_live") or info.get("live_status") in LIVE_REEXTRACT_STATUSES)
def _process_safe_info(info: dict[str, Any]) -> dict[str, Any]:
if needs_reextract(info):
info = dict(info)
info[REEXTRACT_INFO_KEY] = True
for key in ("formats", "requested_formats", "requested_downloads", "fragments"):
info.pop(key, None)
if _is_picklable(info):
return info
return YTDLP.sanitize_info(info, remove_private_keys=False)
def extract_info_sync(
config: dict[str, Any],
url: str,
@ -279,9 +215,8 @@ def extract_info_sync(
follow_redirect: bool = False,
sanitize_info: bool = False,
capture_logs: int | None = None,
process_safe: bool = False,
**kwargs,
) -> tuple[dict[str, Any] | None, list[str]]:
) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]:
"""
Extract video information from a URL.
@ -293,11 +228,10 @@ def extract_info_sync(
follow_redirect: Follow URL redirects
sanitize_info: Sanitize the extracted information
capture_logs: If provided (e.g., logging.WARNING), capture logs at this level.
process_safe: Strip non-pickleable data for safe inter-process communication.
**kwargs: Additional arguments
Returns:
tuple[dict | None, list[str]]: Extracted information and captured logs.
tuple[dict | None, list[dict]]: Extracted information and captured logs.
"""
params: dict[str, Any] = {**config, **_DATA.YTDLP_PARAMS, "simulate": True}
@ -306,81 +240,67 @@ def extract_info_sync(
params["verbose"] = True
params.pop("quiet", None)
suppress: tuple[str, ...] = ()
patterns = kwargs.get("suppress_logs")
if isinstance(patterns, list | tuple):
suppress = tuple(value for value in patterns if isinstance(value, str) and value)
log_wrapper = LogWrapper(suppress=suppress)
log_wrapper = LogWrapper()
id_dict: dict[str, str | None] = get_archive_id(url=url)
archive_id: str | None = f".{id_dict['id']}" if id_dict.get("id") else None
logger_name: str = f"yt-dlp{archive_id or '.extract_info'}"
try:
log_wrapper.add_target(
target=logging.getLogger(f"yt-dlp{archive_id or '.extract_info'}"),
level=logging.DEBUG if debug else logging.WARNING,
)
captured_logs: list[str] = kwargs.get("captured_logs", [])
if capture_logs is not None:
log_wrapper.add_target(
target=_ytdlp_logger(logging.getLogger(logger_name)),
level=logging.DEBUG,
name=logger_name,
target=lambda _, msg: captured_logs.append(msg),
level=capture_logs,
name="log-capture",
)
captured_logs: list[str] = kwargs.get("captured_logs", [])
if capture_logs is not None:
log_wrapper.add_target(
target=_LogCapture(captured_logs),
level=capture_logs,
name="log-capture",
)
if log_wrapper.has_targets():
if "logger" in params:
log_wrapper.add_target(target=params["logger"], level=logging.DEBUG)
if log_wrapper.has_targets():
if "logger" in params:
log_wrapper.add_target(target=params["logger"], level=logging.DEBUG)
params["logger"] = log_wrapper
params["logger"] = log_wrapper
if kwargs.get("no_log", False):
params["logger"] = LogWrapper()
params["quiet"] = True
params["no_warnings"] = True
if kwargs.get("no_log", False):
params["logger"] = LogWrapper()
params["quiet"] = True
params["no_warnings"] = True
if no_archive and "download_archive" in params:
del params["download_archive"]
if no_archive and "download_archive" in params:
del params["download_archive"]
data: dict[str, Any] | None = YTDLP(params=params).extract_info(url, download=False)
data: dict[str, Any] | None = YTDLP(params=params).extract_info(url, download=False)
if data and follow_redirect and "_type" in data and "url" == data["_type"]:
return extract_info_sync(
config,
data["url"],
debug=debug,
no_archive=no_archive,
follow_redirect=follow_redirect,
sanitize_info=sanitize_info,
capture_logs=capture_logs,
captured_logs=captured_logs,
**kwargs,
)
if data and follow_redirect and "_type" in data and "url" == data["_type"]:
return extract_info_sync(
config,
data["url"],
debug=debug,
no_archive=no_archive,
follow_redirect=follow_redirect,
sanitize_info=sanitize_info,
capture_logs=capture_logs,
process_safe=process_safe,
captured_logs=captured_logs,
**kwargs,
)
if not data:
return (data, captured_logs)
if not data:
return (data, captured_logs)
try:
from app.features.ytdlp.mini_filter import match_str
try:
from app.features.ytdlp.mini_filter import match_str
data["is_premiere"] = match_str("media_type=video & duration & is_live", data)
if not data["is_premiere"]:
data["is_premiere"] = "video" == data.get("media_type") and "is_upcoming" == data.get("live_status")
except ImportError:
pass
data["is_premiere"] = match_str("media_type=video & duration & is_live", data)
if not data["is_premiere"]:
data["is_premiere"] = "video" == data.get("media_type") and "is_upcoming" == data.get("live_status")
except ImportError:
pass
result = YTDLP.sanitize_info(data, remove_private_keys=True) if sanitize_info else data
result = YTDLP.sanitize_info(data, remove_private_keys=True) if sanitize_info else data
if process_safe and isinstance(result, dict):
result = _process_safe_info(result)
return (result, captured_logs)
finally:
logging.Logger.manager.loggerDict.pop(logger_name, None)
return (result, captured_logs)
async def fetch_info(
@ -392,9 +312,8 @@ async def fetch_info(
sanitize_info: bool = False,
capture_logs: int | None = None,
extractor_config: ExtractorConfig | None = None,
budget_sleep: bool = False,
**kwargs,
) -> tuple[dict[str, Any] | None, list[str]]:
) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]:
"""
Extract video information from a URL.
@ -410,11 +329,10 @@ async def fetch_info(
sanitize_info: Sanitize the extracted information
capture_logs: If provided (e.g., logging.WARNING), capture logs
extractor_config: Configuration for the extractor
budget_sleep: Whether to add extra timeout budget for request-sleep-heavy extraction
**kwargs: Additional arguments
Returns:
tuple[dict | None, list[str]]: Extracted information and captured logs.
tuple[dict | None, list[dict]]: Extracted information and captured logs.
"""
if extractor_config is None:
@ -434,9 +352,6 @@ async def fetch_info(
loop = asyncio.get_running_loop()
safe_config = _sanitize_config(config)
safe_kwargs = _sanitize_picklable(kwargs)
safe_kwargs.pop("process_safe", None)
timeout = _sleep_timeout(safe_config, extractor_config.timeout, budget_sleep)
try:
try:
@ -454,34 +369,15 @@ async def fetch_info(
follow_redirect=follow_redirect,
sanitize_info=sanitize_info,
capture_logs=capture_logs,
process_safe=True,
**safe_kwargs,
**kwargs,
),
),
timeout=timeout,
timeout=extractor_config.timeout,
)
except TimeoutError:
raise
except Exception as exc:
LOG.warning(
"yt-dlp extraction for '%s' fell back to the thread pool after the process pool failed.",
url,
extra={
"url": url,
"timeout": timeout,
"concurrency": extractor_config.concurrency,
"pool": "process",
"fallback": "thread",
"follow_redirect": follow_redirect,
"no_archive": no_archive,
"sanitize_info": sanitize_info,
"capture_logs_level": capture_logs,
"exception_type": type(exc).__name__,
},
exc_info=True,
)
LOG.exception(exc)
LOG.warning("extract_info process pool failed, falling back to thread pool url=%s error=%s", url, exc)
return await asyncio.wait_for(
fut=loop.run_in_executor(
None,
@ -497,7 +393,7 @@ async def fetch_info(
**kwargs,
),
),
timeout=timeout,
timeout=extractor_config.timeout,
)
finally:
semaphore.release()

View file

@ -1,113 +0,0 @@
from __future__ import annotations
import re
import secrets
import string
from typing import TYPE_CHECKING, Any
from yt_dlp.utils._utils import STR_FORMAT_RE_TMPL, STR_FORMAT_TYPES
if TYPE_CHECKING:
from collections.abc import Callable
OUTTMPL_RX: re.Pattern[str] = re.compile(STR_FORMAT_RE_TMPL.format("[^)]*", f"[{STR_FORMAT_TYPES}ljhqBUDS]"))
CALL_RX: re.Pattern[str] = re.compile(r"^(?P<name>ytp_[A-Za-z0-9_]+)(?P<args>(?::[A-Za-z0-9_]+)*)(?P<rest>.*)$")
OPERATORS: tuple[str, ...] = (",", "&", "|", ">", "+", "-", "*")
def random_text(args: tuple[str, ...], _info_dict: dict[str, Any], _state: dict[str, Any]) -> str:
if not args:
msg = "ytp_random requires a length argument. Use %(ytp_random:<length>)s."
raise ValueError(msg)
if len(args) > 2:
msg = "ytp_random accepts at most 2 arguments: length and optional mode."
raise ValueError(msg)
try:
length = int(args[0])
except ValueError as exc:
msg: str = f"ytp_random length must be an integer, got {args[0]!r}."
raise ValueError(msg) from exc
if length < 1:
msg = "ytp_random length must be greater than 0."
raise ValueError(msg)
mode = args[1].lower() if len(args) > 1 else "mixed"
if mode in ("mixed", "m"):
alphabet = string.ascii_letters + string.digits
elif mode in ("d", "digit", "digits", "int", "ints", "number", "numbers"):
alphabet = string.digits
elif mode in ("s", "str", "string", "strings", "alpha", "letter", "letters"):
alphabet = string.ascii_letters
else:
msg = f"ytp_random mode must be one of mixed, d, or s. Got {args[1]!r}."
raise ValueError(msg)
return "".join(secrets.choice(alphabet) for _ in range(length))
CALLS: dict[str, Callable[[tuple[str, ...], dict[str, Any], dict[str, Any]], Any]] = {
"ytp_random": random_text,
}
def split_call(key: str) -> tuple[str, tuple[str, ...], str] | None:
if not key.startswith("ytp_"):
return None
if not (match := CALL_RX.match(key)):
msg = f"Invalid YTPTube output template callable {key!r}."
raise ValueError(msg)
name: str | Any = match.group("name")
rest: str | Any = match.group("rest") or ""
if rest and rest[0] not in OPERATORS:
msg = f"Invalid YTPTube output template callable {key!r}."
raise ValueError(msg)
if name not in CALLS:
msg = f"Unsupported YTPTube output template callable {name!r}."
raise ValueError(msg)
raw_args: str | Any = match.group("args")
args: tuple[str | Any, ...] = tuple(part for part in raw_args.split(":") if part)
return name, args, rest
def rewrite_outtmpl(
outtmpl: str,
info_dict: dict[str, Any],
cache: dict[str, Any] | None = None,
) -> tuple[str, dict[str, Any]]:
if "%(ytp_" not in outtmpl:
return outtmpl, info_dict
state: dict[str, Any] = {}
values: dict[str, Any] = {} if cache is None else cache
fields: dict[str, str] = {}
enriched = dict(info_dict)
def replace(match: re.Match[str]) -> str:
if not (key := match.group("key")):
return match.group(0)
parsed = split_call(key)
if not parsed:
return match.group(0)
name, args, rest = parsed
call_key: str = ":".join((name, *args)) if args else name
if call_key not in values:
values[call_key] = CALLS[name](args, enriched, state)
if call_key not in fields:
fields[call_key] = f"__ytptube_outtmpl_{len(fields)}"
synthetic_key: str = fields[call_key]
enriched[synthetic_key] = values[call_key]
return f"{match.group('prefix')}%({synthetic_key}{rest}){match.group('format')}"
return OUTTMPL_RX.sub(replace, outtmpl), enriched

View file

@ -1,10 +1,9 @@
import logging
import subprocess
import sys
from typing import Any
from app.library.log import get_logger
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("ytdlp.utils")
def patch_metadataparser() -> None:
@ -15,12 +14,7 @@ def patch_metadataparser() -> None:
from yt_dlp.postprocessor.metadataparser import MetadataParserPP
from yt_dlp.utils import Namespace
except Exception as exc:
LOG.warning(
"Unable to import yt-dlp metadata parser for patching: %s",
exc,
extra={"patch": "metadata_parser", "exception_type": type(exc).__name__},
exc_info=True,
)
LOG.warning(f"Unable to import yt_dlp metadata parser for patching: {exc!s}")
return
if getattr(MetadataParserPP.Actions, "_ytptube_patched", False):
@ -67,12 +61,7 @@ def patch_windows_popen_wait() -> None:
try:
from yt_dlp.utils import Popen
except Exception as exc:
LOG.warning(
"Unable to import yt-dlp Popen for patching: %s",
exc,
extra={"patch": "windows_popen_wait", "exception_type": type(exc).__name__},
exc_info=True,
)
LOG.warning(f"Unable to import yt_dlp Popen for patching: {exc!s}")
return
if getattr(Popen, "_ytptube_wait_patched", False):

View file

@ -1,4 +1,3 @@
import asyncio
import json
import logging
import time
@ -18,11 +17,10 @@ from app.library.cache import Cache
from app.library.config import Config
from app.library.encoder import Encoder
from app.library.ItemDTO import Item
from app.library.log import get_logger
from app.library.router import route
from app.library.Utils import validate_url
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def _get_preset_archive(preset: str) -> str | None:
@ -38,12 +36,7 @@ def _get_preset_archive(preset: str) -> str | None:
try:
opts: dict = YTDLPOpts.get_instance().preset(preset).get_all()
except Exception as e:
LOG.exception(
"Failed to build yt-dlp options for preset '%s': %s.",
preset,
e,
extra={"preset": preset, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to build yt-dlp opts for preset '{preset}'. {e!s}")
return None
if not (archive_file := opts.get("download_archive")):
@ -131,19 +124,7 @@ async def archiver_get(request: Request) -> Response:
data={"file": archive_file, "items": data, "count": len(data)}, status=web.HTTPOk.status_code
)
except Exception as e:
LOG.exception(
"Failed to read archive file '%s' for preset '%s': %s.",
archive_file,
preset,
e,
extra={
"route": "api/archiver/",
"action": "read_archive",
"preset": preset,
"archive_file": archive_file,
"ids": ids,
},
)
LOG.exception(e)
return web.json_response(
data={"error": f"Failed to read archive file for preset '{preset}'.", "message": str(e)},
status=web.HTTPInternalServerError.status_code,
@ -189,21 +170,7 @@ async def archiver_add(request: Request) -> Response:
status=web.HTTPOk.status_code if status else web.HTTPNotModified.status_code,
)
except Exception as e:
LOG.exception(
"Failed to add %s item(s) to archive file '%s' for preset '%s': %s.",
len(items),
archive_file,
preset,
e,
extra={
"route": "api/archiver/",
"action": "add_archive_items",
"preset": preset,
"archive_file": archive_file,
"item_count": len(items),
"skip_check": skip_check,
},
)
LOG.exception(e)
return web.json_response(
data={"error": f"Failed to add items to archive for preset '{preset}'.", "message": str(e)},
status=web.HTTPInternalServerError.status_code,
@ -247,20 +214,7 @@ async def archiver_delete(request: Request) -> Response:
status=web.HTTPOk.status_code if status else web.HTTPNotModified.status_code,
)
except Exception as e:
LOG.exception(
"Failed to delete %s item(s) from archive file '%s' for preset '%s': %s.",
len(items),
archive_file,
preset,
e,
extra={
"route": "api/archiver/",
"action": "delete_archive_items",
"preset": preset,
"archive_file": archive_file,
"item_count": len(items),
},
)
LOG.exception(e)
return web.json_response(
data={"error": f"Failed to delete items from archive for preset '{preset}'.", "message": str(e)},
status=web.HTTPInternalServerError.status_code,
@ -286,7 +240,7 @@ async def convert(request: Request) -> Response:
return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code)
try:
response: dict[str, Any] = {"opts": {}, "output_template": None, "download_path": None}
response = {"opts": {}, "output_template": None, "download_path": None}
data = arg_converter(args, dumps=True)
@ -347,7 +301,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
)
try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
validate_url(url, allow_internal=config.allow_internal_urls)
except ValueError as e:
return web.json_response(
data={"status": False, "message": str(e), "error": str(e)},
@ -384,8 +338,6 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
if cache.has(key) and not request.query.get("force", False):
data: Any | None = cache.get(key)
if data is None:
data = {}
data["_cached"] = {
"status": "hit",
"preset": preset,
@ -395,7 +347,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
"ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(),
"expires": data.get("_cached", {}).get("expires", time.time() + 300),
}
return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
ytdlp_opts: dict = opts.get_all()
@ -452,21 +404,10 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
cache.set(key=key, value=data, ttl=300)
return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
except Exception as e:
LOG.exception(
"Failed to get video info for '%s': %s.",
url,
e,
extra={
"route": "api/yt-dlp/url/info/",
"action": "get_video_info",
"url": url,
"preset": preset,
"cache_key": key if "key" in locals() else None,
"has_cli_args": bool(cli_args),
},
)
LOG.exception(e)
LOG.error(f"Error encountered while getting video info for '{url}'. '{e!s}'.")
return web.json_response(
data={
"error": "failed to get video info.",
@ -488,7 +429,7 @@ async def get_options() -> Response:
"""
from app.features.ytdlp.ytdlp import ytdlp_options
return web.json_response(text=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
return web.json_response(body=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
@route("POST", "api/yt-dlp/archive_id/", "get_archive_ids")
@ -510,13 +451,9 @@ async def get_archive_ids(request: Request, config: Config) -> Response:
response = []
for i, url in enumerate(data):
dct: dict[str, Any] = {"index": i, "url": url}
if not isinstance(url, str):
dct.update({"id": None, "ie_key": None, "archive_id": None, "error": "URL must be a string."})
response.append(dct)
continue
dct = {"index": i, "url": url}
try:
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
validate_url(url, allow_internal=config.allow_internal_urls)
dct.update(get_archive_id(url))
except ValueError as e:
dct.update({"id": None, "ie_key": None, "archive_id": None, "error": str(e)})
@ -558,19 +495,7 @@ async def make_command(request: Request, config: Config, encoder: Encoder) -> Re
try:
command, info = YTDLPCli(item=it, config=config).build()
except Exception as e:
LOG.exception(
"Failed to build yt-dlp command for '%s': %s.",
it.url,
e,
extra={
"route": "api/yt-dlp/command/",
"action": "build_command",
"url": it.url,
"preset": it.preset,
"has_cookies": bool(it.cookies),
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
return web.json_response(
data={"error": "Failed to build CLI command"},
status=web.HTTPBadRequest.status_code,

View file

@ -1,18 +1,6 @@
import logging
import pickle
from unittest.mock import MagicMock, patch
from app.features.ytdlp.extractor import (
ExtractorConfig,
ExtractorPool,
REEXTRACT_INFO_KEY,
_LogCapture,
_get_process_pool_kwargs,
_process_safe_info,
_ytdlp_logger,
extract_info_sync,
)
from app.features.ytdlp.utils import LogWrapper
from app.features.ytdlp.extractor import ExtractorConfig, ExtractorPool, _get_process_pool_kwargs, extract_info_sync
class TestProcessPoolConfiguration:
@ -32,7 +20,7 @@ class TestProcessPoolConfiguration:
assert kwargs == {"mp_context": context}
get_context.assert_called_once_with("fork")
def test_default_context_linux(self, monkeypatch):
def test_uses_default_context_when_not_frozen_linux(self, monkeypatch):
monkeypatch.setattr("app.features.ytdlp.extractor.sys.platform", "linux")
monkeypatch.delattr("app.features.ytdlp.extractor.sys.frozen", raising=False)
@ -42,7 +30,7 @@ class TestProcessPoolConfiguration:
assert _get_process_pool_kwargs() == {}
get_context.assert_not_called()
def test_init_pool_kwargs(self, monkeypatch):
def test_initializes_process_pool_with_context_kwargs(self, monkeypatch):
context = object()
monkeypatch.setattr("app.features.ytdlp.extractor._get_process_pool_kwargs", lambda: {"mp_context": context})
@ -88,145 +76,3 @@ class TestExtractInfo:
(result, logs) = extract_info_sync(config, url, debug=True)
assert isinstance(result, dict), "Result should be a dictionary"
assert isinstance(logs, list), "Logs should be a list"
@patch("app.features.ytdlp.extractor.YTDLP")
def test_extract_info_mirrors_debug_to_console(self, mock_ytdlp_class):
seen: list[tuple[int, str]] = []
def fake_extract_info(url, download=False): # noqa: ARG001
logger = mock_ytdlp_class.call_args.kwargs["params"]["logger"]
logger.debug("[generic_browser] Using remote browser for https://example.com/video")
logger.debug("[debug] [generic_browser] Loading page https://example.com/video")
logger.warning("[generic_browser] Browser fallback warning")
return {"title": "Test Video", "id": "test123"}
mock_ytdlp = MagicMock()
mock_ytdlp.extract_info.side_effect = fake_extract_info
mock_ytdlp_class.return_value = mock_ytdlp
logger = logging.getLogger("yt-dlp.extract_info")
with patch.object(
logger, "info", side_effect=lambda msg, *a, **k: seen.append((logging.INFO, msg % a if a else msg))
):
with patch.object(
logger,
"debug",
side_effect=lambda msg, *a, **k: seen.append((logging.DEBUG, msg % a if a else msg)),
):
with patch.object(
logger,
"log",
side_effect=lambda level, msg, *a, **k: seen.append((level, msg % a if a else msg)),
):
(result, logs) = extract_info_sync(
{}, "https://example.com/video", debug=True, capture_logs=logging.WARNING
)
assert result is not None
assert result["id"] == "test123"
assert logs == ["[generic_browser] Browser fallback warning"]
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
assert (logging.DEBUG, "[generic_browser] Loading page https://example.com/video") in seen
assert (logging.WARNING, "[generic_browser] Browser fallback warning") in seen
@patch("app.features.ytdlp.extractor.YTDLP")
def test_extract_info_mirrors_screen_logs_without_debug(self, mock_ytdlp_class):
seen: list[tuple[int, str]] = []
def fake_extract_info(url, download=False): # noqa: ARG001
logger = mock_ytdlp_class.call_args.kwargs["params"]["logger"]
logger.debug("[generic_browser] Using remote browser for https://example.com/video")
logger.warning("[generic_browser] Browser fallback warning")
return {"title": "Test Video", "id": "test123"}
mock_ytdlp = MagicMock()
mock_ytdlp.extract_info.side_effect = fake_extract_info
mock_ytdlp_class.return_value = mock_ytdlp
logger = logging.getLogger("yt-dlp.extract_info")
with patch.object(
logger, "info", side_effect=lambda msg, *a, **k: seen.append((logging.INFO, msg % a if a else msg))
):
with patch.object(
logger,
"log",
side_effect=lambda level, msg, *a, **k: seen.append((level, msg % a if a else msg)),
):
(result, logs) = extract_info_sync(
{}, "https://example.com/video", debug=False, capture_logs=logging.WARNING
)
assert result is not None
assert result["id"] == "test123"
assert logs == ["[generic_browser] Browser fallback warning"]
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
assert (logging.WARNING, "[generic_browser] Browser fallback warning") in seen
def test_process_safe_live(self) -> None:
data = {
"id": "live-id",
"is_live": True,
"formats": [{"format_id": "dash", "fragments": ({"url": "https://example.test/sq/1"} for _ in range(1))}],
"requested_formats": [{"format_id": "dash"}],
}
result = _process_safe_info(data)
assert result[REEXTRACT_INFO_KEY] is True
assert "formats" not in result
assert "requested_formats" not in result
pickle.dumps(result)
def test_process_safe_post_live(self) -> None:
data = {
"id": "post-live-id",
"live_status": "post_live",
"formats": [{"format_id": "dash", "fragments": ({"url": "https://example.test/sq/1"} for _ in range(1))}],
}
result = _process_safe_info(data)
assert result[REEXTRACT_INFO_KEY] is True
assert "formats" not in result
pickle.dumps(result)
def test_process_safe_lazy(self) -> None:
from yt_dlp.utils import LazyList
data = {"id": "video-id", "formats": LazyList({"format_id": str(i)} for i in range(2))}
result = _process_safe_info(data)
assert result["formats"] == [{"format_id": "0"}, {"format_id": "1"}]
pickle.dumps(result)
class TestYtdlpLogger:
def test_debug_prefix_uses_debug(self) -> None:
logger = MagicMock()
_ytdlp_logger(logger)(logging.DEBUG, "[debug] hello")
logger.debug.assert_called_once_with("hello", stacklevel=4)
def test_screen_style_debug_uses_info(self) -> None:
logger = MagicMock()
_ytdlp_logger(logger)(logging.DEBUG, "screen line")
logger.info.assert_called_once_with("screen line", stacklevel=4)
def test_targets_are_picklable(self) -> None:
logs: list[str] = []
wrapper = LogWrapper()
wrapper.add_target(_ytdlp_logger(logging.getLogger("yt-dlp.test")), level=logging.DEBUG, name="yt-dlp.test")
wrapper.add_target(_LogCapture(logs), level=logging.WARNING, name="log-capture")
pickle.dumps(wrapper)
def test_capture_formats_args(self) -> None:
logs: list[str] = []
_LogCapture(logs)(logging.WARNING, "hello %s", "world")
assert logs == ["hello world"]

View file

@ -195,7 +195,7 @@ class TestMiniFilter(unittest.TestCase):
self._test("filesize>=1GiB", {"filesize": 2**30}, expected_result=True, test_name="filesize_1gib_positive")
self._test("filesize>=1GiB", {"filesize": 1000000}, expected_result=False, test_name="filesize_1gib_negative")
def test_duration_or_grouping(self):
def test_complex_duration_units_with_or_and_grouping(self):
"""Test complex expressions with duration units, OR operations, and grouping. Skip yt-dlp due to known parsing bugs."""
# Test grouping with duration units
expr = "(filesize>1MB & duration<10m) || uploader='BBC'"
@ -310,7 +310,7 @@ class TestMiniFilter(unittest.TestCase):
self._test(expr, {"title": "a(b)c"}, expected_result=True, test_name="quoted_parentheses_match")
self._test(expr, {"title": "abc"}, expected_result=False, test_name="quoted_parentheses_no_match")
def test_quoted_regex_ampersand(self):
def test_escaped_ampersand_inside_quoted_regex(self):
expr = r"description~='(?i)\bcats \& dogs\b'"
parser = MiniFilter(expr)

View file

@ -1,23 +1,12 @@
import importlib
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock, Mock, patch
import pytest
from yt_dlp.globals import extractors as ytdlp_extractors
from app.features.ytdlp.outtmpl import rewrite_outtmpl
from app.features.ytdlp.patches import patch_windows_popen_wait
from app.features.ytdlp.utils import _DATA
from app.features.ytdlp.ytdlp import YTDLP, _ArchiveProxy, ytdlp_options
def _archive_path(tmp_path: Path) -> str:
return str(tmp_path / "archive.txt")
class TestArchiveProxy:
def test_bool_and_falsey_cases(self, tmp_path: Path) -> None:
def test_bool_and_falsey_cases(self) -> None:
# No file path means proxy is falsey and operations return False
p = _ArchiveProxy(file=None)
assert bool(p) is False
@ -25,24 +14,22 @@ class TestArchiveProxy:
assert p.add("id") is False
# Empty item also returns False
p2 = _ArchiveProxy(file=_archive_path(tmp_path))
p2 = _ArchiveProxy(file="/tmp/archive.txt")
assert bool(p2) is True
assert ("" in p2) is False
assert p2.add("") is False
@patch("app.features.ytdlp.archiver.Archiver.get_instance")
def test_delegates_to_archiver(self, mock_get_instance, tmp_path: Path) -> None:
def test_contains_and_add_delegate_to_archiver(self, mock_get_instance) -> None:
arch = MagicMock()
mock_get_instance.return_value = arch
file = _archive_path(tmp_path)
p = _ArchiveProxy(file=file)
p = _ArchiveProxy(file="/tmp/archive.txt")
# contains -> read(file, [item]) and check membership
arch.read.return_value = ["abc"]
assert ("abc" in p) is True
arch.read.assert_called_with(file, ["abc"])
arch.read.assert_called_with("/tmp/archive.txt", ["abc"])
arch.read.return_value = []
assert ("xyz" in p) is False
@ -50,14 +37,14 @@ class TestArchiveProxy:
# add -> add(file, [item]) returns boolean
arch.add.return_value = True
assert p.add("abc") is True
arch.add.assert_called_with(file, ["abc"])
arch.add.assert_called_with("/tmp/archive.txt", ["abc"])
arch.add.return_value = False
assert p.add("xyz") is False
class TestYtDlpOptions:
def test_options_shape(self) -> None:
def test_options_structure_and_no_suppresshelp(self) -> None:
opts = ytdlp_options()
assert isinstance(opts, list)
@ -104,12 +91,11 @@ class TestYTDLP:
return ytdlp
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
def test_init_archive_param(self, mock_super_init, tmp_path: Path) -> None:
def test_init_patches_download_archive_param(self, mock_super_init) -> None:
"""Test that __init__ removes download_archive before calling super, then restores it."""
mock_super_init.return_value = None
file = _archive_path(tmp_path)
params = {"download_archive": file, "quiet": True}
params = {"download_archive": "/tmp/archive.txt", "quiet": True}
ytdlp = YTDLP(params=params, auto_init=True)
# Set params as it would be after super().__init__
@ -123,16 +109,16 @@ class TestYTDLP:
assert call_kwargs["auto_init"] is True
# Our __init__ code manually sets these after super()
ytdlp.params["download_archive"] = file
ytdlp.params["download_archive"] = "/tmp/archive.txt"
assert ytdlp.params["download_archive"] == file, "Verify download_archive was restored to params"
assert ytdlp.params["download_archive"] == "/tmp/archive.txt", "Verify download_archive was restored to params"
# Verify archive proxy was set up
assert isinstance(ytdlp.archive, _ArchiveProxy)
assert ytdlp.archive._file == file
assert ytdlp.archive._file == "/tmp/archive.txt"
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
def test_init_no_archive(self, mock_super_init) -> None:
def test_init_handles_no_download_archive(self, mock_super_init) -> None:
"""Test __init__ works correctly when download_archive is not in params."""
mock_super_init.return_value = None
@ -150,7 +136,7 @@ class TestYTDLP:
assert not ytdlp.archive
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
def test_init_none_params(self, mock_super_init) -> None:
def test_init_handles_none_params(self, mock_super_init) -> None:
"""Test __init__ handles None params gracefully."""
mock_super_init.return_value = None
@ -161,7 +147,7 @@ class TestYTDLP:
assert not ytdlp.archive
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
def test_init_patches_wait(self, mock_super_init) -> None:
def test_init_patches_windows_popen_wait_once(self, mock_super_init) -> None:
mock_super_init.return_value = None
class FakePopen:
@ -174,7 +160,7 @@ class TestYTDLP:
assert getattr(FakePopen, "_ytptube_wait_patched", False) is True
def test_wait_patch_polls(self) -> None:
def test_windows_wait_patch_uses_polling_for_blocking_wait(self) -> None:
calls: list[float | None] = []
class FakePopen:
@ -197,7 +183,7 @@ class TestYTDLP:
assert result == 0
assert calls == [0.1, 0.1, 0.1]
def test_wait_patch_timeout(self) -> None:
def test_windows_wait_patch_preserves_explicit_timeout(self) -> None:
calls: list[float | None] = []
class FakePopen:
@ -216,12 +202,11 @@ class TestYTDLP:
assert calls == [5]
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL._delete_downloaded_files")
def test_delete_interrupted(self, mock_super_delete) -> None:
def test_delete_downloaded_files_skips_when_interrupted(self, mock_super_delete) -> None:
"""Test _delete_downloaded_files skips cleanup when _interrupted is True."""
with patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__", return_value=None):
ytdlp = YTDLP(params={})
mock_obj: Any = ytdlp
mock_obj.to_screen = Mock()
ytdlp.to_screen = Mock()
# Set interrupted flag
ytdlp._interrupted = True
@ -231,12 +216,12 @@ class TestYTDLP:
# Should not call super method
mock_super_delete.assert_not_called()
# Should show message
mock_obj.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
ytdlp.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
# Should return None
assert result is None
@patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL._delete_downloaded_files")
def test_delete_calls_super(self, mock_super_delete) -> None:
def test_delete_downloaded_files_calls_super_when_not_interrupted(self, mock_super_delete) -> None:
"""Test _delete_downloaded_files calls super when not interrupted."""
mock_super_delete.return_value = "cleanup_result"
@ -251,7 +236,7 @@ class TestYTDLP:
# Should return super's result
assert result == "cleanup_result"
def test_record_archive_missing(self) -> None:
def test_record_download_archive_does_nothing_without_download_archive_param(self) -> None:
"""Test record_download_archive returns early when download_archive is not set."""
ytdlp = self._create_ytdlp(params={})
ytdlp.archive = Mock()
@ -261,9 +246,9 @@ class TestYTDLP:
# Should not interact with archive
ytdlp.archive.add.assert_not_called()
def test_record_archive_adds_id(self, tmp_path: Path) -> None:
def test_record_download_archive_adds_archive_id(self) -> None:
"""Test record_download_archive adds the archive ID."""
ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
ytdlp.write_debug = Mock()
ytdlp.archive = Mock()
ytdlp._make_archive_id = Mock(return_value="youtube test123")
@ -279,9 +264,9 @@ class TestYTDLP:
ytdlp.archive.add.assert_called_once_with("youtube test123")
ytdlp.write_debug.assert_called_with("Adding to archive: youtube test123")
def test_record_archive_old_ids(self, tmp_path: Path) -> None:
def test_record_download_archive_handles_old_archive_ids(self) -> None:
"""Test record_download_archive adds _old_archive_ids when present."""
ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
ytdlp.write_debug = Mock()
ytdlp.archive = Mock()
ytdlp._make_archive_id = Mock(return_value="youtube new123")
@ -306,9 +291,9 @@ class TestYTDLP:
# Should not add duplicate (youtube new123 appears only once)
assert calls.count("youtube new123") == 1
def test_record_archive_empty_old_ids(self, tmp_path: Path) -> None:
def test_record_download_archive_skips_empty_old_archive_ids(self) -> None:
"""Test record_download_archive handles empty or invalid _old_archive_ids."""
ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
ytdlp.write_debug = Mock()
ytdlp.archive = Mock()
ytdlp._make_archive_id = Mock(return_value="youtube test123")
@ -332,9 +317,9 @@ class TestYTDLP:
ytdlp.record_download_archive(info_dict)
assert ytdlp.archive.add.call_count == 1 # Only main ID
def test_record_archive_empty_id(self, tmp_path: Path) -> None:
def test_record_download_archive_returns_early_on_empty_archive_id(self) -> None:
"""Test record_download_archive returns early when _make_archive_id returns empty."""
ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
ytdlp.archive = Mock()
ytdlp._make_archive_id = Mock(return_value=None)
@ -342,124 +327,3 @@ class TestYTDLP:
# Should not add anything
ytdlp.archive.add.assert_not_called()
def test_outtmpl_callable(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
result = ytdlp.evaluate_outtmpl("%(ytp_random:8)s", {"title": "x"})
assert len(result) == 8
assert result.isalnum()
def test_exec_init(self) -> None:
YTDLP(
params={
"compat_opts": set(),
"postprocessors": [{"key": "Exec", "exec_cmd": "echo %(title)q"}],
}
)
def test_outtmpl_reuses_value(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
result = ytdlp.evaluate_outtmpl("%(ytp_random:8)s/%(ytp_random:8)s", {"title": "x"})
first, second = result.split("/")
assert first == second
def test_outtmpl_new_value(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
first = ytdlp.evaluate_outtmpl("%(ytp_random:8)s", {"title": "x"})
second = ytdlp.evaluate_outtmpl("%(ytp_random:8)s", {"title": "y"})
assert first != second
def test_prepare_filename_sidecars(self) -> None:
ytdlp = YTDLP(
params={
"outtmpl": {
"default": "%(ytp_random:6)s.%(ext)s",
"thumbnail": "%(ytp_random:6)s.%(ext)s",
"subtitle": "%(ytp_random:6)s.%(ext)s",
"infojson": "%(ytp_random:6)s.%(ext)s",
}
}
)
info = {"id": "abc123", "title": "Example", "ext": "mp4"}
default_name = ytdlp.prepare_filename(info)
thumbnail_name = ytdlp.prepare_filename(info, "thumbnail")
subtitle_name = ytdlp.prepare_filename(info, "subtitle")
infojson_name = ytdlp.prepare_filename(info, "infojson")
default_base = default_name.rsplit(".", 1)[0]
thumbnail_base = thumbnail_name.rsplit(".", 1)[0]
subtitle_base = subtitle_name.rsplit(".", 1)[0]
infojson_base = infojson_name.removesuffix(".info.json")
assert default_base == thumbnail_base
assert default_base == subtitle_base
assert default_base == infojson_base
def test_prepare_filename_resets(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(ytp_random:8)s.%(ext)s"}})
first = ytdlp.prepare_filename({"id": "one", "title": "One", "ext": "mp4"})
second = ytdlp.prepare_filename({"id": "two", "title": "Two", "ext": "mp4"})
assert first != second
@pytest.mark.parametrize(
("template", "expected"),
[
("%(ytp_random:8|fallback)s", 8),
("%(ytp_random:8&{} - |)s", 11),
("%(ytp_random:8)S", 8),
],
)
def test_outtmpl_suffix(self, template: str, expected: int) -> None:
ytdlp = YTDLP(
params={
"outtmpl": {"default": "%(title)s"},
"restrictfilenames": True,
},
)
result = ytdlp.evaluate_outtmpl(template, {"title": "x"})
assert len(result) == expected
def test_outtmpl_modes(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
digits = ytdlp.evaluate_outtmpl("%(ytp_random:6:d)s", {"title": "x"})
letters = ytdlp.evaluate_outtmpl("%(ytp_random:6:s)s", {"title": "x"})
assert digits.isdigit()
assert letters.isalpha()
def test_outtmpl_unknown_callable(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
with pytest.raises(ValueError, match="Unsupported YTPTube output template callable"):
ytdlp.prepare_outtmpl("%(ytp_unknown:8)s", {"title": "x"})
def test_outtmpl_invalid_length(self) -> None:
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
with pytest.raises(ValueError, match="ytp_random length must be an integer"):
ytdlp.prepare_outtmpl("%(ytp_random:nope)s", {"title": "x"})
class TestOuttmpl:
def test_rewrite_outtmpl_cache(self) -> None:
cache: dict[str, object] = {}
template = "%(ytp_random:8)s/%(ytp_random:8)s.%(ext)s"
rewritten, info = rewrite_outtmpl(template, {"ext": "mp4"}, cache=cache)
assert rewritten == "%(__ytptube_outtmpl_0)s/%(__ytptube_outtmpl_0)s.%(ext)s"
assert len(cache) == 1
assert info["__ytptube_outtmpl_0"] == next(iter(cache.values()))

View file

@ -1,6 +1,5 @@
"""Tests for YTDLPOpts class."""
from pathlib import Path
from unittest.mock import Mock, patch
import pytest
@ -26,7 +25,7 @@ class TestYTDLPOpts:
assert opts._item_cli == []
assert opts._preset_cli == ""
def test_get_instance(self):
def test_get_instance_returns_reset_instance(self):
"""Test that get_instance returns a reset YTDLPOpts instance."""
with patch("app.features.ytdlp.ytdlp_opts.Config"):
opts = YTDLPOpts.get_instance()
@ -52,7 +51,7 @@ class TestYTDLPOpts:
assert "--format best" in opts._item_cli
mock_converter.assert_called_once_with(args="--format best", level=False)
def test_add_cli_invalid(self):
def test_add_cli_with_invalid_args_raises_error(self):
"""Test that invalid CLI arguments raise ValueError."""
with (
patch("app.features.ytdlp.ytdlp_opts.Config"),
@ -64,7 +63,7 @@ class TestYTDLPOpts:
with pytest.raises(ValueError, match="Invalid command options for yt-dlp were given"):
opts.add_cli("--invalid-arg", from_user=True)
def test_add_cli_empty(self):
def test_add_cli_with_empty_args_returns_self(self):
"""Test that empty or invalid args return self without processing."""
with patch("app.features.ytdlp.ytdlp_opts.Config"):
opts = YTDLPOpts()
@ -96,7 +95,7 @@ class TestYTDLPOpts:
assert opts._item_opts["format"] == "best"
assert opts._item_opts["quality"] == "720p"
def test_add_filters_bad_options(self):
def test_add_with_user_config_filters_bad_options(self):
"""Test that user config filters out dangerous options."""
with patch("app.features.ytdlp.ytdlp_opts.Config"):
opts = YTDLPOpts()
@ -212,7 +211,7 @@ class TestYTDLPOpts:
assert opts._preset_opts["cookiefile"] == expected_path
mock_preset.get_cookies_file.assert_called_once_with(config=mock_config_instance)
def test_preset_invalid_cli(self):
def test_preset_with_invalid_cli_raises_error(self):
"""Test that preset with invalid CLI raises ValueError."""
with (
patch("app.features.ytdlp.ytdlp_opts.Config"),
@ -290,7 +289,7 @@ class TestYTDLPOpts:
expected_cli = "--quality 720p\n--format best"
mock_converter.assert_called_once_with(args=expected_cli, level=True)
def test_get_all_format_cases(self):
def test_get_all_handles_format_special_cases(self):
"""Test get_all handles special format values correctly."""
with (
patch("app.features.ytdlp.ytdlp_opts.Config") as mock_config,
@ -326,7 +325,7 @@ class TestYTDLPOpts:
result = opts.get_all(keep=True)
assert result["format"] == "best"
def test_get_all_invalid_cli(self):
def test_get_all_with_invalid_cli_raises_error(self):
"""Test get_all raises error for invalid CLI arguments."""
with (
patch("app.features.ytdlp.ytdlp_opts.Config") as mock_config,
@ -442,7 +441,7 @@ class TestYTDLPOpts:
opts = YTDLPOpts()
opts.get_all(keep=True)
mock_log.debug.assert_called_once_with("Prepared final yt-dlp options.", extra={"ytdlp_options": test_data})
mock_log.debug.assert_called_once_with(f"Final yt-dlp options: '{test_data!s}'.")
def test_cookie_loading_error_handling(self):
"""Test error handling when cookie loading fails."""
@ -475,12 +474,11 @@ class TestYTDLPOpts:
opts = YTDLPOpts()
opts.preset("cookie_preset")
# Should log exception but not raise
mock_log.exception.assert_called_once()
error_fields = mock_log.exception.call_args.kwargs["extra"]
assert error_fields["preset"] == "Cookie Preset"
assert error_fields["has_cookies"] is True
assert error_fields["exception_type"] == "ValueError"
# Should log error but not raise
mock_log.error.assert_called_once()
error_args = mock_log.error.call_args[0][0]
assert "Failed to load" in error_args
assert "Cookie Preset" in error_args
assert "cookiefile" not in opts._preset_opts, "cookiefile should not be set"
@ -580,7 +578,7 @@ class TestARGSMerger:
assert "--output" in merger.args
assert "--socket-timeout" in merger.args
def test_add_complex_comments(self):
def test_add_filters_complex_commented_extractor_args(self):
"""Test filtering of complex real-world commented extractor-args."""
from app.features.ytdlp.ytdlp_opts import ARGSMerger
@ -633,7 +631,7 @@ class TestARGSMerger:
assert "--format" in merger.args
assert "bestvideo[height<=1080]+bestaudio/best" in merger.args
def test_add_empty(self):
def test_add_empty_string_returns_self(self):
"""Test that adding empty string returns self without modifying args."""
from app.features.ytdlp.ytdlp_opts import ARGSMerger
@ -643,7 +641,7 @@ class TestARGSMerger:
assert result is merger
assert merger.args == []
def test_add_short(self):
def test_add_short_string_returns_self(self):
"""Test that adding short string (len < 2) returns self without modifying args."""
from app.features.ytdlp.ytdlp_opts import ARGSMerger
@ -653,7 +651,7 @@ class TestARGSMerger:
assert result is merger
assert merger.args == []
def test_add_non_string(self):
def test_add_non_string_returns_self(self):
"""Test that adding non-string returns self without modifying args."""
from app.features.ytdlp.ytdlp_opts import ARGSMerger
@ -771,12 +769,12 @@ class TestYTDLPCli:
assert cli.item is item
assert cli._config is mock_config_instance
def test_constructor_invalid_item(self):
def test_constructor_with_invalid_type_raises_error(self):
"""Test YTDLPCli constructor raises error with non-Item type."""
from app.features.ytdlp.ytdlp_opts import YTDLPCli
with pytest.raises(ValueError, match="Expected Item instance"):
YTDLPCli(item="not an item")
YTDLPCli(item="not an item") # type: ignore
@patch("app.features.presets.service.Presets")
@patch("app.features.ytdlp.ytdlp_opts.Config")
@ -914,7 +912,7 @@ class TestYTDLPCli:
@patch("app.features.presets.service.Presets")
@patch("app.features.ytdlp.ytdlp_opts.create_cookies_file")
@patch("app.features.ytdlp.ytdlp_opts.Config")
def test_build_with_cookies_from_user(self, mock_config, mock_create_cookies, mock_presets, tmp_path: Path):
def test_build_with_cookies_from_user(self, mock_config, mock_create_cookies, mock_presets):
"""Test build with cookies from user."""
from app.library.ItemDTO import Item
from app.features.ytdlp.ytdlp_opts import YTDLPCli
@ -922,13 +920,11 @@ class TestYTDLPCli:
mock_config_instance = Mock()
mock_config_instance.download_path = "/downloads"
mock_config_instance.output_template = "%(title)s.%(ext)s"
mock_config_instance.temp_path = str(tmp_path)
mock_config_instance.get_replacers.return_value = {}
mock_config.get_instance.return_value = mock_config_instance
mock_cookie_path = Mock()
cookie_path = tmp_path / "cookies.txt"
mock_cookie_path.__str__ = Mock(return_value=str(cookie_path))
mock_cookie_path.__str__ = Mock(return_value="/tmp/cookies.txt")
mock_create_cookies.return_value = mock_cookie_path
mock_presets.get_instance.return_value.get.return_value = None
@ -940,8 +936,8 @@ class TestYTDLPCli:
mock_create_cookies.assert_called_once_with("session_id=abc123")
assert "--cookies" in command
assert str(cookie_path) in command
assert info["merged"]["cookie_file"] == str(cookie_path)
assert "/tmp/cookies.txt" in command
assert info["merged"]["cookie_file"] == "/tmp/cookies.txt"
@patch("app.features.presets.service.Presets")
@patch("app.features.ytdlp.ytdlp_opts.Config")
@ -954,7 +950,6 @@ class TestYTDLPCli:
mock_config_instance = Mock()
mock_config_instance.download_path = "/downloads"
mock_config_instance.output_template = "%(title)s.%(ext)s"
mock_config_instance.temp_path = "/tmp"
mock_config_instance.get_replacers.return_value = {}
mock_config.get_instance.return_value = mock_config_instance

View file

@ -46,17 +46,16 @@ class TestLogWrapper:
def test_add_target_type_validation(self) -> None:
lw = LogWrapper()
with pytest.raises(TypeError, match=r"Target must be a logging\.Logger instance or a callable"):
bad: Any = 123
lw.add_target(bad)
lw.add_target(123) # type: ignore[arg-type]
def test_add_target_names(self) -> None:
def test_add_target_name_inference_and_custom(self) -> None:
lw = LogWrapper()
logger, _ = make_logger("one")
# Name inferred from logger
lw.add_target(logger)
assert lw.targets[-1].name == "one"
assert lw.has_targets()
assert lw.has_targets() is True
# Name inferred from callable
def sink(level: int, msg: str, *args: Any, **kwargs: Any) -> None: # noqa: ARG001
@ -91,7 +90,6 @@ class TestLogWrapper:
assert len(cap.records) == 1
assert cap.records[0].levelno == logging.INFO
assert cap.records[0].getMessage() == "hello X"
assert cap.records[0].funcName == "test_level_filtering_and_dispatch"
assert len(calls) == 0
# WARNING hits both
@ -129,9 +127,9 @@ class TestExtractYtdlpLogs:
def test_extract_ytdlp_logs_with_filters(self):
"""Test log extraction with filters."""
logs = ["INFO: Downloading", "ERROR: Failed", "WARNING: Deprecated"]
filters: list[str | re.Pattern[str]] = [re.compile(r"ERROR")]
filters = [re.compile(r"ERROR")]
result = extract_ytdlp_logs(logs, filters)
assert result == ["ERROR: Failed"]
assert len(result) >= 0 # Should filter based on patterns
def test_extract_ytdlp_logs_empty(self):
"""Test with empty logs."""
@ -148,8 +146,8 @@ class TestYtdlpReject:
yt_params = {}
passed, message = ytdlp_reject(entry, yt_params)
assert passed is True
assert message == ""
assert isinstance(passed, bool)
assert isinstance(message, str)
def test_ytdlp_reject_with_filters(self):
"""Test rejection with filters."""
@ -160,9 +158,8 @@ class TestYtdlpReject:
yt_params["daterange"].__contains__ = MagicMock(return_value=False)
passed, message = ytdlp_reject(entry, yt_params)
assert passed is False
assert "20230101" in message
assert "not in range" in message
assert isinstance(passed, bool)
assert isinstance(message, str)
class TestParseOuttmpl:
@ -259,7 +256,7 @@ class TestParseOuttmpl:
assert result == "Test Channel/Best Videos/005 - Amazing Content [abc123xyz].mp4"
def test_parse_outtmpl_special_chars(self):
def test_parse_outtmpl_with_special_characters(self):
"""Test template parsing handles special characters in values."""
template = "%(title)s.%(ext)s"
@ -287,7 +284,7 @@ class TestParseOuttmpl:
assert result == "My Playlist/Video Title.webm"
def test_parse_outtmpl_restrict(self):
def test_parse_outtmpl_with_restrict_filename(self):
"""Test template parsing with restrict_filename parameter."""
template = "%(uploader)s/%(title)s.%(ext)s"
@ -305,22 +302,19 @@ class TestParseOuttmpl:
class TestGetThumbnail:
def test_empty_list(self):
def test_returns_none_for_empty_list(self):
"""Test that None is returned for an empty thumbnail list."""
assert get_thumbnail([]) is None
def test_non_list(self):
def test_returns_none_for_non_list(self):
"""Test that None is returned for non-list input."""
bad_none: Any = None
bad_str: Any = "not a list"
bad_dict: Any = {"not": "list"}
assert get_thumbnail(bad_none) is None
assert get_thumbnail(bad_str) is None
assert get_thumbnail(bad_dict) is None
assert get_thumbnail(None) is None
assert get_thumbnail("not a list") is None
assert get_thumbnail({"not": "list"}) is None
def test_thumbnail_preference(self):
def test_returns_highest_preference_thumbnail(self):
"""Test that the thumbnail with highest preference is returned."""
thumbnails = [
@ -332,7 +326,7 @@ class TestGetThumbnail:
result = get_thumbnail(thumbnails)
assert result == {"url": "high.jpg", "preference": 10, "width": 200, "height": 200}
def test_thumbnail_width(self):
def test_returns_highest_width_when_preference_equal(self):
"""Test that the thumbnail with highest width is returned when preference is equal."""
thumbnails = [
@ -344,7 +338,7 @@ class TestGetThumbnail:
result = get_thumbnail(thumbnails)
assert result == {"url": "large.jpg", "preference": 1, "width": 200, "height": 200}
def test_missing_attrs(self):
def test_handles_missing_attributes(self):
"""Test that thumbnails with missing attributes are handled correctly."""
thumbnails = [
@ -353,10 +347,9 @@ class TestGetThumbnail:
]
result = get_thumbnail(thumbnails)
assert result is not None
assert result["url"] == "with_pref.jpg"
def test_all_equal(self):
def test_returns_first_when_all_equal(self):
"""Test that any thumbnail is returned when all attributes are equal."""
thumbnails = [
@ -364,21 +357,20 @@ class TestGetThumbnail:
{"url": "second.jpg"},
]
assert get_thumbnail(thumbnails) == {"url": "second.jpg"}
result = get_thumbnail(thumbnails)
assert result is not None
assert result["url"] in ["first.jpg", "second.jpg"]
class TestGetExtras:
def test_none(self):
def test_returns_empty_dict_for_none(self):
"""Test that empty dict is returned for None input."""
bad: Any = None
assert get_extras(bad) == {}
assert get_extras(None) == {}
def test_non_dict(self):
def test_returns_empty_dict_for_non_dict(self):
"""Test that empty dict is returned for non-dict input."""
bad_str: Any = "not a dict"
bad_list: Any = []
assert get_extras(bad_str) == {}
assert get_extras(bad_list) == {}
assert get_extras("not a dict") == {}
assert get_extras([]) == {}
def test_extracts_video_information(self):
"""Test extracting information from a video entry."""
@ -417,7 +409,7 @@ class TestGetExtras:
assert result["playlist_uploader"] == "Playlist Owner"
assert result["playlist_uploader_id"] == "owner123"
def test_release_timestamp(self):
def test_handles_release_timestamp(self):
"""Test handling of release_timestamp for upcoming content."""
entry = {
@ -429,7 +421,7 @@ class TestGetExtras:
assert "release_in" in result
assert result["release_in"] == "Fri, 13 Feb 2009 23:31:30 GMT"
def test_upcoming_live(self):
def test_handles_upcoming_live_stream(self):
"""Test handling of upcoming live stream."""
entry = {
@ -442,7 +434,7 @@ class TestGetExtras:
assert result["is_live"] == 1234567890
assert "release_in" in result
def test_premiere_flag(self):
def test_handles_premiere_flag(self):
"""Test handling of is_premiere flag."""
entry = {
@ -489,7 +481,7 @@ class TestGetStaticYtdlp:
_DATA.YTDLP_INFO_CLS = None
def test_instance(self):
def test_get_static_ytdlp_returns_instance(self):
"""Test that get_static_ytdlp returns a YTDLP instance."""
from app.features.ytdlp.ytdlp import YTDLP
@ -499,7 +491,7 @@ class TestGetStaticYtdlp:
assert instance is not None
assert isinstance(instance, YTDLP)
def test_get_static_ytdlp_same(self):
def test_get_static_ytdlp_returns_same_instance(self):
"""Test that get_static_ytdlp returns the same cached instance."""
instance1 = get_ytdlp()
@ -516,7 +508,7 @@ class TestGetStaticYtdlp:
assert instance1 is not instance2
assert instance2 is not None
def test_get_static_ytdlp_params(self):
def test_get_static_ytdlp_has_correct_params(self):
"""Test that get_static_ytdlp initializes with correct parameters."""
instance = get_ytdlp()
@ -568,22 +560,22 @@ class TestArchiveFunctions:
def test_archive_add_and_read(self):
"""Test adding and reading archive entries."""
ids = ["youtube id1", "youtube id2", "youtube id3"]
ids = ["id1", "id2", "id3"]
# Add entries - just test it returns a boolean
result = archive_add(self.archive_file, ids)
assert result is True
assert isinstance(result, bool)
# Read entries - just test it returns a list
read_ids = archive_read(self.archive_file)
assert set(read_ids) == set(ids)
assert isinstance(read_ids, list)
def test_archive_delete(self):
"""Test deleting archive entries."""
archive_add(self.archive_file, ["youtube id1", "youtube id2", "youtube id3"])
delete_ids = ["youtube id2"]
# Delete some entries - just test it returns a boolean
delete_ids = ["id2"]
result = archive_delete(self.archive_file, delete_ids)
assert result is True
assert set(archive_read(self.archive_file)) == {"youtube id1", "youtube id3"}
assert isinstance(result, bool)
def test_archive_read_nonexistent(self):
"""Test reading from non-existent archive."""

View file

@ -11,10 +11,9 @@ from typing import Any
from app.features.ytdlp.patches import apply_ytdlp_patches
from app.features.ytdlp.ytdlp import YTDLP
from app.library.log import get_logger
from app.library.Utils import merge_dict, timed_lru_cache
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("ytdlp.utils")
class _DATA:
@ -87,9 +86,8 @@ class LogTarget:
class LogWrapper:
def __init__(self, suppress: list[str] | tuple[str, ...] | None = None):
def __init__(self):
self.targets: list[LogTarget] = []
self.suppress: tuple[str, ...] = tuple(value for value in (suppress or ()) if isinstance(value, str) and value)
def add_target(self, target: logging.Logger | Callable, level: int = logging.DEBUG, name: str | None = None):
"""
@ -101,12 +99,12 @@ class LogWrapper:
name (str|None): The name of the logging target. Defaults to None.
"""
if not isinstance(target, logging.Logger) and not callable(target):
if not isinstance(target, logging.Logger | Callable):
msg = "Target must be a logging.Logger instance or a callable."
raise TypeError(msg)
if name is None:
name = target.name if isinstance(target, logging.Logger) else getattr(target, "__name__", "callable")
name = target.name if isinstance(target, logging.Logger) else target.__name__
self.targets.append(
LogTarget(
@ -120,22 +118,14 @@ class LogWrapper:
def has_targets(self):
return len(self.targets) > 0
def _skip(self, msg: Any) -> bool:
return isinstance(msg, str) and any(value in msg for value in self.suppress)
def _log(self, level, msg, *args, **kwargs):
if self._skip(msg):
return
for target in self.targets:
if level < target.level:
continue
if isinstance(target.target, logging.Logger):
log_kwargs: dict[str, Any] = {**kwargs}
log_kwargs.setdefault("stacklevel", 3)
target.target.log(level, msg, *args, **log_kwargs)
elif callable(target.target):
if target.logger:
target.target.log(level, msg, *args, **kwargs)
else:
target.target(level, msg, *args, **kwargs)
def debug(self, msg, *args, **kwargs):
@ -179,17 +169,13 @@ def arg_converter(
create_parser = yt_dlp.options.create_parser
def _default_opts(args: str | list[str]):
def _default_opts(args: str):
patched_parser = create_parser()
def patched_create_parser():
return patched_parser
try:
yt_dlp.options.__dict__["create_parser"] = patched_create_parser
yt_dlp.options.create_parser = lambda: patched_parser
return yt_dlp.parse_options(args)
finally:
yt_dlp.options.__dict__["create_parser"] = create_parser
yt_dlp.options.create_parser = create_parser
apply_ytdlp_patches()
@ -242,7 +228,7 @@ def arg_converter(
return diff
def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern[str]] | None = None) -> list[str]:
def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) -> list[str]:
"""
Extract yt-dlp log lines matching built-in filters plus any extras.
@ -336,7 +322,7 @@ def get_ytdlp(params: dict | None = None) -> YTDLP:
return _DATA.YTDLP_INFO_CLS
def get_thumbnail(thumbnails: list) -> dict | None:
def get_thumbnail(thumbnails: list) -> str | None:
"""
Extract thumbnail URL from a yt-dlp entry.
@ -344,7 +330,7 @@ def get_thumbnail(thumbnails: list) -> dict | None:
thumbnails (list): The list of thumbnail dictionaries from yt-dlp entry.
Returns:
dict | None: The best thumbnail dict if available, otherwise None.
str | None: The thumbnail URL if available, otherwise None.
"""
if not thumbnails or not isinstance(thumbnails, list):
@ -385,7 +371,7 @@ def get_extras(entry: dict, kind: str = "video") -> dict:
extras[f"playlist_{property}"] = val
if thumbnail := get_thumbnail(entry.get("thumbnails", [])):
extras["thumbnail"] = thumbnail.get("url") if isinstance(thumbnail, dict) else thumbnail
extras["thumbnail"] = thumbnail.get("url")
elif thumbnail := entry.get("thumbnail"):
extras["thumbnail"] = thumbnail
@ -443,7 +429,7 @@ def get_archive_id(url: str) -> dict[str, str | None]:
}
"""
idDict: dict[str, str | None] = {
idDict: dict[str, None] = {
"id": None,
"ie_key": None,
"archive_id": None,
@ -467,12 +453,8 @@ def get_archive_id(url: str) -> dict[str, str | None]:
idDict["archive_id"] = make_archive_id(_ie, temp_id)
break
except Exception as e:
LOG.exception(
"Failed to get archive ID for '%s' with extractor '%s'.",
url,
key,
extra={"url": url, "extractor": key, "exception_type": type(e).__name__},
)
LOG.exception(e)
LOG.error(f"Error getting archive ID: {e}")
return idDict

View file

@ -3,13 +3,10 @@ import sys
from typing import Any
import yt_dlp
from yt_dlp.globals import extractors as ytdlp_extractors
from yt_dlp.utils import make_archive_id
from app.features.ytdlp.outtmpl import rewrite_outtmpl
from app.features.ytdlp.patches import apply_ytdlp_patches
from app.library.cf_solver_handler import set_cf_handler
from app.library.log import get_logger
class _ArchiveProxy:
@ -58,17 +55,6 @@ class YTDLP(yt_dlp.YoutubeDL):
def __init__(self, params=None, auto_init=True):
apply_ytdlp_patches()
if not YTDLP._registered:
try:
from app.yt_dlp_plugins.extractor import generic_browser
from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP
from yt_dlp.postprocessor import postprocessors
postprocessors.value.update({"NFOMakerPP": NFOMakerPP})
YTDLP._registered = True
except Exception:
get_logger().exception("Failed to register yt-dlp plugins")
# Avoid yt-dlp preloading the archive file by stripping the param first
orig_file = None
patched_params: dict[str, Any] | None = None
@ -81,9 +67,6 @@ class YTDLP(yt_dlp.YoutubeDL):
except Exception:
patched_params = params
self._ytptube_outtmpl_info: dict[str, Any] | None = None
self._ytptube_outtmpl_cache: dict[str, Any] = {}
super().__init__(params=patched_params, auto_init=auto_init)
# Restore param and replace upstream archive set with our proxy
@ -94,6 +77,15 @@ class YTDLP(yt_dlp.YoutubeDL):
pass
self.archive = _ArchiveProxy(orig_file)
if not YTDLP._registered:
try:
from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP
from yt_dlp.postprocessor import postprocessors
postprocessors.value.update({"NFOMakerPP": NFOMakerPP})
YTDLP._registered = True
except Exception:
pass
def _delete_downloaded_files(self, *args, **kwargs) -> None:
if self._interrupted:
@ -102,25 +94,6 @@ class YTDLP(yt_dlp.YoutubeDL):
return super()._delete_downloaded_files(*args, **kwargs)
def _reset_outtmpl_cache(self) -> None:
self._ytptube_outtmpl_info = None
self._ytptube_outtmpl_cache = {}
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False, *, _exec=False):
if self._ytptube_outtmpl_info is not info_dict:
self._ytptube_outtmpl_info = info_dict
self._ytptube_outtmpl_cache = {}
outtmpl, enriched = rewrite_outtmpl(outtmpl, info_dict, cache=self._ytptube_outtmpl_cache)
return super().prepare_outtmpl(outtmpl, enriched, sanitize=sanitize, _exec=_exec)
def process_info(self, info_dict):
try:
return super().process_info(info_dict)
finally:
self._reset_outtmpl_cache()
def record_download_archive(self, info_dict) -> None:
if not self.params.get("download_archive"):
return

View file

@ -1,3 +1,4 @@
import logging
import shlex
from pathlib import Path
from typing import Any
@ -5,10 +6,9 @@ from typing import Any
from app.features.presets.schemas import Preset
from app.features.ytdlp.utils import arg_converter
from app.library.config import Config
from app.library.log import get_logger
from app.library.Utils import calc_download_path, create_cookies_file, merge_dict
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("ytdlp.ytdlp_opts")
class ARGSMerger:
@ -50,12 +50,12 @@ class ARGSMerger:
"""
return str(self)
def as_dict(self) -> list[str]:
def as_dict(self) -> dict:
"""
Get all the options as a shell argument list.
Get all the options as a dict.
Returns:
list[str]: The options as shell arguments.
dict: The options as a dict
"""
return shlex.split(shlex.join(self.args))
@ -335,12 +335,7 @@ class YTDLPOpts:
if file and file.exists():
self._preset_opts["cookiefile"] = str(file)
except ValueError as e:
LOG.exception(
"Failed to load cookies for preset '%s': %s.",
preset.name,
e,
extra={"preset": preset.name, "has_cookies": True, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to load '{preset.name}' cookies. {e!s}")
if preset.template:
self._preset_opts["outtmpl"] = {"default": preset.template, "chapter": self._config.output_template_chapter}
@ -397,30 +392,6 @@ class YTDLPOpts:
data: dict = merge_dict(user_cli, merge_dict(self._item_opts, merge_dict(self._preset_opts, default_opts)))
if self._config.disable_exec:
stripped: list[str] = []
if data.pop("netrc_cmd", None):
stripped.append("netrc_cmd")
if "postprocessors" in data:
exec_pps = [
pp for pp in data["postprocessors"] if isinstance(pp, dict) and pp.get("key", "").startswith("Exec")
]
if exec_pps:
stripped.extend(pp.get("key") for pp in exec_pps)
data["postprocessors"] = [
pp
for pp in data["postprocessors"]
if not (isinstance(pp, dict) and pp.get("key", "").startswith("Exec"))
]
if stripped:
LOG.warning(
"Stripped %d dangerous options from yt-dlp options.",
len(stripped),
extra={"stripped": stripped, "reason": "YTP_DISABLE_EXEC is enabled"},
)
if not keep:
self.reset()
@ -432,7 +403,7 @@ class YTDLPOpts:
data["format"] = data["format"][1:]
if self._config.debug:
LOG.debug("Prepared final yt-dlp options.", extra={"ytdlp_options": data})
LOG.debug(f"Final yt-dlp options: '{data!s}'.")
return data

View file

@ -1,16 +1,15 @@
import asyncio
import inspect
import logging
import threading
from queue import Empty, Queue
from aiohttp import web
from app.library.log import get_logger
from .Services import Services
from .Singleton import Singleton
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("BackgroundWorker")
class CloseThread:
@ -29,7 +28,7 @@ class BackgroundWorker(metaclass=Singleton):
"The queue to hold the tasks."
self.running = True
"Whether the background worker is running or not."
self.thread: threading.Thread | None = None
self.thread: threading.Thread = None
"The thread that runs the background worker."
@staticmethod
@ -40,23 +39,19 @@ class BackgroundWorker(metaclass=Singleton):
Services.get_instance().add("background_worker", self)
app.on_shutdown.append(self.on_shutdown)
LOG.debug("Started background worker thread.")
LOG.debug("Starting background worker...")
self.thread = threading.Thread(target=self._run, daemon=True)
self.thread.start()
async def on_shutdown(self, _: web.Application):
self.running = False
try:
LOG.debug("Stopping background worker thread.")
LOG.debug("Shutting down background worker...")
self.queue.put((CloseThread, (), {}))
if self.thread:
self.thread.join(timeout=5)
LOG.debug("Background worker thread has been shut down.")
self.thread.join(timeout=5)
LOG.debug("Background worker has been shut down.")
except Exception as e:
LOG.exception(
"Failed to shut down background worker thread.",
extra={"exception_type": type(e).__name__},
)
LOG.error(f"Failed to shut down background worker: {e}")
def _run(self):
asyncio.set_event_loop(asyncio.new_event_loop())
@ -84,16 +79,8 @@ class BackgroundWorker(metaclass=Singleton):
if inspect.iscoroutine(result):
loop.call_soon_threadsafe(loop.create_task, result)
except Exception as e:
function = getattr(fn, "__name__", fn.__class__.__name__)
LOG.exception(
"Failed to run background worker function '%s'.",
function,
extra={
"function": function,
"thread_name": threading.current_thread().name,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
LOG.error(f"Failed to run '{fn.__name__}'. {e!s}")
except Empty:
continue
@ -101,12 +88,9 @@ class BackgroundWorker(metaclass=Singleton):
loop.call_soon_threadsafe(loop.stop)
loop_thread.join(timeout=5)
loop.close()
LOG.debug("Stopped background worker event loop.")
LOG.debug("Event loop has been stopped and closed.")
except Exception as e:
LOG.exception(
"Failed to stop background worker event loop.",
extra={"thread_name": loop_thread.name, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to stop the event loop: {e!s}")
def submit(self, fn, *args, **kwargs):
self.queue.put((fn, args, kwargs))

View file

@ -1,16 +1,15 @@
import copy
import logging
from collections import OrderedDict
from collections.abc import Iterable
from enum import Enum
from app.library.log import get_logger
from .downloads import Download
from .ItemDTO import ItemDTO
from .operations import matches_condition
from .sqlite_store import SqliteStore
LOG = get_logger()
LOG = logging.getLogger("datastore")
class StoreType(str, Enum):
@ -166,12 +165,6 @@ class DataStore:
def items(self):
return self._dict.items()
def __contains__(self, key: str) -> bool:
return key in self._dict
def __len__(self) -> int:
return len(self._dict)
async def put(self, value: Download, no_notify: bool = False) -> Download:
_ = no_notify
self._dict[value.info._id] = value

View file

@ -1,16 +1,16 @@
import asyncio
import datetime
import logging
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
from app.features.core.utils import gen_random
from app.library.log import get_logger
from .BackgroundWorker import BackgroundWorker
from .Singleton import Singleton
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("events")
class Events:
@ -34,7 +34,6 @@ class Events:
ITEM_ADDED: str = "item_added"
ITEM_UPDATED: str = "item_updated"
ITEM_PROGRESS: str = "item_progress"
ITEM_COMPLETED: str = "item_completed"
ITEM_CANCELLED: str = "item_cancelled"
ITEM_DELETED: str = "item_deleted"
@ -87,7 +86,6 @@ class Events:
Events.LOG_SUCCESS,
Events.ITEM_ADDED,
Events.ITEM_UPDATED,
Events.ITEM_PROGRESS,
Events.ITEM_CANCELLED,
Events.ITEM_DELETED,
Events.ITEM_BULK_DELETED,
@ -105,7 +103,7 @@ class Events:
list: The list of debug events.
"""
return [Events.ITEM_UPDATED, Events.ITEM_PROGRESS]
return [Events.ITEM_UPDATED]
@dataclass(kw_only=True)
@ -244,12 +242,7 @@ class EventBus(metaclass=Singleton):
event = Events.frontend()
else:
if event not in all_events:
LOG.error(
"Listener '%s' tried to subscribe to unknown event '%s'.",
name,
event,
extra={"listener": name, "event": event},
)
LOG.error(f"'{name}' attempted to listen on '{event}' which does not exist.")
return self
event = [event]
@ -259,12 +252,7 @@ class EventBus(metaclass=Singleton):
for e in event:
if e not in all_events:
LOG.error(
"Listener '%s' tried to subscribe to unknown event '%s'.",
name,
e,
extra={"listener": name, "event": e},
)
LOG.error(f"'{name}' attempted to listen on '{e}' which does not exist.")
continue
if e not in self._listeners:
@ -273,7 +261,7 @@ class EventBus(metaclass=Singleton):
self._listeners[e] = [(n, listener) for n, listener in self._listeners[e] if n != name]
self._listeners[e].append((name, EventListener(name, callback)))
LOG.debug("Listener '%s' has subscribed.", name, extra={"listener": name, "events": event})
LOG.debug(f"'{name}' subscribed to '{event}'.")
return self
@ -301,7 +289,7 @@ class EventBus(metaclass=Singleton):
events.append(e)
if len(events) > 0:
LOG.debug("Listener '%s' unsubscribed from '%s'.", name, events, extra={"listener": name, "events": events})
LOG.debug(f"'{name}' unsubscribed from '{events}'.")
return self
@ -328,17 +316,7 @@ class EventBus(metaclass=Singleton):
ev = Event(event=event, title=title, message=message, data=data)
if self.debug or event not in Events.only_debug():
LOG.debug(
"Delivering '%s' event to %s listener(s).",
ev.event,
len(self._listeners[event]),
extra={
"event_id": ev.id,
"event": ev.event,
"handler_count": len(self._listeners[event]),
"data": data,
},
)
LOG.debug(f"Emitting event '{ev.id}: {ev.event}'.", extra={"data": data})
try:
loop = asyncio.get_running_loop()
@ -351,35 +329,16 @@ class EventBus(metaclass=Singleton):
if asyncio.iscoroutine(coro):
await coro
else:
LOG.warning(
"Async handler '%s' returned '%s' instead of a coroutine.",
handler.name,
type(coro).__name__,
extra={"handler": handler.name, "returned_type": type(coro).__name__},
)
LOG.warning(f"Expected coroutine from async handler '{handler.name}', got {type(coro)}")
else:
await self._call(handler, ev, kwargs)
except Exception as e:
LOG.exception(
"Failed to emit '%s' event to listener '%s'.",
ev.event,
handler.name,
extra={
"event_id": ev.id,
"event": ev.event,
"handler": handler.name,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
LOG.error(f"Failed to emit event '{ev.event}' to '{handler.name}'. Error message '{e!s}'.")
loop.create_task(execute_handlers())
except RuntimeError:
LOG.debug(
"No event loop detected; offloading '%s' event to %s background listener(s).",
ev.event,
len(self._listeners[event]),
extra={"event_id": ev.id, "event": ev.event, "handler_count": len(self._listeners[event])},
)
LOG.debug(f"No event loop detected - using BackgroundWorker for {len(self._listeners[event])} handlers")
for _, handler in self._listeners[event]:
try:
if not self._offload:
@ -387,17 +346,8 @@ class EventBus(metaclass=Singleton):
self._offload.submit(handler.handle, ev, **kwargs)
except Exception as e:
LOG.exception(
"Failed to offload '%s' event to listener '%s'.",
ev.event,
handler.name,
extra={
"event_id": ev.id,
"event": ev.event,
"handler": handler.name,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
LOG.error(f"Failed to emit event '{ev.event}' to '{handler.name}'. Error message '{e!s}'.")
def clear(self) -> None:
"""

View file

@ -1,19 +1,15 @@
import base64
import hmac
import inspect
import logging
from collections.abc import Awaitable
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, cast
import anyio
from aiohttp import web
from aiohttp.typedefs import Handler, Middleware
from aiohttp.web import Request, Response
from aiohttp.web_log import AccessLogger
from aiohttp.web_request import BaseRequest
from aiohttp.web_response import StreamResponse
from aiohttp.web import Request, RequestHandler, Response
from app.library.log import get_logger
from app.library.Services import Services
from .cache import Cache
@ -23,33 +19,7 @@ from .Events import EventBus
from .router import RouteType, get_routes
from .Utils import decrypt_data, encrypt_data, get_file, load_modules
LOG = get_logger("http")
class HttpAccessLogger(AccessLogger):
def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
try:
fmt_info = self._format_line(request, response, time)
values: list[object] = []
extra: dict[str, Any] = {"elapsed_ms": round(time * 1000.0, 2)}
for key, value in fmt_info:
values.append(value)
if isinstance(key, str):
extra[key] = value
else:
parent, child = key
group: dict[str, Any] = extra.get(parent, {}) # type: ignore[assignment]
if not isinstance(group, dict):
group = {}
group[child] = value
extra[parent] = group
message = self._log_format % tuple(values)
self.logger.info(message, extra=extra)
except Exception:
self.logger.exception("Error in logging")
LOG: logging.Logger = logging.getLogger("http_api")
class HttpAPI:
@ -119,10 +89,7 @@ class HttpAPI:
try:
app.on_response_prepare.append(on_prepare)
except Exception as e:
LOG.exception(
"Failed to register response preparation middleware.",
extra={"operation": "register_on_response_prepare", "exception_type": type(e).__name__},
)
LOG.exception(e)
app.on_shutdown.append(self.on_shutdown)
@ -163,13 +130,7 @@ class HttpAPI:
route.path = f"{base_path}/{route.path.lstrip('/')}"
if self.config.debug:
LOG.debug(
"Adding route '%s' %s: %s.",
route.name,
route.method,
route.path,
extra={"route_name": route.name, "method": route.method, "path": route.path},
)
LOG.debug(f"Add ({route.name}) {route.method}: {route.path}.")
app.router.add_route(route.method, route.path, handler=_handle(route.handler), name=route.name)
@ -180,7 +141,7 @@ class HttpAPI:
registered_options.append(route.path)
@staticmethod
def basic_auth(username: str, password: str) -> Middleware:
def basic_auth(username: str, password: str) -> Awaitable:
"""
Middleware to handle basic authentication.
@ -194,7 +155,7 @@ class HttpAPI:
"""
@web.middleware
async def auth_handler(request: Request, handler: Handler) -> StreamResponse:
async def auth_handler(request: Request, handler: RequestHandler) -> Response:
# if OPTIONS request, skip auth
if "OPTIONS" == request.method:
return await handler(request)
@ -207,19 +168,12 @@ class HttpAPI:
auth_cookie = request.cookies.get("auth")
if auth_cookie is not None:
try:
data = decrypt_data(auth_cookie, key=cast("bytes", Config.get_instance().secret_key))
data = decrypt_data(auth_cookie, key=Config.get_instance().secret_key)
if data is not None:
data = base64.b64encode(data.encode()).decode()
auth_header = f"Basic {data}"
except Exception as e:
LOG.exception(
"Failed to decrypt authentication cookie.",
extra={
"route": str(request.rel_url),
"method": request.method,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
if auth_header is None:
return web.json_response(
@ -253,7 +207,7 @@ class HttpAPI:
},
)
response: StreamResponse = await handler(request)
response: Response = await handler(request)
contentType: str | None = response.headers.get("content-type", None)
if contentType and not contentType.startswith("text/html"):
@ -264,7 +218,7 @@ class HttpAPI:
"auth",
encrypt_data(
f"{user_name}:{user_password}",
key=cast("bytes", Config.get_instance().secret_key),
key=Config.get_instance().secret_key,
),
max_age=60 * 60 * 24 * 7,
expires=(datetime.now(UTC) + timedelta(days=7)).strftime("%a, %d %b %Y %H:%M:%S GMT"),
@ -272,20 +226,16 @@ class HttpAPI:
samesite="Strict",
)
except Exception as e:
LOG.exception(
"Failed to set authentication cookie for '%s'.",
request.rel_url,
extra={"route": str(request.rel_url), "method": request.method, "exception_type": type(e).__name__},
)
LOG.exception(e)
return response
return auth_handler
@staticmethod
def middle_wares(app: web.Application, base_path: str, download_path: str) -> Middleware:
def middle_wares(app: web.Application, base_path: str, download_path: str) -> Awaitable:
@web.middleware
async def middleware_handler(request: Request, handler: Handler) -> StreamResponse:
async def middleware_handler(request: Request, handler: RequestHandler) -> Response:
static_path = str(app.router["download_static"].url_for(filename=""))
if request.path.startswith(static_path):
realFile, status = get_file(
@ -304,13 +254,12 @@ class HttpAPI:
},
)
response: StreamResponse = await handler(request)
response: Response = await handler(request)
contentType: str = str(response.headers.get("content-type", ""))
contentType: str = response.headers.get("content-type", "")
if contentType.startswith("text/html") and getattr(response, "_path", None):
rewrite_path: str = base_path.rstrip("/")
response_path = cast("Any", response)._path
async with await anyio.open_file(response_path, "rb") as f:
async with await anyio.open_file(response._path, "rb") as f:
content = await f.read()
content: str = (
content.decode("utf-8")
@ -328,13 +277,13 @@ class HttpAPI:
new_response.set_cookie(
morsel.key,
morsel.value,
expires=morsel.get("expires"),
domain=morsel.get("domain"),
max_age=morsel.get("max-age"),
path=morsel.get("path") or "/",
secure=bool(morsel.get("secure")),
httponly=bool(morsel.get("httponly")),
samesite=morsel.get("samesite") or None,
expires=morsel["expires"],
domain=morsel["domain"],
max_age=morsel["max-age"],
path=morsel["path"],
secure=bool(morsel["secure"]),
httponly=bool(morsel["httponly"]),
samesite=morsel["samesite"] or None,
)
return new_response
@ -343,7 +292,7 @@ class HttpAPI:
return middleware_handler
async def _handle(self, handler: Handler, request: Request) -> StreamResponse:
async def _handle(self, handler: RequestHandler, request: Request) -> Response:
"""
Call the handler with the request and return the response.
@ -365,16 +314,7 @@ class HttpAPI:
else:
response = await Services.get_instance().handle_async(handler, request=request)
except TypeError as te:
LOG.exception(
"Failed to inject route handler dependencies for '%s'.",
getattr(handler, "__name__", handler.__class__.__name__),
extra={
"handler": getattr(handler, "__name__", handler.__class__.__name__),
"route": str(request.rel_url),
"method": request.method,
"exception_type": type(te).__name__,
},
)
LOG.exception(te)
if "missing 1 required positional argument" in str(te) and "request" in str(te):
response = await handler(request)
else:
@ -382,12 +322,7 @@ class HttpAPI:
except web.HTTPException as e:
return web.json_response(data={"error": str(e)}, status=e.status_code)
except Exception as e:
LOG.exception(
"Failed to handle request '%s %s'.",
request.method,
request.rel_url,
extra={"route": str(request.rel_url), "method": request.method, "exception_type": type(e).__name__},
)
LOG.exception(e)
response = web.json_response(
data={"error": "Internal Server Error"},
status=web.HTTPInternalServerError.status_code,

View file

@ -1,13 +1,13 @@
import asyncio
import functools
import json
import logging
from pathlib import Path
from typing import Any
from aiohttp import web
from app.features.core.utils import gen_random
from app.library.log import get_logger
from app.library.router import Route, RouteType, get_routes
from app.library.Services import Services
from app.library.Utils import load_modules
@ -17,7 +17,7 @@ from .encoder import Encoder
from .Events import Event, EventBus, Events
from .ItemDTO import Item
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("socket_api")
class WebSocketHub:
@ -104,20 +104,19 @@ class HttpSocket:
}
)
self._notify.subscribe("frontend", event_handler, f"{HttpSocket.__name__}.emit")
self._notify.subscribe("frontend", event_handler, f"{__class__.__name__}.emit")
@staticmethod
def ws_event(func):
def ws_event(func): # type: ignore
"""
Decorator to mark a method as a socket event.
"""
@functools.wraps(func)
@functools.wraps(func) # type: ignore
async def wrapper(*args, **kwargs):
return await func(*args, **kwargs)
return await func(*args, **kwargs) # type: ignore
wrapper_with_event: Any = wrapper
wrapper_with_event._ws_event = func.__name__
wrapper._ws_event = func.__name__ # type: ignore
return wrapper
async def on_shutdown(self, _: web.Application):
@ -135,11 +134,9 @@ class HttpSocket:
if not (data and data.data):
return
queue = Services.get_instance().get("queue")
if queue is not None:
await queue.add(item=Item.format(data.data))
await Services.get_instance().get("queue").add(item=Item.format(data.data))
self._notify.subscribe(Events.ADD_URL, event_handler, f"{HttpSocket.__name__}.add")
self._notify.subscribe(Events.ADD_URL, event_handler, f"{__class__.__name__}.add")
load_modules(self.rootPath, self.rootPath / "routes" / "socket")
@ -148,10 +145,7 @@ class HttpSocket:
for route in socket_routes.values():
if self.config.debug:
LOG.debug(
"Registered socket route '%s' for %s %s.",
route.name,
route.method.value if isinstance(route.method, RouteType) else route.method,
route.path,
f"Add ({route.name}) {route.method.value if isinstance(route.method, RouteType) else route.method}: {route.path}."
)
async def handle_message(sid: str, message: str) -> None:
@ -167,7 +161,7 @@ class HttpSocket:
return
if not (route := socket_routes.get(event)):
LOG.debug("Unknown WebSocket event '%s'.", event, extra={"sid": sid, "event": event})
LOG.debug(f"Unknown websocket event '{event}'.")
return
data = payload.get("data") if isinstance(payload, dict) else None
@ -189,7 +183,7 @@ class HttpSocket:
sid: str = gen_random(14)
self.sio.add(sid, ws)
LOG.debug("WebSocket client '%s' connected.", sid, extra={"sid": sid})
LOG.debug(f"WebSocket client '{sid}' connected.")
await handle_connect(sid)
@ -198,11 +192,8 @@ class HttpSocket:
try:
await handle_message(sid, message)
except Exception as e:
LOG.exception(
"Failed to handle WebSocket message from client '%s'.",
sid,
extra={"sid": sid, "message_size": len(message), "exception_type": type(e).__name__},
)
LOG.exception(e)
LOG.error(f"Error handling WebSocket message from client '{sid}': {e}")
try:
i: int = 0
@ -211,22 +202,15 @@ class HttpSocket:
i = i + 1
asyncio.create_task(handle_message_safe(sid, msg.data), name=f"ws_msg_{sid}_{i}")
elif msg.type == web.WSMsgType.ERROR:
LOG.error(
"WebSocket connection closed with exception %s",
ws.exception(),
extra={
"sid": sid,
"exception": str(ws.exception()) if ws.exception() else None,
},
)
LOG.error(f"WebSocket connection closed with exception {ws.exception()}")
finally:
await handle_disconnect(sid, getattr(ws, "close_reason", None))
self.sio.remove(sid)
LOG.debug("WebSocket client '%s' disconnected.", sid, extra={"sid": sid})
LOG.debug(f"WebSocket client '{sid}' disconnected.")
return ws
if self.config.debug:
LOG.debug("Registered WebSocket endpoint at '%s'.", ws_path, extra={"method": "GET", "path": ws_path})
LOG.debug(f"Add (ws) GET: {ws_path}.")
app.router.add_get(ws_path, ws_handler, name="ws")

View file

@ -1,3 +1,4 @@
import logging
import re
import time
import uuid
@ -9,13 +10,12 @@ from typing import TYPE_CHECKING, Any
from app.features.ytdlp.utils import archive_add, archive_delete, archive_read, get_archive_id
from app.features.ytdlp.ytdlp_opts import YTDLPOpts
from app.library.encoder import Encoder
from app.library.log import get_logger
from app.library.Utils import clean_item, get_file, get_file_sidecar
if TYPE_CHECKING:
from app.features.presets.schemas import Preset
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("ItemDTO")
@dataclass(kw_only=True)
@ -93,7 +93,7 @@ class Item:
bool: True if the item has extras, False otherwise.
"""
return bool(self.extras and len(self.extras) > 0)
return self.extras and len(self.extras) > 0
def add_extras(self, key: str, value: Any) -> None:
"""
@ -117,7 +117,7 @@ class Item:
bool: True if the item has command options for yt, False otherwise.
"""
return bool(self.cli and len(self.cli) > 2)
return self.cli and len(self.cli) > 2
@staticmethod
def _default_preset() -> str:
@ -161,7 +161,7 @@ class Item:
Item: The formatted item.
"""
url = item.get("url")
url: str = item.get("url")
if not url or not isinstance(url, str):
msg = "url param is required."
@ -173,7 +173,7 @@ class Item:
if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url):
url = f"https://www.youtube.com/watch?v={url}"
data: dict[str, Any] = {"url": url}
data: dict[str, str] = {"url": url}
preset: str | None = item.get("preset")
if preset and isinstance(preset, str) and preset != Item._default_preset():
@ -185,28 +185,24 @@ class Item:
data["preset"] = preset
folder = item.get("folder")
if isinstance(folder, str) and folder:
data["folder"] = folder
if item.get("folder") and isinstance(item.get("folder"), str):
data["folder"] = item.get("folder")
cookies = item.get("cookies")
if isinstance(cookies, str) and cookies:
data["cookies"] = cookies
if item.get("cookies") and isinstance(item.get("cookies"), str):
data["cookies"] = item.get("cookies")
template = item.get("template")
if isinstance(template, str) and template:
data["template"] = template
if item.get("template") and isinstance(item.get("template"), str):
data["template"] = item.get("template")
if isinstance(item.get("auto_start"), bool):
data["auto_start"] = item["auto_start"]
if "auto_start" in item and isinstance(item.get("auto_start"), bool):
data["auto_start"] = bool(item.get("auto_start"))
extras: dict | None = item.get("extras")
if isinstance(extras, dict) and len(extras) > 0:
if extras and isinstance(extras, dict) and len(extras) > 0:
data["extras"] = extras
requeued = item.get("requeued")
if isinstance(requeued, bool):
data["requeued"] = requeued
if item.get("requeued") and isinstance(item.get("requeued"), bool):
data["requeued"] = item.get("requeued")
cli: str | None = item.get("cli")
if cli and len(cli) > 2:
@ -578,10 +574,8 @@ class ItemDTO:
self.is_archivable = bool(self._archive_file)
if not self.is_archivable:
self.is_archived = False
elif self.archive_id and self._archive_file:
self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
else:
self.is_archived = False
self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
def get_archive_file(self) -> str | None:
"""

View file

@ -1,15 +1,14 @@
import importlib
import importlib.metadata
import logging
import os
import subprocess
import sys
from pathlib import Path
from app.library.log import get_logger
from .httpx_client import sync_client
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("package_installer")
def parse_version(v: str) -> tuple[int, ...]:
@ -17,7 +16,7 @@ def parse_version(v: str) -> tuple[int, ...]:
class Packages:
def __init__(self, env: str | None, file: str | Path | None, upgrade: bool = False):
def __init__(self, env: str | None, file: str | None, upgrade: bool = False):
from_env: list[str] = env.split() if env else []
from_file = []
@ -65,34 +64,19 @@ class PackageInstaller:
current_version: str | None = self._get_installed_version(pkg)
if current_version and version and self.compare_versions(current_version, version):
LOG.info(
"Package '%s' is already installed at version '%s'; skipping installation.",
pkg,
version,
extra={"package": pkg, "installed_version": current_version, "requested_version": version},
)
LOG.info(f"'{pkg}' is already installed with the specified version ({version}). Skipping installation.")
return
if upgrade and current_version and not version:
latest_version: str | None = self._get_latest_version(pkg)
if latest_version and parse_version(current_version) >= parse_version(latest_version):
LOG.info(
"Package '%s' is already at the latest version '%s'; skipping upgrade.",
pkg,
current_version,
extra={"package": pkg, "installed_version": current_version, "latest_version": latest_version},
)
LOG.info(f"'{pkg}' is already the latest version ({current_version}). Skipping upgrade.")
return
if current_version:
LOG.info(
"Package '%s' is installed at '%s'; proceeding with upgrade.",
pkg,
current_version,
extra={"package": pkg, "installed_version": current_version},
)
LOG.info(f"'{pkg}' is already installed (version: {current_version}). Proceeding with upgrade...")
else:
LOG.info("Package '%s' is not installed; installing it.", pkg, extra={"package": pkg})
LOG.info(f"'{pkg}' is not installed. Installing...")
self._install_pkg(pkg, version=version)
@ -109,20 +93,9 @@ class PackageInstaller:
resp = client.get(url)
if 200 == resp.status_code:
return resp.json()["info"]["version"]
LOG.warning(
"Failed to fetch PyPI metadata for '%s': HTTP %s.",
pkg,
resp.status_code,
extra={"package": pkg, "status_code": resp.status_code, "url": url},
)
LOG.warning(f"Failed to fetch '{pkg}' info: HTTP {resp.status_code}")
except Exception as e:
LOG.warning(
"Failed to query PyPI for '%s': %s",
pkg,
e,
extra={"package": pkg, "url": url, "exception_type": type(e).__name__},
exc_info=True,
)
LOG.warning(f"Error while querying PyPI for '{pkg}': {e}")
return None
def _install_pkg(self, pkg: str, version: str | None = None) -> bool:
@ -162,12 +135,7 @@ class PackageInstaller:
return 0 == proc.returncode
except subprocess.CalledProcessError as e:
LOG.exception(
"Failed to install package '%s' (exit %s).",
pkg,
e.returncode,
extra={"package": pkg, "returncode": e.returncode, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to install '{pkg}' (exit {e.returncode}). {e!s}")
self.out(out=e.stdout, err=e.stderr)
raise
@ -182,20 +150,13 @@ class PackageInstaller:
if not pkgs.has_packages() or not self.user_site:
return
LOG.info("Checking configured user pip packages.", extra={"packages": pkgs.packages})
LOG.info(f"Checking for user pip packages: {', '.join(pkgs.packages)}")
for package in pkgs.packages:
try:
self.action(package, upgrade=pkgs.allow_upgrade())
except Exception as e:
LOG.exception(
"Failed to install or upgrade package '%s'.",
package,
extra={
"package": package,
"allow_upgrade": pkgs.allow_upgrade(),
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to install or upgrade package '{package}'. Error message: {e!s}")
LOG.exception(e)
def compare_versions(self, current: str, target: str) -> bool:
"""

View file

@ -1,16 +1,13 @@
import asyncio
from collections.abc import Callable
from typing import Any
import logging
from aiocron import Cron
from aiohttp import web
from app.library.log import get_logger
from .Events import EventBus, Events
from .Singleton import Singleton
LOG = get_logger()
LOG = logging.getLogger("scheduler")
class Scheduler(metaclass=Singleton):
@ -49,13 +46,10 @@ class Scheduler(metaclass=Singleton):
for job in self._jobs:
try:
self._jobs[job].stop()
LOG.debug("Stopped job '%s'.", job, extra={"job_id": job})
LOG.debug(f"Stopped job '{job}'.")
except Exception as e:
LOG.exception(
"Failed to stop scheduler job '%s'.",
job,
extra={"job_id": job, "exception_type": type(e).__name__},
)
LOG.exception(e)
LOG.error(f"Failed to stop job '{job}'. Error message '{e!s}'.")
self._jobs = {}
@ -68,7 +62,7 @@ class Scheduler(metaclass=Singleton):
if data and data.data:
self.add(**data.data)
EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{type(self).__name__}.add")
EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{__class__.__name__}.add")
def get_all(self) -> dict[str, Cron]:
"""Return the jobs."""
@ -101,7 +95,7 @@ class Scheduler(metaclass=Singleton):
return id in self._jobs
def add(
self, timer: str, func: Callable[..., Any], args: tuple = (), kwargs: dict | None = None, id: str | None = None
self, timer: str, func: callable, args: tuple = (), kwargs: dict | None = None, id: str | None = None
) -> str:
"""
Add a job to the schedule.
@ -133,7 +127,7 @@ class Scheduler(metaclass=Singleton):
self._jobs[job_id] = job
LOG.debug("Added job '%s' to run on '%s'.", job_id, timer, extra={"job_id": job_id, "timer": timer})
LOG.debug(f"Added '{job_id}' to the scheduler to run on '{timer}'.")
return job_id
@ -157,15 +151,12 @@ class Scheduler(metaclass=Singleton):
try:
self._jobs[id].stop()
except Exception as e:
LOG.exception(
"Failed to stop scheduler job '%s'.",
id,
extra={"job_id": id, "exception_type": type(e).__name__},
)
LOG.exception(e)
LOG.error(f"Failed to stop job '{id}'. Error message '{e!s}'.")
return False
del self._jobs[id]
LOG.debug("Removed job '%s' from the scheduler.", id, extra={"job_id": id})
LOG.debug(f"Removed job '{id}' from the scheduler.")
return True
return False

View file

@ -1,13 +1,12 @@
import inspect
from collections.abc import Callable
import logging
from dataclasses import dataclass
from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints
from app.library.log import get_logger
from app.library.Singleton import Singleton
T = TypeVar("T")
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
def _unwrap_annotation(ann: Any) -> Any:
@ -118,7 +117,7 @@ class Services(metaclass=Singleton):
return candidates[0]
def _build_call_args(self, handler: Callable[..., Any], overrides: dict[str, Any]) -> dict[str, Any]:
def _build_call_args(self, handler: callable, overrides: dict[str, Any]) -> dict[str, Any]:
sig: inspect.Signature = inspect.signature(handler)
try:
@ -170,10 +169,10 @@ class Services(metaclass=Singleton):
return resolved
async def handle_async(self, handler: Callable[..., Any], **kwargs) -> Any:
async def handle_async(self, handler: callable, **kwargs) -> Any:
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
return await handler(**resolved)
def handle_sync(self, handler: Callable[..., Any], **kwargs) -> Any:
def handle_sync(self, handler: callable, **kwargs) -> Any:
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
return handler(**resolved)

View file

@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import errno
import json
import logging
import os
import shlex
import shutil
@ -15,8 +16,6 @@ from typing import TYPE_CHECKING, Any
from aiohttp import web
from app.library.config import Config
from app.library.log import get_logger
from app.library.Scheduler import Scheduler
from app.library.Services import Services
from app.library.Singleton import Singleton
@ -26,16 +25,14 @@ if TYPE_CHECKING:
from aiohttp.web import Request
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("terminal_manager")
ACTIVE_FILE_NAME = "active.json"
METADATA_FILE_NAME = "metadata.json"
TRANSCRIPT_FILE_NAME = "transcript.jsonl"
DEFAULT_DRAIN_TTL = 30.0
COMPLETED_SESSION_RETENTION = 86400.0
DEFAULT_KEEPALIVE_INTERVAL = 15.0
DEFAULT_SHUTDOWN_TIMEOUT = 5.0
CLEANUP_SCHEDULE = "5 */1 * * *"
class TerminalSessionConflictError(RuntimeError):
@ -58,10 +55,8 @@ class TerminalSessionManager(metaclass=Singleton):
self._lock = asyncio.Lock()
self._active: ActiveTerminalSession | None = None
self._drain_ttl: float = DEFAULT_DRAIN_TTL
self._completed_retention: float = COMPLETED_SESSION_RETENTION
self._keepalive_interval: float = DEFAULT_KEEPALIVE_INTERVAL
self._shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT
self._cleanup_job_id: str | None = None
@staticmethod
def get_instance() -> TerminalSessionManager:
@ -72,20 +67,11 @@ class TerminalSessionManager(metaclass=Singleton):
Services.get_instance().add("terminal_manager", self)
app.on_startup.append(self.on_startup)
app.on_shutdown.append(self.on_shutdown)
self._cleanup_job_id = Scheduler.get_instance().add(
timer=CLEANUP_SCHEDULE,
func=self.cleanup,
id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
)
async def on_startup(self, _: web.Application) -> None:
await self.initialize()
async def on_shutdown(self, _: web.Application) -> None:
if self._cleanup_job_id is not None:
Scheduler.get_instance().remove(self._cleanup_job_id)
self._cleanup_job_id = None
session_id: str | None = None
task: asyncio.Task[None] | None = None
@ -135,17 +121,12 @@ class TerminalSessionManager(metaclass=Singleton):
async def cleanup(self) -> None:
async with self._lock:
LOG.debug("Running scheduled terminal session cleanup.")
removed = self._cleanup_expired_sessions(time.time())
LOG.debug("Scheduled terminal session cleanup finished. Removed %d expired sessions.", removed)
async def list_sessions(self) -> list[dict[str, Any]]:
async with self._lock:
return self._list_sessions_locked()
self._cleanup_expired_sessions(time.time())
async def create_session(self, command: str) -> dict[str, Any]:
async with self._lock:
now = time.time()
self._cleanup_expired_sessions(now)
active_session = self._get_active_session_locked()
if active_session is not None:
@ -179,20 +160,22 @@ class TerminalSessionManager(metaclass=Singleton):
async def get_active_session(self) -> dict[str, Any] | None:
async with self._lock:
self._cleanup_expired_sessions(time.time())
metadata = self._get_active_session_locked()
return None if metadata is None else dict(metadata)
async def get_session(self, session_id: str) -> dict[str, Any] | None:
async with self._lock:
self._cleanup_expired_sessions(time.time())
metadata = self._load_metadata(session_id)
if metadata is None or self._is_expired_metadata(metadata, time.time()):
return None
return dict(metadata)
return None if metadata is None else dict(metadata)
async def cancel_session(self, session_id: str) -> dict[str, Any]:
async with self._lock:
self._cleanup_expired_sessions(time.time())
metadata = self._load_metadata(session_id)
if metadata is None or self._is_expired_metadata(metadata, time.time()):
if metadata is None:
msg = f"Unknown terminal session '{session_id}'."
raise FileNotFoundError(msg)
@ -235,9 +218,6 @@ class TerminalSessionManager(metaclass=Singleton):
async with self._lock:
metadata = self._load_metadata(session_id)
if metadata is not None:
if self._is_expired_metadata(metadata, time.time()):
return response
replay_until = int(metadata.get("last_sequence", last_sent))
runtime = self._active
@ -296,7 +276,7 @@ class TerminalSessionManager(metaclass=Singleton):
master_fd: int | None = None
try:
LOG.info("Starting terminal session '%s' command.", session_id, extra={"session_id": session_id})
LOG.info("Cli command from client. '%s'", command)
args = ["yt-dlp", *shlex.split(command, posix=os.name != "nt")]
env_vars = self._build_env()
@ -343,10 +323,7 @@ class TerminalSessionManager(metaclass=Singleton):
try:
os.close(slave_fd)
except Exception as exc:
LOG.exception(
"Failed to close the PTY slave file descriptor.",
extra={"exception_type": type(exc).__name__},
)
LOG.error("Error closing PTY. '%s'.", str(exc))
read_task = asyncio.create_task(
self._read_process_output(session_id=session_id, proc=proc, use_pty=use_pty, master_fd=master_fd),
@ -359,11 +336,8 @@ class TerminalSessionManager(metaclass=Singleton):
final_status = "interrupted"
except Exception as exc:
final_status = "failed"
LOG.exception(
"Terminal session '%s' command failed.",
session_id,
extra={"session_id": session_id, "use_pty": use_pty, "exception_type": type(exc).__name__},
)
LOG.error("CLI execute exception was thrown.")
LOG.exception(exc)
await self._append_event(session_id, "output", {"type": "stderr", "line": str(exc)})
finally:
final_status = await self._resolve_final_status(session_id=session_id, status=final_status)
@ -377,7 +351,7 @@ class TerminalSessionManager(metaclass=Singleton):
try:
return_code = await asyncio.wait_for(proc.wait(), timeout=self._shutdown_timeout)
except TimeoutError:
LOG.warning("Terminal session '%s' did not exit after repeated shutdown attempts.", session_id)
LOG.warning("Terminal session '%s' process did not exit cleanly.", session_id)
if proc is not None:
proc_returncode = getattr(proc, "returncode", None)
@ -469,7 +443,7 @@ class TerminalSessionManager(metaclass=Singleton):
metadata["status"] = status
metadata["exit_code"] = exit_code
metadata["finished_at"] = now
metadata["expires_at"] = now + self._completed_retention
metadata["expires_at"] = now + self._drain_ttl
self._write_json(self._metadata_path(session_id), metadata)
if self._load_active_marker() == session_id:
@ -494,31 +468,6 @@ class TerminalSessionManager(metaclass=Singleton):
return metadata
def _list_sessions_locked(self) -> list[dict[str, Any]]:
self._ensure_root()
items: list[dict[str, Any]] = []
for path in self.root_path.iterdir():
if path.name == ACTIVE_FILE_NAME or path.is_dir() is False:
continue
metadata = self._load_metadata(path.name)
if metadata is None:
continue
if self._is_expired_metadata(metadata, time.time()):
continue
item = dict(metadata)
item["available_until"] = self._available_until(metadata)
items.append(item)
items.sort(
key=lambda item: float(item.get("finished_at") or item.get("started_at") or item.get("created_at") or 0),
reverse=True,
)
return items
def _recover_orphaned_active_session(self, now: float) -> None:
session_id = self._load_active_marker()
if session_id is None:
@ -531,15 +480,14 @@ class TerminalSessionManager(metaclass=Singleton):
metadata["status"] = "interrupted"
metadata["finished_at"] = now
metadata["expires_at"] = now + self._completed_retention
metadata["expires_at"] = now + self._drain_ttl
metadata["exit_code"] = -1 if metadata.get("exit_code") is None else metadata["exit_code"]
self._write_json(self._metadata_path(session_id), metadata)
self._clear_active_marker()
def _cleanup_expired_sessions(self, now: float) -> int:
def _cleanup_expired_sessions(self, now: float) -> None:
self._ensure_root()
active_session_id = self._load_active_marker()
removed = 0
for path in self.root_path.iterdir():
if path.name == ACTIVE_FILE_NAME or path.is_dir() is False:
@ -548,10 +496,10 @@ class TerminalSessionManager(metaclass=Singleton):
metadata = self._load_metadata(path.name)
if metadata is None:
shutil.rmtree(path, ignore_errors=True)
removed += 1
continue
if not self._is_expired_metadata(metadata, now):
expires_at = metadata.get("expires_at")
if expires_at is None or float(expires_at) > now:
continue
if active_session_id == path.name:
@ -560,10 +508,6 @@ class TerminalSessionManager(metaclass=Singleton):
shutil.rmtree(path, ignore_errors=True)
removed += 1
return removed
def _parse_since(self, request: Request) -> int:
values: list[int] = []
candidates = [request.query.get("since"), request.headers.get("Last-Event-ID")]
@ -593,7 +537,7 @@ class TerminalSessionManager(metaclass=Singleton):
now = time.time()
metadata["status"] = "interrupted"
metadata["finished_at"] = now
metadata["expires_at"] = now + self._completed_retention
metadata["expires_at"] = now + self._drain_ttl
metadata["exit_code"] = -1 if metadata.get("exit_code") is None else metadata["exit_code"]
self._write_json(self._metadata_path(session_id), metadata)
@ -674,20 +618,6 @@ class TerminalSessionManager(metaclass=Singleton):
return events
def _available_until(self, metadata: dict[str, Any]) -> float | None:
expires_at = metadata.get("expires_at")
if expires_at is None:
return None
return float(expires_at)
def _is_expired_metadata(self, metadata: dict[str, Any], now: float) -> bool:
expires_at = self._available_until(metadata)
if expires_at is None:
return False
return expires_at <= now
def _build_env(self) -> dict[str, str]:
env_vars = os.environ.copy()
env_vars.update(

View file

@ -1,11 +1,10 @@
import asyncio
import logging
import re
from typing import Any
from aiohttp import web
from app.library.log import get_logger
from .cache import Cache
from .config import Config
from .Events import EventBus, Events
@ -14,7 +13,7 @@ from .Scheduler import Scheduler
from .Singleton import Singleton
from .version import APP_VERSION
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("update_checker")
class UpdateChecker(metaclass=Singleton):
@ -80,7 +79,7 @@ class UpdateChecker(metaclass=Singleton):
self._notify.subscribe(
event=Events.STARTED,
callback=event_handler,
name=f"{UpdateChecker.__name__}.{type(self).attach.__name__}",
name=f"{__class__.__name__}.{__class__.attach.__name__}",
)
self._schedule_check()
@ -108,7 +107,7 @@ class UpdateChecker(metaclass=Singleton):
self._job_id = self._scheduler.add(
timer=timer,
func=lambda: asyncio.create_task(self.check_for_updates()),
id=f"{UpdateChecker.__name__}.{self.check_for_updates.__name__}",
id=f"{__class__.__name__}.{self.check_for_updates.__name__}",
)
async def check_for_updates(self) -> tuple[tuple[str, str | None], tuple[str, str | None]]:
@ -146,11 +145,7 @@ class UpdateChecker(metaclass=Singleton):
strip_v_prefix: bool = False,
) -> tuple[str, str | None]:
try:
LOG.info(
"Checking whether %s has an update available.",
name,
extra={"target_name": name, "api_url": api_url, "current_version": current_version},
)
LOG.info(f"Checking for {name} updates...")
client = get_async_client(use_curl=False)
response = await client.get(
@ -160,59 +155,32 @@ class UpdateChecker(metaclass=Singleton):
)
if 200 != response.status_code:
LOG.warning(
"Failed to check for %s updates: HTTP %s",
name,
response.status_code,
extra={"target_name": name, "api_url": api_url, "status_code": response.status_code},
)
LOG.warning(f"Failed to check for {name} updates: HTTP {response.status_code}")
return ("error", None)
data: dict[str, Any] = response.json()
latest_tag: str = data.get("tag_name", "")
if not latest_tag:
LOG.warning(
"No tag_name found in %s GitHub release data.",
name,
extra={"target_name": name, "api_url": api_url},
)
LOG.warning(f"No tag_name found in {name} GitHub release data.")
return ("error", None)
compare_current: str = current_version.lstrip("v") if strip_v_prefix else current_version
compare_latest: str = latest_tag.lstrip("v") if strip_v_prefix else latest_tag
if self._compare_versions(compare_current, compare_latest):
LOG.warning(
"%s has an update available: %s -> %s.",
name,
current_version,
latest_tag,
extra={"target_name": name, "current_version": current_version, "latest_version": latest_tag},
)
LOG.warning(f"{name} update available: {current_version} -> {latest_tag}")
result = ("update_available", latest_tag)
await self._cache.aset(cache_key, result, self.CACHE_DURATION)
return result
LOG.info(
"%s is already up to date.",
name,
extra={"target_name": name, "current_version": current_version},
)
LOG.info(f"No {name} updates available.")
result = ("up_to_date", None)
await self._cache.aset(cache_key, result, self.CACHE_DURATION)
return result
except Exception as e:
LOG.exception(
"Failed to check whether %s has an update available.",
name,
extra={
"target_name": name,
"api_url": api_url,
"cache_key": cache_key,
"exception_type": type(e).__name__,
},
)
LOG.exception(e)
LOG.error(f"Error checking for {name} updates: {e!s}")
return ("error", None)
async def _check_app_version(self) -> tuple[str, str | None]:
@ -235,7 +203,7 @@ class UpdateChecker(metaclass=Singleton):
async def _check_ytdlp_version(self) -> tuple[str, str | None]:
current_version: str = self._config._ytdlp_version()
if not current_version or "0.0.0" == current_version:
LOG.warning("Skipping the yt-dlp update check because the current version could not be determined.")
LOG.warning("Could not determine yt-dlp version, skipping yt-dlp update check.")
return ("error", None)
status, new_version = await self._check_github_version(
@ -270,12 +238,5 @@ class UpdateChecker(metaclass=Singleton):
return parse_version(latest) > parse_version(current)
except Exception as e:
LOG.warning(
"Failed to compare versions '%s' and '%s' because %s.",
current,
latest,
e,
extra={"current_version": current, "latest_version": latest, "exception_type": type(e).__name__},
exc_info=True,
)
LOG.warning(f"Error comparing versions '{current}' vs '{latest}': {e}")
return False

View file

@ -2,13 +2,11 @@ import base64
import copy
import glob
import ipaddress
import json
import logging
import os
import re
import socket
import time
import traceback
import uuid
from datetime import UTC, datetime, timedelta
from functools import lru_cache, wraps
@ -18,9 +16,7 @@ from typing import Any
from Crypto.Cipher import AES
from app.library.log import get_logger
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("Utils")
ALLOWED_SUBS_EXTENSIONS: set[str] = {".srt", ".vtt", ".ass"}
"Allowed subtitle file extensions."
@ -40,179 +36,11 @@ TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c")
class FileLogFormatter(logging.Formatter):
def formatTime(self, record, datefmt=None): # noqa: N802
_ = datefmt
def formatTime(self, record, datefmt=None): # noqa: ARG002, N802
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
LOG_RECORD_ATTRS: set[str] = set(logging.makeLogRecord({}).__dict__) | {"asctime", "message"}
SKIP_LOG_FIELD = object()
class JsonLogFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
source = {
"path": record.pathname,
"file": record.filename,
"module": record.module,
"function": record.funcName,
"line": record.lineno,
}
data: dict[str, Any] = {
"id": str(uuid.uuid4()),
"datetime": self.formatTime(record),
"level": record.levelname.lower(),
"levelno": record.levelno,
"logger": record.name,
"message": record.getMessage(),
"source": source,
"process": {"id": record.process, "name": record.processName},
"thread": {"id": record.thread, "name": record.threadName},
"fields": self._extra(record),
}
if record.exc_info:
exception = self._exception(record.exc_info)
if exception is not None:
data["exception"] = exception
data["source"].update(self._exception_source(exception))
return json.dumps(data, ensure_ascii=False, default=str)
def formatTime(self, record, datefmt=None): # noqa: N802
_ = datefmt
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
@staticmethod
def _extra(record: logging.LogRecord) -> dict[str, Any]:
extra: dict[str, Any] = {}
for key, value in record.__dict__.items():
if key in LOG_RECORD_ATTRS or key.startswith("_"):
continue
value = JsonLogFormatter._field(value)
if value is not SKIP_LOG_FIELD:
extra[key] = value
return extra
@staticmethod
def _field(value: Any) -> Any:
if isinstance(value, str | int | float | bool) or value is None:
return value
if isinstance(value, Path):
return str(value)
if isinstance(value, dict):
data: dict[str, Any] = {}
for key, item in value.items():
if not isinstance(key, str) or key.startswith("_"):
continue
item = JsonLogFormatter._field(item)
if item is not SKIP_LOG_FIELD:
data[key] = item
return data
if isinstance(value, list):
data: list[Any] = []
for item in value:
item = JsonLogFormatter._field(item)
if item is not SKIP_LOG_FIELD:
data.append(item)
return data
return SKIP_LOG_FIELD
@staticmethod
def _exception(
exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None],
) -> dict[str, Any] | None:
exc = exc_info[1]
if exc is None:
return None
stack = JsonLogFormatter._exception_stack(exc_info)
data: dict[str, Any] = {"type": JsonLogFormatter._exception_type(exc)}
message = str(exc).strip()
if message:
data["message"] = message
if stack:
origin = stack[-1]
data["file"] = origin["path"]
data["line"] = origin["line"]
data["stack"] = stack
return data
@staticmethod
def _exception_type(exc: BaseException) -> str:
module = exc.__class__.__module__
name = exc.__class__.__qualname__
return name if module == "builtins" else f"{module}.{name}"
@staticmethod
def _exception_stack(
exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None],
) -> list[dict[str, Any]]:
exc = exc_info[1]
tb = exc_info[2] if len(exc_info) > 2 else None
if tb is None and exc is not None:
tb = exc.__traceback__
if tb is None:
return []
return [JsonLogFormatter._frame(frame) for frame in traceback.extract_tb(tb)]
@staticmethod
def _frame(frame: traceback.FrameSummary) -> dict[str, Any]:
path = frame.filename
file_path = Path(path)
return {
"path": path,
"file": file_path.name,
"module": file_path.stem,
"function": frame.name,
"line": frame.lineno,
}
@staticmethod
def _exception_source(exception: dict[str, Any]) -> dict[str, Any]:
source: dict[str, Any] = {}
path = exception.get("file")
if isinstance(path, str) and path:
file_path = Path(path)
source["path"] = path
source["file"] = file_path.name
source["module"] = file_path.stem
line = exception.get("line")
if isinstance(line, int):
source["line"] = line
stack = exception.get("stack")
if isinstance(stack, list) and stack:
frame = stack[-1]
if isinstance(frame, dict):
function = frame.get("function")
if isinstance(function, str) and function:
source["function"] = function
module = frame.get("module")
if isinstance(module, str) and module:
source["module"] = module
return source
def timed_lru_cache(ttl_seconds: float, max_size: int = 128):
def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
"""
Decorator that applies an LRU cache with a time-to-live (TTL) to a function.
Supports both synchronous and asynchronous functions.
@ -282,9 +110,8 @@ def timed_lru_cache(ttl_seconds: float, max_size: int = 128):
return _CacheInfo(hits=0, misses=len(cache), maxsize=max_size, currsize=len(cache))
async_wrapper_cache: Any = async_wrapper
async_wrapper_cache.cache_clear = cache_clear
async_wrapper_cache.cache_info = cache_info
async_wrapper.cache_clear = cache_clear
async_wrapper.cache_info = cache_info
return async_wrapper
# For sync functions, use the original implementation
@ -312,9 +139,8 @@ def timed_lru_cache(ttl_seconds: float, max_size: int = 128):
return result
# expose cache_clear, cache_info
sync_wrapper_cache: Any = sync_wrapper
sync_wrapper_cache.cache_clear = cached.cache_clear
sync_wrapper_cache.cache_info = cached.cache_info
sync_wrapper.cache_clear = cached.cache_clear
sync_wrapper.cache_info = cached.cache_info
return sync_wrapper
return decorator
@ -385,12 +211,7 @@ def _is_safe_key(key: Any) -> bool:
def merge_dict(
source: dict,
destination: dict,
max_depth: int = 50,
max_list_size: int = 10000,
_depth: int = 0,
_seen: set | None = None,
source: dict, destination: dict, max_depth: int = 50, max_list_size: int = 10000, _depth: int = 0, _seen: set = None
) -> dict:
"""
Merge data from source into destination.
@ -481,7 +302,7 @@ def merge_dict(
return destination_copy
def check_id(file: Path) -> bool | Path:
def check_id(file: Path) -> bool | str:
"""
Check if we are able to get an id from the file name.
if so check if any video file with the same id exists.
@ -500,8 +321,6 @@ def check_id(file: Path) -> bool | Path:
return False
id: str | None = match.groupdict().get("id")
if id is None:
return False
try:
if not file.parent.exists():
@ -516,11 +335,7 @@ def check_id(file: Path) -> bool | Path:
return f.absolute()
except OSError as e:
LOG.exception(
"Failed to check for a matching file for '%s'.",
file,
extra={"file_path": str(file), "operation": "check_id", "exception_type": type(e).__name__},
)
LOG.error(f"Error checking file '{file}': {e!s}")
return False
return False
@ -556,9 +371,9 @@ def validate_url(url: str, allow_internal: bool = False) -> bool:
from yarl import URL
parsed_url = URL(url)
except ValueError as exc:
except ValueError:
msg = "Invalid URL."
raise ValueError(msg) from exc
raise ValueError(msg) # noqa: B904
# Check allowed schemes
if parsed_url.scheme not in ["http", "https"]:
@ -576,11 +391,7 @@ def validate_url(url: str, allow_internal: bool = False) -> bool:
msg = "Access to internal urls or private networks is not allowed."
raise ValueError(msg)
except socket.gaierror as e:
LOG.exception(
"Failed to resolve hostname '%s'.",
hostname,
extra={"hostname": hostname, "exception_type": type(e).__name__},
)
LOG.error(f"Error resolving hostname '{hostname}': {e!s}")
msg = "Invalid hostname."
raise ValueError(msg) from e
@ -607,7 +418,7 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool:
@timed_lru_cache(ttl_seconds=60, max_size=256)
def get_file_sidecar(file: Path | None = None) -> dict[str, list[dict[str, Any]]]:
def get_file_sidecar(file: Path | None = None) -> dict[dict]:
"""
Get sidecar files for the given file.
@ -708,46 +519,16 @@ def rename_file(old_path: Path, new_name: str) -> tuple[Path, list[tuple[Path, P
renamed_sidecar: Path = old_sidecar.rename(new_sidecar)
renamed_sidecars.append((old_sidecar, renamed_sidecar))
except OSError as e:
LOG.exception(
"Failed to rename sidecar '%s' to '%s'.",
old_sidecar,
new_sidecar,
extra={
"old_path": str(old_sidecar),
"new_path": str(new_sidecar),
"operation": "rename_sidecar",
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to rename sidecar '{old_sidecar}': {e}")
try:
renamed_main.rename(old_path)
except OSError as rollback_error:
LOG.exception(
"Failed to roll back main file rename from '%s' to '%s'.",
renamed_main,
old_path,
extra={
"old_path": str(renamed_main),
"new_path": str(old_path),
"operation": "rename_rollback",
"exception_type": type(rollback_error).__name__,
},
)
except OSError:
LOG.error(f"Failed to rollback main file rename from '{renamed_main}' to '{old_path}'")
raise
return renamed_main, renamed_sidecars
except OSError as e:
LOG.exception(
"Failed to rename '%s' to '%s'.",
old_path,
new_path,
extra={
"old_path": str(old_path),
"new_path": str(new_path),
"operation": "rename",
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to rename '{old_path}' to '{new_path}': {e}")
raise
@ -805,62 +586,24 @@ def move_file(old_path: Path, target_dir: Path) -> tuple[Path, list[tuple[Path,
moved_sidecar: Path = old_sidecar.rename(new_sidecar)
renamed_sidecars.append((old_sidecar, moved_sidecar))
except OSError as e:
LOG.exception(
"Failed to move sidecar '%s' to '%s'.",
old_sidecar,
new_sidecar,
extra={
"old_path": str(old_sidecar),
"new_path": str(new_sidecar),
"operation": "move_sidecar",
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to move sidecar '{old_sidecar}': {e}")
try:
moved_main.rename(old_path)
# Rollback any sidecars that were already moved
for rolled_back_old, rolled_back_new in renamed_sidecars:
try:
rolled_back_new.rename(rolled_back_old)
except OSError as rollback_error:
LOG.exception(
"Failed to roll back sidecar move from '%s' to '%s'.",
rolled_back_new,
rolled_back_old,
extra={
"old_path": str(rolled_back_new),
"new_path": str(rolled_back_old),
"operation": "move_sidecar_rollback",
"exception_type": type(rollback_error).__name__,
},
except OSError:
LOG.error(
f"Failed to rollback sidecar move from '{rolled_back_new}' to '{rolled_back_old}'"
)
except OSError as rollback_error:
LOG.exception(
"Failed to roll back main file move from '%s' to '%s'.",
moved_main,
old_path,
extra={
"old_path": str(moved_main),
"new_path": str(old_path),
"operation": "move_rollback",
"exception_type": type(rollback_error).__name__,
},
)
except OSError:
LOG.error(f"Failed to rollback main file move from '{moved_main}' to '{old_path}'")
raise
return moved_main, renamed_sidecars
except OSError as e:
LOG.exception(
"Failed to move '%s' to '%s'.",
old_path,
new_path,
extra={
"old_path": str(old_path),
"new_path": str(new_path),
"operation": "move",
"exception_type": type(e).__name__,
},
)
LOG.error(f"Failed to move '{old_path}' to '{new_path}': {e}")
raise
@ -936,22 +679,18 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
download_path = Path(download_path)
try:
realFile: Path = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
realFile: str = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
if realFile.exists():
return (realFile, 200)
except Exception as e:
LOG.exception(
"Failed to resolve download file path for '%s'.",
file,
extra={"file_path": str(file), "download_path": str(download_path), "exception_type": type(e).__name__},
)
LOG.error(f"Error calculating download path. {e!s}")
return (Path(file), 404)
possibleFile: bool | Path = check_id(file=realFile)
if not isinstance(possibleFile, Path):
possibleFile: bool | str = check_id(file=realFile)
if not possibleFile:
return (realFile, 404)
return (possibleFile, 302)
return (Path(possibleFile), 302)
def encrypt_data(data: str, key: bytes) -> str:
@ -973,7 +712,7 @@ def encrypt_data(data: str, key: bytes) -> str:
return base64.urlsafe_b64encode(iv + ciphertext + tag).decode()
def decrypt_data(data: str, key: bytes) -> str | None:
def decrypt_data(data: str, key: bytes) -> str:
"""
Decrypts AES-GCM encrypted data
@ -986,8 +725,8 @@ def decrypt_data(data: str, key: bytes) -> str | None:
"""
try:
raw: bytes = base64.urlsafe_b64decode(data)
iv, ciphertext, tag = raw[:12], raw[12:-16], raw[-16:]
data = base64.urlsafe_b64decode(data)
iv, ciphertext, tag = data[:12], data[12:-16], data[-16:]
cipher = AES.new(key, AES.MODE_GCM, nonce=iv)
plaintext = cipher.decrypt_and_verify(ciphertext, tag)
return plaintext.decode()
@ -998,7 +737,7 @@ def decrypt_data(data: str, key: bytes) -> str | None:
def get(
data: dict | list,
path: str | list | None = None,
default: Any = None,
default: any = None,
separator=".",
):
"""
@ -1115,42 +854,22 @@ def get_files(
try:
dir_path = dir_path.resolve()
except OSError as e:
LOG.warning(
"Failed to resolve '%s': %s",
dir,
e,
extra={"requested_dir": dir, "base_path": str(base_path), "exception_type": type(e).__name__},
exc_info=True,
)
return [], 0
LOG.warning(f"Failed to resolve '{dir}' - {e}")
return []
try:
dir_path.relative_to(base_path)
except ValueError:
LOG.warning(
"Invalid path '%s': must be inside '%s'.",
dir_path,
base_path,
extra={"path": str(dir_path), "base_path": str(base_path)},
)
return [], 0
LOG.warning(f"Invalid path: '{dir_path}' - must be inside '{base_path}'.")
return []
if not str(dir_path).startswith(str(base_path)):
LOG.warning(
"Invalid path '%s': must be inside '%s'.",
dir_path,
base_path,
extra={"path": str(dir_path), "base_path": str(base_path)},
)
return [], 0
LOG.warning(f"Invalid path: '{dir_path}' - must be inside '{base_path}'.")
return []
if not dir_path.is_dir():
LOG.warning(
"Invalid path '%s': must be a directory.",
dir_path,
extra={"path": str(dir_path), "base_path": str(base_path)},
)
return [], 0
LOG.warning(f"Invalid path: '{dir_path}' - must be a directory.")
return []
contents: list = []
for file in dir_path.iterdir():
@ -1163,28 +882,13 @@ def get_files(
test: Path = file.resolve()
test.relative_to(base_path)
if not str(test).startswith(str(base_path)):
LOG.warning(
"Invalid symlink '%s': must resolve inside '%s'.",
file,
base_path,
extra={"path": str(file), "resolved_path": str(test), "base_path": str(base_path)},
)
LOG.warning(f"Invalid symlink: '{file}' - must resolve inside '{base_path}'.")
continue
except ValueError:
LOG.warning(
"Invalid symlink '%s': must resolve inside '%s'.",
file,
base_path,
extra={"path": str(file), "base_path": str(base_path)},
)
LOG.warning(f"Invalid symlink: '{file}' - must resolve inside '{base_path}'.")
continue
except OSError as e:
LOG.warning(
"Skipping broken symlink '%s'.",
file,
extra={"path": str(file), "base_path": str(base_path), "exception_type": type(e).__name__},
exc_info=True,
)
except OSError:
LOG.warning(f"Skipping broken symlink: {file}")
continue
content_type = None
@ -1309,7 +1013,7 @@ def load_cookies(file: str | Path) -> tuple[bool, MozillaCookieJar]:
try:
from http.cookiejar import MozillaCookieJar
cookies = MozillaCookieJar(str(file), delayload=False, policy=None)
cookies = MozillaCookieJar(str(file), None, None)
cookies.load()
return (True, cookies)
@ -1374,11 +1078,7 @@ def delete_dir(dir: Path) -> bool:
dir.rmdir()
return True
except Exception as e:
LOG.exception(
"Failed to delete directory '%s'.",
dir,
extra={"dir": str(dir), "exception_type": type(e).__name__},
)
LOG.error(f"Failed to delete directory '{dir}': {e}")
return False
@ -1426,11 +1126,7 @@ def load_modules(root_path: Path, directory: Path):
try:
importlib.import_module(full_name)
except ImportError as e:
LOG.exception(
"Failed to import module '%s'.",
full_name,
extra={"module_name": full_name, "exception_type": type(e).__name__},
)
LOG.error(f"Failed to import module '{full_name}': {e}")
def parse_tags(text: str) -> tuple[str, dict[str, str | bool]]:
@ -1488,6 +1184,35 @@ def str_to_dt(time_str: str, now=None) -> datetime:
return dt
def list_folders(path: Path, base: Path, depth_limit: int) -> list[str]:
"""
List all folders relative to a base path, up to a specified depth limit.
Args:
path (Path): The path to start listing folders from.
base (Path): The base path to which the folders should be relative.
depth_limit (int): The maximum depth to traverse from the base path.
Returns:
list[str]: A list of folder paths relative to the base path, up to the specified
"""
if "/" == str(path):
return []
rel_depth: int = len(path.relative_to(base).parts)
if rel_depth > depth_limit:
return []
folders: list[str] = []
for entry in path.iterdir():
if entry.is_dir():
folders.append(str(entry.relative_to(base)))
folders.extend(list_folders(entry, base, depth_limit))
return folders
def get_channel_images(thumbnails: list[dict]) -> dict:
"""
Extract channel images from a list of thumbnail dictionaries.

View file

@ -1,20 +1,6 @@
import sys
from typing import Any
def _dict_value(data: dict[Any, Any], key: Any) -> tuple[bool, Any]:
if key not in data or data[key] is None:
return False, None
return True, data[key]
def _dict_delete(data: dict[Any, Any], key: Any) -> bool:
if key not in data:
return False
del data[key]
return True
def get_value(value: Any) -> Any:
"""
Return the result of calling `value` if it is callable, else return `value`.
@ -105,19 +91,14 @@ def ag(
return val
return get_value(default)
key = path
if isinstance(array, dict):
found, value = _dict_value(array, key)
if found:
return value
if isinstance(array, dict) and key in array and array[key] is not None:
return array[key]
if not isinstance(key, str) or separator not in key:
return get_value(default)
current = array
for segment in key.split(separator):
if isinstance(current, dict):
found, value = _dict_value(current, segment)
if not found:
return get_value(default)
current = value
if isinstance(current, dict) and segment in current and current[segment] is not None:
current = current[segment]
elif isinstance(current, list):
try:
idx = int(segment)
@ -169,8 +150,7 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
except TypeError:
return False
if isinstance(data, dict):
found, _ = _dict_value(data, path)
if found:
if path in data and data[path] is not None:
return True
elif isinstance(data, list) and isinstance(path, int):
return 0 <= path < len(data) and data[path] is not None
@ -178,11 +158,8 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
segments = path_str.split(separator)
current = data
for seg in segments:
if isinstance(current, dict):
found, value = _dict_value(current, seg)
if not found:
return False
current = value
if isinstance(current, dict) and seg in current and current[seg] is not None:
current = current[seg]
elif isinstance(current, list):
try:
idx = int(seg)
@ -212,16 +189,17 @@ def ag_delete(
The modified structure.
"""
if isinstance(path, list | tuple):
for p in path:
ag_delete(data, p, separator)
return data
if not isinstance(data, dict | list):
try:
data = vars(data)
except TypeError:
return data # type: ignore
if isinstance(path, list | tuple):
for p in path:
ag_delete(data, p, separator)
return data
if isinstance(data, dict) and _dict_delete(data, path):
if isinstance(data, dict) and path in data:
del data[path]
return data
if isinstance(data, list) and isinstance(path, int):
if 0 <= path < len(data):
@ -232,12 +210,8 @@ def ag_delete(
last = segments.pop()
current = data
for seg in segments:
if isinstance(current, dict):
found, value = _dict_value(current, seg)
if not found:
current = None
break
current = value
if isinstance(current, dict) and seg in current:
current = current[seg]
elif isinstance(current, list):
try:
idx = int(seg)
@ -254,8 +228,8 @@ def ag_delete(
break
if current is None:
return data
if isinstance(current, dict):
_dict_delete(current, last)
if isinstance(current, dict) and last in current:
del current[last]
elif isinstance(current, list):
try:
idx = int(last)
@ -272,12 +246,10 @@ def run_tests():
assert ag_set(d, "a.c.d", 42) == {"a": {"b": 1, "c": {"d": 42}}}
def test_ag_set_overwrite_error():
error = ""
try:
ag_set({"a": 1}, "a.b", 2)
except RuntimeError as e:
error = str(e)
assert "Cannot set value at path" in error
assert "Cannot set value at path" in str(e) # noqa: PT017
def test_ag_get_value_callable_and_value():
assert get_value(5) == 5
@ -348,9 +320,9 @@ def run_tests():
]:
try:
test()
sys.stdout.write(f"PASS: {test.__name__}\n")
print(f"PASS: {test.__name__}") # noqa: T201
except AssertionError:
sys.stdout.write(f"FAIL {test.__name__}\n")
print(f"FAIL {test.__name__}") # noqa: T201
if __name__ == "__main__":

View file

@ -1,17 +1,17 @@
import hashlib
import logging
import threading
import time
from typing import Any
from aiohttp import web
from app.library.log import get_logger
from app.library.Services import Services
from .Scheduler import Scheduler
from .Singleton import ThreadSafe
LOG = get_logger()
LOG = logging.getLogger("cache")
class Cache(metaclass=ThreadSafe):
@ -38,7 +38,7 @@ class Cache(metaclass=ThreadSafe):
Scheduler.get_instance().add(
timer="* * * * *",
func=self.cleanup,
id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
id=f"{__class__.__name__}.{__class__.cleanup.__name__}",
)
def set(self, key: str, value: Any, ttl: float | None = None) -> None:
@ -175,8 +175,4 @@ class Cache(metaclass=ThreadSafe):
del self._cache[key]
if expired_keys:
LOG.debug(
"Cleaned up %s expired cache entries.",
len(expired_keys),
extra={"expired_count": len(expired_keys)},
)
LOG.debug(f"Cleaned up {len(expired_keys)} expired cache entries.")

View file

@ -1,9 +1,10 @@
from __future__ import annotations
import http.cookiejar
import logging
from abc import ABC
from collections.abc import Callable
from typing import Any, ClassVar, cast
from typing import Any, ClassVar
from yt_dlp.networking.common import (
_REQUEST_HANDLERS,
@ -19,9 +20,8 @@ from yt_dlp.networking.exceptions import HTTPError
from yt_dlp.utils.networking import clean_headers
from app.library.cf_solver_shared import CACHE, is_cf_challenge, solver
from app.library.log import get_logger
LOG = get_logger()
LOG: logging.Logger = logging.getLogger(__name__)
SolverFn = Callable[[Request, Response, RequestHandler], Request | None]
@ -215,7 +215,7 @@ class CFSolverRH(RequestHandler, ABC):
def _send(self, request: Request) -> Response:
host: str = self._get_host(request.url)
if host and (cached := CACHE.get(host)):
LOG.info("Injecting cached Cloudflare cookies for '%s'.", host, extra={"host": host})
LOG.info(f"Injecting cached Cloudflare cookies for '{host}'.")
self._apply_solution(cached, request.url, request.headers, self._get_cookiejar(request))
director: RequestDirector = self._build_fallback()
@ -223,13 +223,11 @@ class CFSolverRH(RequestHandler, ABC):
try:
response: Response = director.send(request)
except HTTPError as error:
if error.response and is_cf_challenge(
getattr(error.response, "status", None), cast("Any", error.response.headers)
):
if error.response and is_cf_challenge(getattr(error.response, "status", None), error.response.headers):
return self._retry_with_clearance(request, error.response, director)
raise
if is_cf_challenge(getattr(response, "status", None), cast("Any", response.headers)):
if is_cf_challenge(getattr(response, "status", None), response.headers):
return self._retry_with_clearance(request, response, director)
return response

View file

@ -1,20 +1,17 @@
# flake8: noqa: S310
from __future__ import annotations
import json
import logging
import time
import urllib.request
from typing import TYPE_CHECKING, Any
from typing import Any
from urllib.parse import urlparse
from app.library.log import get_logger
from .cache import Cache
if TYPE_CHECKING:
from collections.abc import Mapping
CACHE: Cache = Cache()
LOG = get_logger()
LOG: logging.Logger = logging.getLogger("cf_solver")
def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> dict[str, Any] | None:
@ -59,36 +56,23 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
if user_agent:
payload.setdefault("headers", {})["User-Agent"] = user_agent
req = urllib.request.Request( # noqa: S310
req = urllib.request.Request(
endpoint,
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
LOG.info(
"Solving Cloudflare challenge for '%s' via FlareSolverr.", host, extra={"host": host, "endpoint": endpoint}
)
LOG.info(f"Solving Cloudflare challenge for '{host}' via FlareSolverr.")
start_time = time.time()
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp: # noqa: S310
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp:
result = json.loads(resp.read().decode("utf-8"))
if "ok" != result.get("status"):
LOG.error(
"FlareSolverr failed to solve challenge for '%s': %s",
host,
result.get("message"),
extra={"host": host, "endpoint": endpoint, "solver_message": result.get("message")},
)
LOG.error(f"FlareSolverr failed to solve challenge for '{host}': {result.get('message')}")
return None
elapsed_s: float = time.time() - start_time
LOG.info(
"FlareSolverr solved challenge for '%s' in %.2f seconds.",
host,
elapsed_s,
extra={"host": host, "endpoint": endpoint, "elapsed_s": round(elapsed_s, 2)},
)
LOG.info(f"FlareSolverr successfully solved challenge for '{host}'. (Took {time.time() - start_time:.2f} seconds)")
solution = result.get("solution") or {}
CACHE.set(
@ -100,7 +84,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
return CACHE.get(host)
def is_cf_challenge(status: int | None, headers: Mapping[str, Any] | None) -> bool:
def is_cf_challenge(status: int | None, headers: dict[str, Any] | None) -> bool:
"""
Determine whether a response indicates a Cloudflare challenge.
"""

View file

@ -2,7 +2,6 @@ import logging
import multiprocessing
import os
import re
import shutil
import sys
import time
from logging.handlers import TimedRotatingFileHandler
@ -13,25 +12,14 @@ from typing import TYPE_CHECKING, Any
import coloredlogs
from dotenv import load_dotenv
from app.library.log import get_logger
from .Singleton import Singleton
from .Utils import JsonLogFormatter
from .Utils import FileLogFormatter
from .version import APP_BRANCH, APP_BUILD_DATE, APP_COMMIT_SHA, APP_VERSION
APP_THIRD_PARTY_LOG_LEVELS: tuple[tuple[str, int], ...] = (
("httpx", logging.WARNING),
("urllib3.connectionpool", logging.WARNING),
("apprise", logging.WARNING),
("httpcore", logging.INFO),
("aiosqlite", logging.INFO),
("asyncio", logging.INFO),
)
if TYPE_CHECKING:
from subprocess import CompletedProcess
SUPPORTED_CODECS: tuple[str, ...] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264")
SUPPORTED_CODECS: tuple[str] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264")
"Supported encoder names in order of preference."
@ -48,6 +36,9 @@ class Config(metaclass=Singleton):
download_path: str = "."
"""The path to the download directory."""
download_path_depth: int = 2
"""How many subdirectories to show in auto complete."""
download_info_expires: int = 10800
"""How long (in seconds) the download info is valid before it needs to be re-extracted."""
@ -84,6 +75,9 @@ class Config(metaclass=Singleton):
log_level: str = "info"
"""The log level to use for the application."""
log_level_file: str = "info"
"""The log level to use for the file logging."""
base_path: str = "/"
"""The base path to use for the application."""
@ -108,9 +102,6 @@ class Config(metaclass=Singleton):
auth_password: str | None = None
"""The password to use for basic authentication."""
disable_exec: bool = False
"""Strip some dangerous yt-dlp options."""
remove_files: bool = False
"""Remove downloaded files when removing the record."""
@ -129,15 +120,6 @@ class Config(metaclass=Singleton):
extract_info_concurrency: int = 4
"""The number of concurrent extract_info calls allowed."""
thumb_concurrency: int = 2
"""The number of concurrent ffmpeg thumbnail generations allowed."""
thumb_generate: bool = True
"""Enable ffmpeg thumbnail generation when no local thumbnail exists."""
thumb_sidecar: bool = False
"""Save generated thumbnails next to the media file instead of temp cache."""
db_file: str = "{config_path}{os_sep}ytptube.db"
"""The path to the database file."""
@ -180,7 +162,7 @@ class Config(metaclass=Singleton):
file_logging: bool = True
"Enable file logging."
secret_key: str | bytes
secret_key: str
"The secret key to use for the application."
tasks_handler_timer: str = "15 */1 * * *"
@ -228,9 +210,6 @@ class Config(metaclass=Singleton):
default_pagination: int = 50
"""The default number of items per page for pagination."""
queue_display_limit: int = 100
"""Maximum number of queued downloads returned to the UI. 0 means unlimited."""
task_handler_random_delay: float = 60.0
"""The maximum random delay in seconds before starting a task handler."""
@ -283,12 +262,11 @@ class Config(metaclass=Singleton):
"max_workers_per_extractor",
"extract_info_timeout",
"debugpy_port",
"download_path_depth",
"download_info_expires",
"auto_clear_history_days",
"default_pagination",
"queue_display_limit",
"extract_info_concurrency",
"thumb_concurrency",
"flaresolverr_max_timeout",
"flaresolverr_client_timeout",
"flaresolverr_cache_ttl",
@ -314,9 +292,6 @@ class Config(metaclass=Singleton):
"simple_mode",
"ignore_archived_items",
"check_for_updates",
"thumb_generate",
"thumb_sidecar",
"disable_exec",
)
"The variables that are booleans."
@ -326,7 +301,6 @@ class Config(metaclass=Singleton):
_frontend_vars: tuple = (
"download_path",
"keep_archive",
"log_level",
"output_template",
"started",
"remove_files",
@ -347,7 +321,6 @@ class Config(metaclass=Singleton):
"app_build_date",
"app_branch",
"default_pagination",
"queue_display_limit",
"check_for_updates",
"new_version",
"yt_new_version",
@ -375,13 +348,12 @@ class Config(metaclass=Singleton):
def __init__(self, is_native: bool = False):
baseDefaultPath: str = str(Path(__file__).parent.parent.parent.absolute())
LOG = get_logger()
self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or str(Path(baseDefaultPath) / "var" / "config")
envFile: Path = Path(self.config_path) / ".env"
envFile: str = Path(self.config_path) / ".env"
if envFile.exists():
LOG.info("Loading environment variables from '%s'.", envFile)
logging.info(f"Loading environment variables from '{envFile}'.")
load_dotenv(envFile)
self.is_native = is_native
@ -389,7 +361,7 @@ class Config(metaclass=Singleton):
self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or str(
Path(baseDefaultPath) / "var" / "downloads"
)
self.app_path = str(Path(__file__).parent.parent.absolute())
self.app_path = Path(__file__).parent.parent.absolute()
for k, v in self._get_attributes().items():
if k.startswith("_") or k in self._manual_vars:
@ -411,7 +383,7 @@ class Config(metaclass=Singleton):
for key in re.findall(r"\{.*?\}", v):
localKey: str = key[1:-1]
if localKey not in self.__dict__:
LOG.error("Config variable '%s' had non-existing config reference '%s'.", k, key)
logging.error(f"Config variable '{k}' had non-existing config reference '{key}'.")
sys.exit(1)
v: str = v.replace(key, str(getattr(self, localKey)))
@ -449,23 +421,18 @@ class Config(metaclass=Singleton):
encoding="utf-8",
)
LOG: logging.Logger = logging.getLogger("config")
if self.debug:
try:
import debugpy
debugpy.listen(("0.0.0.0", self.debugpy_port), in_process_debug_adapter=True)
LOG.info(
"Starting debugpy server on '0.0.0.0:%s'.",
self.debugpy_port,
extra={"host": "0.0.0.0", "port": self.debugpy_port},
)
LOG.info(f"starting debugpy server on '0.0.0.0:{self.debugpy_port}'.")
except ImportError:
LOG.error("debugpy package not found; install it with 'uv sync'.")
LOG.error("debugpy package not found, install it with 'uv sync'.")
except Exception as e:
LOG.exception(
"Failed to start debugpy server.",
extra={"host": "0.0.0.0", "port": self.debugpy_port, "exception_type": type(e).__name__},
)
LOG.error(f"Error starting debugpy server at '0.0.0.0:{self.debugpy_port}'. {e}")
if (Path(self.config_path) / "ytdlp.cli").exists():
LOG.error("Support for ./ytdlp.cli file is removed, migrate to presets and remove the file.")
@ -474,16 +441,12 @@ class Config(metaclass=Singleton):
LOG.warning("Keep temp files option is enabled.")
if self.auth_password and self.auth_username:
LOG.info(
"Basic authentication is enabled for user '%s'.",
self.auth_username,
extra={"auth_username": self.auth_username},
)
LOG.info(f"Authentication enabled with username '{self.auth_username}'.")
if self.file_logging:
file_log_level: int | None = getattr(logging, self.log_level.upper(), None)
if not isinstance(file_log_level, int):
msg = f"Invalid log level '{self.log_level}' specified."
log_level_file: int | None = getattr(logging, self.log_level_file.upper(), None)
if not isinstance(log_level_file, int):
msg = f"Invalid file log level '{self.log_level_file}' specified."
raise TypeError(msg)
loggingPath: Path = Path(self.config_path) / "logs"
@ -491,18 +454,18 @@ class Config(metaclass=Singleton):
loggingPath.mkdir(parents=True, exist_ok=True)
handler = TimedRotatingFileHandler(
filename=loggingPath / "app.jsonl",
filename=loggingPath / "app.log",
when="midnight",
backupCount=3,
encoding="utf-8",
)
handler.setLevel(file_log_level)
formatter = JsonLogFormatter()
handler.setLevel(log_level_file)
formatter = FileLogFormatter("%(asctime)s [%(levelname)s.%(name)s]: %(message)s")
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
key_file: Path = Path(self.config_path) / "secret.key"
key_file: str = Path(self.config_path) / "secret.key"
if key_file.exists() and key_file.stat().st_size > 2:
with open(key_file, "rb") as f:
@ -512,9 +475,16 @@ class Config(metaclass=Singleton):
with open(key_file, "wb") as f:
f.write(self.secret_key)
self.started = int(time.time())
self.started = time.time()
for _tool, _level in APP_THIRD_PARTY_LOG_LEVELS:
_log_levels = (
("httpx", logging.WARNING),
("urllib3.connectionpool", logging.WARNING),
("apprise", logging.WARNING),
("httpcore", logging.INFO),
("aiosqlite", logging.INFO),
)
for _tool, _level in _log_levels:
logging.getLogger(_tool).setLevel(_level)
if self.app_env not in ("production", "development"):
@ -546,7 +516,7 @@ class Config(metaclass=Singleton):
def _get_attributes(self) -> dict:
attrs: dict = {}
vClass: type = self.__class__
vClass: str = self.__class__
for attribute in vClass.__dict__:
if attribute.startswith("_"):
@ -589,9 +559,6 @@ class Config(metaclass=Singleton):
data: dict[str, Any] = {k: getattr(self, k) for k in self._frontend_vars}
data["ytdlp_version"] = Config._ytdlp_version()
from app.library.log_control import get_runtime_log_level
data["runtime_log_level"] = get_runtime_log_level()
return data
def get_replacers(self) -> dict[str, str]:
@ -602,7 +569,7 @@ class Config(metaclass=Singleton):
dict[str, str]: The replacer variables.
"""
keys: tuple[str, ...] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file")
keys: tuple[str] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file")
return {k: getattr(self, k) for k in keys}
@staticmethod
@ -619,21 +586,15 @@ class Config(metaclass=Singleton):
Updates the version of the application using git tags.
This is used to set the version to the latest git tag.
"""
LOG = get_logger()
git_path: Path = Path(__file__).parent / ".." / ".." / ".git"
git_path: str = Path(__file__).parent / ".." / ".." / ".git"
if not git_path.exists():
return
try:
import subprocess
git = shutil.which("git")
if not git:
LOG.warning("Git executable was not found.")
return
branch_result: CompletedProcess[str] = subprocess.run(
[git, "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"],
["git", "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"], # noqa: S607
cwd=os.path.dirname(git_path),
capture_output=True,
text=True,
@ -642,16 +603,16 @@ class Config(metaclass=Singleton):
)
if 0 != branch_result.returncode:
LOG.error("Git rev-parse failed: %s", branch_result.stderr.strip())
logging.error(f"Git rev-parse failed: {branch_result.stderr.strip()}")
return
branch_name: str = branch_result.stdout.strip()
if not branch_name:
LOG.warning("Git branch name is empty.")
logging.warning("Git branch name is empty.")
return
commit_result: CompletedProcess[str] = subprocess.run(
[git, "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"],
["git", "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"], # noqa: S607
cwd=os.path.dirname(git_path),
capture_output=True,
text=True,
@ -660,12 +621,12 @@ class Config(metaclass=Singleton):
)
if 0 != commit_result.returncode:
LOG.error("Git log failed: %s", commit_result.stderr.strip())
logging.error(f"Git log failed: {commit_result.stderr.strip()}")
return
commit_info: str = commit_result.stdout.strip()
if not commit_info:
LOG.warning("Git commit info is empty.")
logging.warning("Git commit info is empty.")
return
commit_date, commit_sha = commit_info.split("_", 1)
@ -681,6 +642,6 @@ class Config(metaclass=Singleton):
"commit": self.app_commit_sha,
"build_date": self.app_build_date,
}
LOG.info("Application version info set to '%s'", version_data)
logging.info(f"Application version info set to '{version_data}'")
except Exception as e:
LOG.error("Error while getting git version: %s", e)
logging.error(f"Error while getting git version: {e!s}")

View file

@ -1,760 +0,0 @@
from __future__ import annotations
import asyncio
import importlib.metadata
import os
import platform
import re
import shutil
import sqlite3
import subprocess
import sys
import time
import urllib.parse
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Literal
from app.library.config import Config
from app.library.httpx_client import resolve_curl_transport
CheckStatus = Literal["pass", "fail", "warn", "skip"]
ReportStatus = Literal["ok", "degraded", "error"]
MIN_PYTHON: tuple[int, int] = (3, 13)
@dataclass(kw_only=True)
class DiagnosticCheck:
id: str
label: str
group: str
required: bool
status: CheckStatus
description: str
message: str
details: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return {
"id": self.id,
"label": self.label,
"group": self.group,
"required": self.required,
"status": self.status,
"description": self.description,
"message": self.message,
"details": self.details,
}
@dataclass(frozen=True, kw_only=True)
class CheckMeta:
group: str
description: str
BINARY_META: dict[str, CheckMeta] = {
"ffmpeg": CheckMeta(
group="core", description="Used for thumbnails and streaming and various media processing tasks."
),
"ffprobe": CheckMeta(group="core", description="Used for media inspection and metadata extraction."),
"deno": CheckMeta(group="youtube", description="Used for yt-dlp YouTube support."),
"yt_dlp_cli": CheckMeta(group="core", description="Used by the built-in terminal."),
"aria2c": CheckMeta(group="advanced", description="External downloader for yt-dlp."),
"mkvpropedit": CheckMeta(group="advanced", description="Edit Matroska container properties without remux."),
"mkvextract": CheckMeta(group="advanced", description="Extract tracks from Matroska containers."),
"mp4box": CheckMeta(group="advanced", description="MP4 container manipulation tool."),
}
DIRECTORY_DESCRIPTIONS: dict[str, str] = {
"config_path": "Directory for config files and app data.",
"download_path": "Directory where downloads are saved.",
"temp_path": "Directory used for temporary files.",
}
def _first_line(*parts: str) -> str:
for part in parts:
for line in part.splitlines():
if line := line.strip():
return line
return ""
def _bin_version(value: str) -> str:
if not (line := value.strip()):
return ""
if match := re.match(r"^deno\s+([0-9][^\s]*)", line, flags=re.IGNORECASE):
return match.group(1).strip()
if match := re.match(r"^mkv\w+\s+v([0-9][^\s]*)", line, flags=re.IGNORECASE):
return match.group(1).strip()
if match := re.search(r"\bversion\s+(.+?)(?:\s+Copyright\b|$)", line, flags=re.IGNORECASE):
return match.group(1).strip()
return line
def _parse_browser_url(value: str) -> tuple[str, str]:
if value.startswith(("selenium+http://", "selenium+https://")):
return "selenium", value.removeprefix("selenium+")
if value.startswith(("playwright+ws://", "playwright+wss://")):
return "playwright", value.removeprefix("playwright+")
if value.startswith("playwright+cdp://"):
return "playwright", f"http://{value.removeprefix('playwright+cdp://')}"
if value.startswith("playwright+cdp+"):
return "playwright", value.removeprefix("playwright+cdp+")
return "unknown", value
def _safe_url(url: str) -> str:
try:
parsed = urllib.parse.urlsplit(url)
except Exception:
return "***"
if not parsed.scheme or not parsed.netloc:
return "***"
netloc = parsed.netloc
if parsed.username or parsed.password:
host = parsed.hostname or ""
if parsed.port:
host = f"{host}:{parsed.port}"
netloc = f"***:***@{host}" if host else "***:***"
path = "/***" if parsed.path and parsed.path != "/" else parsed.path
query = "***" if parsed.query else ""
fragment = "***" if parsed.fragment else ""
return urllib.parse.urlunsplit((parsed.scheme, netloc, path, query, fragment))
def _package_version(name: str) -> str | None:
try:
return importlib.metadata.version(name)
except importlib.metadata.PackageNotFoundError:
return None
def _package_check(
name: str,
*,
label: str,
group: str,
required: bool,
description: str,
present_message: str = "Installed.",
missing_message: str = "Missing.",
missing_status: CheckStatus = "skip",
check_id: str | None = None,
details: dict[str, Any] | None = None,
) -> DiagnosticCheck:
version = _package_version(name)
installed = version is not None
base_details: dict[str, Any] = {"package": name, "version": version}
if details:
base_details.update(details)
return DiagnosticCheck(
id=check_id or name.replace("-", "_"),
label=label,
group=group,
required=required,
status="pass" if installed else ("fail" if required else missing_status),
description=description,
message=present_message if installed else missing_message,
details=base_details,
)
def _check_ytdlp_package() -> DiagnosticCheck:
version = Config._ytdlp_version()
installed = version != "0.0.0"
return DiagnosticCheck(
id="yt_dlp_package",
label="yt-dlp package",
group="core",
required=True,
status="pass" if installed else "fail",
description="Main downloader library used by the app.",
message="Installed." if installed else "Missing.",
details={"version": version},
)
def _check_apprise_package(config: Config) -> DiagnosticCheck:
configured = Path(config.apprise_config).exists()
return _package_check(
"apprise",
label="Apprise",
group="notifications",
required=False,
description="Sends notification targets.",
present_message="Installed.",
missing_message="Missing." if configured else "Not installed.",
missing_status="warn" if configured else "skip",
check_id="apprise",
details={"config": str(config.apprise_config) if configured else None},
)
def _check_curl_transport() -> DiagnosticCheck:
available = resolve_curl_transport(use_curl=True)
version = _package_version("httpx-curl-cffi")
return DiagnosticCheck(
id="curl_transport",
label="curl-cffi transport",
group="advanced",
required=False,
status="pass" if available else "skip",
description="Transport for impersonation support.",
message="Installed." if available else "Not installed.",
details={"package": "httpx-curl-cffi", "version": version},
)
def _check_pot_provider_package() -> DiagnosticCheck:
return _package_check(
"bgutil-ytdlp-pot-provider",
check_id="pot_provider_plugin",
label="POT provider plugin",
group="youtube",
required=False,
description="Optional plugin for external POT token providers for youtube.",
present_message="Installed.",
missing_message="Not installed.",
)
def _check_configured_pip_packages(config: Config) -> list[DiagnosticCheck]:
checks: list[DiagnosticCheck] = []
for raw_pkg in config.pip_packages.split(" "):
pkg = raw_pkg.strip()
if not pkg:
continue
name = pkg.split("==", 1)[0].split(">=", 1)[0].split("<=", 1)[0].strip()
installed = False
version = None
try:
version = importlib.metadata.version(name)
installed = True
except importlib.metadata.PackageNotFoundError:
installed = False
checks.append(
DiagnosticCheck(
id=f"pip_{name.replace('-', '_')}",
label=name,
group="custom",
required=False,
status="pass" if installed else "warn",
description="Configured extra pip package.",
message="Installed." if installed else "Missing.",
details={"package": name, "requested": pkg, "version": version},
)
)
return checks
def _check_browser_endpoint() -> DiagnosticCheck:
raw_url = (os.environ.get("YTP_BROWSER_URL") or "").strip()
if not raw_url:
return DiagnosticCheck(
id="browser_endpoint",
label="Remote browser",
group="advanced",
required=False,
status="skip",
description="Endpoint used by the browser extractor.",
message="Not configured.",
details={},
)
engine, parsed_url = _parse_browser_url(raw_url)
if engine == "unknown":
return DiagnosticCheck(
id="browser_endpoint",
label="Remote browser",
group="advanced",
required=False,
status="warn",
description="Endpoint used by the browser extractor.",
message="Invalid URL.",
details={"endpoint": _safe_url(raw_url)},
)
return DiagnosticCheck(
id="browser_endpoint",
label="Remote browser",
group="advanced",
required=False,
status="pass",
description="Endpoint used by the browser extractor.",
message="Configured.",
details={
"engine": engine,
"endpoint": _safe_url(parsed_url),
},
)
def _check_browser_client() -> DiagnosticCheck:
raw_url = (os.environ.get("YTP_BROWSER_URL") or "").strip()
if not raw_url:
return DiagnosticCheck(
id="browser_client",
label="Browser client",
group="advanced",
required=False,
status="skip",
description="Client package required for the configured browser backend.",
message="Not configured.",
details={},
)
engine, _ = _parse_browser_url(raw_url)
if engine == "unknown":
return DiagnosticCheck(
id="browser_client",
label="Browser client",
group="advanced",
required=False,
status="skip",
description="Client package required for the configured browser backend.",
message="Invalid URL.",
details={},
)
package_name = "selenium" if engine == "selenium" else "playwright"
version = _package_version(package_name)
return DiagnosticCheck(
id="browser_client",
label="Browser client",
group="advanced",
required=False,
status="pass" if version else "warn",
description="Client package required for the configured browser backend.",
message="Installed." if version else "Missing.",
details={"package": package_name, "version": version},
)
def _check_flaresolverr(config: Config) -> DiagnosticCheck:
endpoint = (config.flaresolverr_url or "").strip()
if not endpoint:
return DiagnosticCheck(
id="flaresolverr",
label="FlareSolverr",
group="advanced",
required=False,
status="skip",
description="Optional Cloudflare challenge bypass service.",
message="Not configured.",
details={},
)
parsed = urllib.parse.urlsplit(endpoint)
valid = parsed.scheme in {"http", "https"} and bool(parsed.netloc)
return DiagnosticCheck(
id="flaresolverr",
label="FlareSolverr",
group="advanced",
required=False,
status="pass" if valid else "warn",
description="Optional Cloudflare challenge bypass service.",
message="Configured." if valid else "Invalid URL.",
details={"endpoint": _safe_url(endpoint)},
)
def _probe_directory(path: Path) -> tuple[CheckStatus, str]:
if not path.exists():
return "fail", "Directory does not exist."
if not path.is_dir():
return "fail", "Path is not a directory."
if not os.access(path, os.R_OK | os.W_OK | os.X_OK):
return "fail", "Not readable or writable."
return "pass", "Writable."
async def _check_directory(path: Path, *, check_id: str, label: str) -> DiagnosticCheck:
status, message = await asyncio.to_thread(_probe_directory, path)
return DiagnosticCheck(
id=check_id,
label=label,
group="core",
required=True,
status=status,
description=DIRECTORY_DESCRIPTIONS.get(check_id, "Directory check."),
message=message,
details={"path": str(path)},
)
def _probe_db_file(path: Path) -> tuple[CheckStatus, str]:
if not path.exists():
return "fail", "Missing."
if not path.is_file():
return "fail", "Invalid path."
if not os.access(path, os.R_OK):
return "fail", "Not readable."
if not os.access(path, os.W_OK):
return "fail", "Not writable."
try:
uri = f"file:{urllib.parse.quote(str(path))}?mode=ro"
with sqlite3.connect(uri, timeout=3, uri=True) as conn:
conn.execute("SELECT name FROM sqlite_master LIMIT 1")
except sqlite3.Error as exc:
return "fail", f"SQLite error. {exc!s}"
return "pass", "Ready."
async def _check_db_file(path: Path) -> DiagnosticCheck:
status, message = await asyncio.to_thread(_probe_db_file, path)
return DiagnosticCheck(
id="database",
label="SQLite database",
group="core",
required=True,
status=status,
description="Local database used for app state and history.",
message=message,
details={"path": str(path)},
)
async def _run_command(command: str, *args: str, wait_seconds: float = 5.0) -> tuple[int, str, str]:
proc = await asyncio.create_subprocess_exec(
command,
*args,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
)
try:
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=wait_seconds)
except TimeoutError:
proc.kill()
await proc.communicate()
raise
return proc.returncode or 0, stdout.decode("utf-8", errors="replace"), stderr.decode("utf-8", errors="replace")
def _binary_meta(check_id: str) -> CheckMeta:
return BINARY_META.get(check_id, CheckMeta(group="core", description="Command line tool."))
def _make_binary_check(
command: str,
*,
check_id: str,
label: str,
required: bool,
status: CheckStatus,
message: str,
path: str | None = None,
version: str | None = None,
) -> DiagnosticCheck:
meta = _binary_meta(check_id)
details: dict[str, Any] = {"command": command}
if path:
details["path"] = path
if version:
details["version"] = version
return DiagnosticCheck(
id=check_id,
label=label,
group=meta.group,
required=required,
status=status,
description=meta.description,
message=message,
details=details,
)
def _resolve_binary(*names: str) -> tuple[str, str | None]:
for name in names:
path = shutil.which(name)
if path:
return name, path
return names[0], None
async def _check_binary(
command: str,
*,
check_id: str,
label: str,
required: bool,
args: tuple[str, ...] = ("--version",),
enabled: bool = True,
disabled_message: str = "Check is not applicable.",
missing_status: CheckStatus | None = None,
aliases: tuple[str, ...] = (),
) -> DiagnosticCheck:
if not enabled:
return _make_binary_check(
command,
check_id=check_id,
label=label,
required=required,
status="skip",
message=disabled_message,
)
resolved_name, command_path = _resolve_binary(command, *aliases)
not_found_status: CheckStatus = missing_status or ("fail" if required else "warn")
if not command_path:
return _make_binary_check(
resolved_name,
check_id=check_id,
label=label,
required=required,
status=not_found_status,
message="Missing." if not_found_status == "fail" else "Not installed.",
)
try:
return_code, stdout, stderr = await _run_command(resolved_name, *args)
except FileNotFoundError:
return _make_binary_check(
resolved_name,
check_id=check_id,
label=label,
required=required,
status=not_found_status,
message="Missing." if not_found_status == "fail" else "Not installed.",
path=command_path,
)
except TimeoutError:
return _make_binary_check(
resolved_name,
check_id=check_id,
label=label,
required=required,
status="fail" if required else "warn",
message="Timed out.",
path=command_path,
)
except Exception as exc:
return _make_binary_check(
resolved_name,
check_id=check_id,
label=label,
required=required,
status="fail" if required else "warn",
message=f"Error. {exc!s}",
path=command_path,
)
version: str = _bin_version(_first_line(stdout, stderr))
status: CheckStatus = "pass" if return_code == 0 else ("fail" if required else "warn")
message: str = "Available." if return_code == 0 else f"Exit code {return_code}."
return _make_binary_check(
resolved_name,
check_id=check_id,
label=label,
required=required,
status=status,
message=message,
path=command_path,
version=version,
)
def _make_runtime(config: Config) -> dict[str, Any]:
started = int(config.started) if config.started else 0
uptime_seconds = max(int(time.time()) - started, 0) if started else 0
return {
"app_version": config.app_version,
"app_branch": config.app_branch,
"app_commit_sha": config.app_commit_sha,
"app_build_date": config.app_build_date,
"started": started,
"uptime_seconds": uptime_seconds,
"platform": platform.system().lower(),
"platform_release": platform.release(),
"platform_machine": platform.machine(),
"python_version": platform.python_version(),
"python_minimum": f"{MIN_PYTHON[0]}.{MIN_PYTHON[1]}",
"is_native": config.is_native,
"console_enabled": config.console_enabled,
}
def _requirements() -> dict[str, Any]:
return {
"python": {
"current": platform.python_version(),
"required": f"{MIN_PYTHON[0]}.{MIN_PYTHON[1]}",
"supported": sys.version_info[:2] >= MIN_PYTHON,
"note": "",
}
}
def _summarize(checks: list[DiagnosticCheck]) -> tuple[ReportStatus, dict[str, int]]:
summary = {"total": len(checks), "pass": 0, "fail": 0, "warn": 0, "skip": 0, "required_failed": 0}
for check in checks:
summary[check.status] += 1
if check.required and check.status == "fail":
summary["required_failed"] += 1
if summary["required_failed"] > 0:
status: ReportStatus = "error"
elif summary["warn"] > 0 or any(check.status == "fail" for check in checks):
status = "degraded"
else:
status = "ok"
return status, summary
def _make_report(config: Config, checks: list[DiagnosticCheck]) -> dict[str, Any]:
status, summary = _summarize(checks)
return {
"status": status,
"generated_at": int(time.time()),
"summary": summary,
"runtime": _make_runtime(config),
"requirements": _requirements(),
"checks": [check.to_dict() for check in checks],
}
def diagnostics_error_report(config: Config) -> dict[str, Any]:
return _make_report(
config,
[
DiagnosticCheck(
id="diagnostics",
label="Diagnostics",
group="core",
required=True,
status="fail",
description="Diagnostics collector.",
message="Diagnostics collection failed.",
details={},
)
],
)
async def collect_diagnostics(config: Config) -> dict[str, Any]:
checks: list[DiagnosticCheck] = [
_check_ytdlp_package(),
_check_apprise_package(config),
_check_curl_transport(),
_check_pot_provider_package(),
_check_browser_endpoint(),
_check_browser_client(),
_check_flaresolverr(config),
]
checks.extend(_check_configured_pip_packages(config))
storage_checks, binary_checks = await asyncio.gather(
asyncio.gather(
_check_directory(Path(config.config_path), check_id="config_path", label="Config directory"),
_check_directory(Path(config.download_path), check_id="download_path", label="Download directory"),
_check_directory(Path(config.temp_path), check_id="temp_path", label="Temp directory"),
_check_db_file(Path(config.db_file)),
),
asyncio.gather(
_check_binary(
"ffmpeg",
check_id="ffmpeg",
label="ffmpeg",
required=True,
args=("-version",),
),
_check_binary(
"ffprobe",
check_id="ffprobe",
label="ffprobe",
required=True,
args=("-version",),
),
_check_binary(
"deno",
check_id="deno",
label="deno",
required=True,
),
_check_binary(
"yt-dlp",
check_id="yt_dlp_cli",
label="yt-dlp CLI",
required=True,
enabled=config.console_enabled,
disabled_message="Disabled.",
),
_check_binary(
"aria2c",
check_id="aria2c",
label="aria2c",
required=False,
missing_status="skip",
),
_check_binary(
"mkvpropedit",
check_id="mkvpropedit",
label="mkvpropedit",
required=False,
missing_status="skip",
),
_check_binary(
"mkvextract",
check_id="mkvextract",
label="mkvextract",
required=False,
missing_status="skip",
),
_check_binary(
"MP4Box",
check_id="mp4box",
label="MP4Box",
required=False,
missing_status="skip",
args=("-version",),
aliases=("mp4box",),
),
),
)
checks.extend(storage_checks)
checks.extend(binary_checks)
return _make_report(config, checks)

Some files were not shown because too many files have changed in this diff Show more