Merge pull request #621 from arabcoders/dev

fix: setting simple_mode cookie was crashing http layer

.github/workflows/build-pr.yml

@@ -15,7 +15,7 @@ on:
 
 env:
   BUN_VERSION: latest
-  NODE_VERSION: 20
+  NODE_VERSION: 22
   PYTHON_VERSION: "3.13"
 
 jobs:

.github/workflows/main.yml

@@ -29,7 +29,7 @@ env:
   DOCKERHUB_SLUG: arabcoders/ytptube
   GHCR_SLUG: ghcr.io/arabcoders/ytptube
   BUN_VERSION: latest
-  NODE_VERSION: 20
+  NODE_VERSION: 22
   PYTHON_VERSION: "3.13"
 
 jobs:

.github/workflows/native-build.yml

@@ -37,9 +37,9 @@ jobs:
       contents: write
 
     env:
-      PYTHON_VERSION: 3.11
+      PYTHON_VERSION: "3.13"
       BUN_VERSION: latest
-      NODE_VERSION: 20
+      NODE_VERSION: 22
       TAG_NAME: ${{ github.event.inputs.tag || github.ref_name }}
 
     steps:
@@ -63,6 +63,37 @@ jobs:
         shell: bash
         run: cp temp-spec/app.spec ./app.spec
 
+      - name: Update app/library/version.py
+        shell: bash
+        run: |
+          VERSION="${TAG_NAME}"
+          SHA=$(git rev-parse HEAD)
+          DATE=$(date -u +"%Y%m%d")
+          BRANCH=$(git branch --show-current)
+
+          if [ -z "${BRANCH}" ]; then
+            BRANCH="${TAG_NAME}"
+          fi
+
+          BRANCH=$(echo "${BRANCH}" | sed 's#/#-#g')
+
+          echo "APP_VERSION=${VERSION}" >> "$GITHUB_ENV"
+          echo "APP_SHA=${SHA}" >> "$GITHUB_ENV"
+          echo "APP_DATE=${DATE}" >> "$GITHUB_ENV"
+          echo "APP_BRANCH=${BRANCH}" >> "$GITHUB_ENV"
+
+          sed -i.bak \
+            -e "s/^APP_VERSION = \".*\"/APP_VERSION = \"${VERSION}\"/" \
+            -e "s/^APP_COMMIT_SHA = \".*\"/APP_COMMIT_SHA = \"${SHA}\"/" \
+            -e "s/^APP_BUILD_DATE = \".*\"/APP_BUILD_DATE = \"${DATE}\"/" \
+            -e "s/^APP_BRANCH = \".*\"/APP_BRANCH = \"${BRANCH}\"/" \
+            app/library/version.py
+
+          rm -f app/library/version.py.bak
+
+          echo "Updated version info:"
+          cat app/library/version.py
+
       - name: Cache Python venv
         id: cache-python
         uses: actions/cache@v4
@@ -82,7 +113,9 @@ jobs:
         shell: bash
         run: |
           pip install uv
-          uv venv --system-site-packages --relocatable
+          if [ ! -d .venv ]; then
+            uv venv --system-site-packages --relocatable
+          fi
           uv sync --link-mode=copy --active --extra installer
 
       - name: Cache bun installation
@@ -161,7 +194,7 @@ jobs:
           rm -rf "dist/${PKG_DIR}"
           mv "dist/YTPTube" "dist/${PKG_DIR}"
 
-          zip -r "release/${ZIP_NAME}" "dist/${PKG_DIR}"
+          zip -yr "release/${ZIP_NAME}" "dist/${PKG_DIR}"
 
       - name: Package artifact (Windows)
         if: startsWith(env.TAG_NAME, 'v') && runner.os == 'Windows'

.vscode/launch.json

@@ -2,7 +2,7 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Node: Nuxt3",
+      "name": "UI: Dev",
       "request": "launch",
       "runtimeArgs": [
         "run",
@@ -21,7 +21,7 @@
       "outputCapture": "std"
     },
     {
-      "name": "Python: main.py ",
+      "name": "Python: main.py",
       "type": "debugpy",
       "request": "launch",
       "program": "app/main.py",
@@ -38,7 +38,7 @@
       },
     },
     {
-      "name": "Node: Generate UI",
+      "name": "UI: Generate",
       "request": "launch",
       "runtimeArgs": [
         "run",

API.md

@@ -24,6 +24,8 @@ This document describes the available endpoints and their usage. All endpoints r
     - [DELETE /api/history](#delete-apihistory)
     - [POST /api/history/{id}](#post-apihistoryid)
     - [GET /api/history/{id}](#get-apihistoryid)
+    - [GET /api/history/{id}/thumbnail](#get-apihistoryidthumbnail)
+    - [POST /api/history/{id}/rename](#post-apihistoryidrename)
     - [GET /api/history](#get-apihistory)
     - [GET /api/history/live](#get-apihistorylive)
     - [POST /api/history/start](#post-apihistorystart)
@@ -55,7 +57,8 @@ This document describes the available endpoints and their usage. All endpoints r
     - [GET /api/player/m3u8/{mode}/{file:.\*}.m3u8](#get-apiplayerm3u8modefilem3u8)
     - [GET /api/player/segments/{segment}/{file:.\*}.ts](#get-apiplayersegmentssegmentfilets)
     - [GET /api/player/subtitle/{file:.\*}.vtt](#get-apiplayersubtitlefilevtt)
-    - [GET /api/thumbnail](#get-apithumbnail)
+    - [GET /api/player/subtitles/manifest/{file:.\*}](#get-apiplayersubtitlesmanifestfile)
+    - [GET /api/player/subtitles/{source\_format}/{file:.\*}](#get-apiplayersubtitlessource_formatfile)
     - [GET /api/file/ffprobe/{file:.\*}](#get-apifileffprobefile)
     - [GET /api/file/info/{file:.\*}](#get-apifileinfofile)
     - [GET /api/file/browser/{path:.\*}](#get-apifilebrowserpath)
@@ -85,6 +88,8 @@ This document describes the available endpoints and their usage. All endpoints r
     - [DELETE /api/conditions/{id}](#delete-apiconditionsid)
     - [GET /api/logs](#get-apilogs)
     - [GET /api/logs/stream](#get-apilogsstream)
+    - [GET /api/logs/level](#get-apilogslevel)
+    - [POST /api/logs/level/{level}](#post-apilogslevellevel)
     - [GET /api/notifications/](#get-apinotifications)
     - [GET /api/notifications/events/](#get-apinotificationsevents)
     - [POST /api/notifications/](#post-apinotifications)
@@ -96,7 +101,15 @@ This document describes the available endpoints and their usage. All endpoints r
     - [POST /api/notifications/test](#post-apinotificationstest)
     - [GET /api/yt-dlp/options](#get-apiyt-dlpoptions)
     - [GET /api/system/configuration](#get-apisystemconfiguration)
+    - [GET /api/system/folders](#get-apisystemfolders)
+    - [GET /api/system/diagnostics](#get-apisystemdiagnostics)
+    - [GET /api/system/limits](#get-apisystemlimits)
     - [POST /api/system/terminal](#post-apisystemterminal)
+    - [GET /api/system/terminal](#get-apisystemterminal)
+    - [GET /api/system/terminal/active](#get-apisystemterminalactive)
+    - [GET /api/system/terminal/{session\_id}](#get-apisystemterminalsession_id)
+    - [DELETE /api/system/terminal/{session\_id}](#delete-apisystemterminalsession_id)
+    - [GET /api/system/terminal/{session\_id}/stream](#get-apisystemterminalsession_idstream)
     - [POST /api/system/pause](#post-apisystempause)
     - [POST /api/system/resume](#post-apisystemresume)
     - [POST /api/system/shutdown](#post-apisystemshutdown)
@@ -128,6 +141,7 @@ This document describes the available endpoints and their usage. All endpoints r
         - [`item_updated`](#item_updated)
         - [`item_cancelled`](#item_cancelled)
         - [`item_deleted`](#item_deleted)
+        - [`item_bulk_deleted`](#item_bulk_deleted)
         - [`item_moved`](#item_moved)
         - [`item_status`](#item_status)
         - [`paused`](#paused)
@@ -342,7 +356,10 @@ or an error:
   "cookies": "...", // -- optional. If provided, it MUST BE in Netscape HTTP Cookie format.
   "template": "%(title)s.%(ext)s", // -- optional. The filename template to use for this item.
   "cli": "--write-subs --embed-subs", // -- optional. Additional command options for yt-dlp to apply to this item.
-  "auto_start": true // -- optional. Whether to auto-start the download after adding it. Defaults to true.
+  "auto_start": true, // -- optional. Whether to auto-start the download after adding it. Defaults to true.
+  "extras": {
+    "ignore_conditions": ["123", "My Condition"] // -- optional. Skip matching conditions by id string or exact name for this request only. Use ["*"] to ignore all conditions.
+  }
 }
 
 // Or multiple items (array of objects)
@@ -377,7 +394,7 @@ or an error:
 **Body Parameters**:
 - `type` (string, required): Type of items - `"queue"` or `"done"`
 - `ids` (array, optional): List of specific item IDs to delete. If provided, `status` filter is ignored
-- `status` (string, optional): Filter by status (e.g., `"finished"`, `"!finished"`). Required if `ids` not provided
+- `status` (string, optional): Filter by status (e.g., `"finished"`, `"!finished"`, `"finished,skip"`, `"!finished,!skip"`). Required if `ids` not provided
 - `remove_file` (boolean, optional): Whether to delete files from disk. Default: `true`.
 
 > [!NOTE]
@@ -412,6 +429,15 @@ or an error:
 }
 ```
 
+**Delete all completed and skipped items in one request:**
+```json
+{
+  "type": "done",
+  "status": "finished,skip",
+  "remove_file": false
+}
+```
+
 **Response**:
 ```json
 {
@@ -444,6 +470,7 @@ or an error:
 **Notes**:
 - When using filter mode, all matching items will be deleted.
 - Filter mode with `{ "status": "!finished" }` is useful for cleaning up failed/pending downloads.
+- `status` also accepts comma-separated include filters (`finished,skip`) and comma-separated exclude filters (`!finished,!skip`).
 - Filter mode returns a `deleted` count indicating how many items were removed.
 
 ---
@@ -510,6 +537,61 @@ or an error:
 
 ---
 
+### GET /api/history/{id}/thumbnail
+**Purpose**: Return thumbnail for a downloaded item.
+
+**Path Parameter**:
+- `id` = item ID.
+
+**Behavior**:
+- Returns an existing local sidecar or artwork image when available.
+- Otherwise generates a representative frame thumbnail with `ffmpeg` and caches it if enabled.
+
+**Response**:
+- `200 OK` with an image file response.
+- `404 Not Found` if the item, downloaded file, or local thumbnail is not available.
+- `400 Bad Request` if `id` is missing.
+
+**Notes**:
+- Audio-only files return `404` unless a local image sidecar already exists.
+
+---
+
+### POST /api/history/{id}/rename
+**Purpose**: Rename a downloaded history file and its sidecars.
+
+**Path Parameter**:
+- `id` = Unique history item ID.
+
+**Body**:
+```json
+{
+  "new_name": "renamed_video.mp4"
+}
+```
+
+**Response**:
+```json
+{
+  "_id": "<uuid>",
+  "title": "Video Title",
+  "url": "https://youtube.com/watch?v=...",
+  ....
+}
+```
+
+**Error Responses**:
+- `400 Bad Request` if `id` or `new_name` is missing, or the item has no downloaded file.
+- `404 Not Found` if the item does not exist.
+- `409 Conflict` if the rename destination already exists.
+- `500 Internal Server Error` if the filesystem rename fails unexpectedly.
+
+**Notes**:
+- Uses the same file rename behavior as the file browser actions.
+- Renames matching sidecar files together with the main media file.
+
+---
+
 ### GET /api/history
 **Purpose**: Returns the download queue and/or download history with optional pagination support.
 
@@ -618,10 +700,16 @@ GET /api/history?type=queue&status=pending&order=ASC
 
 This endpoint returns the current state of active downloads from memory.
 
+**Query Parameters**:
+- `limit` (optional): Override the configured queue display limit for this request. `0` means unlimited.
+
 **Response**:
 ```json
 {
     "history_count": 0, // total number of completed items in history
+    "queue_count": 250, // total number of queued items
+    "queue_loaded": 100, // number of queued items included in this response
+    "queue_limit": 100, // effective display limit, 0 means unlimited
     "queue":{
       "id": "abc123",
       "url": "https://example.com/video",
@@ -1531,14 +1619,48 @@ Binary TS data (`Content-Type: video/mpegts`).
 
 ---
 
-### GET /api/thumbnail
-**Purpose**: Proxy/fetch a remote thumbnail image.  
+### GET /api/player/subtitles/manifest/{file:.*}
+**Purpose**: Returns subtitle track metadata for a local media file.  
 
-**Query Parameter**:
-- `?url=<remote-thumbnail-url>`
+**Path Parameter**:
+- `file` = Relative path of the media file within the `download_path`.
 
-**Response**:  
-Binary image data with the appropriate `Content-Type`.
+**Response**:
+```json
+{
+  "subtitles": [
+    {
+      "lang": "en",
+      "name": "VTT (0) - en",
+      "source_format": "vtt",
+      "delivery_format": "vtt",
+      "renderer": "native",
+      "url": "/api/player/subtitles/vtt/path/to/video.en.vtt"
+    },
+    {
+      "lang": "en",
+      "name": "ASS (1) - en",
+      "source_format": "ass",
+      "delivery_format": "ass",
+      "renderer": "assjs",
+      "url": "/api/player/subtitles/ass/path/to/video.en.ass"
+    }
+  ]
+}
+```
+
+---
+
+### GET /api/player/subtitles/{source_format}/{file:.*}
+**Purpose**: Delivers a subtitle file using its preferred playback format.  
+
+**Path Parameters**:
+- `source_format` = `vtt`, `srt`, or `ass`.
+- `file` = Relative path of the subtitle file.
+
+**Response**:
+- `text/vtt; charset=UTF-8` for `vtt` and `srt` sources.
+- `text/x-ssa; charset=UTF-8` for `ass` sources.
 
 ---
 
@@ -1574,21 +1696,16 @@ Binary image data with the appropriate `Content-Type`.
   },
   "mimetype": "video/mp4",
   "sidecar": {
-    "subtitles": [
+    "subtitle": [
       {
-        "file": "filename.xxx.vtt",
+        "file": "filename.xxx.ass",
         "lang": "xxx",
-        "name": "VTT 0 - XXX|end",
+        "name": "ASS (0) - xxx"
       },
       ...
-      }
     ],
-    "video": [],
-    "audio": [],
     "image": [],
-    "text": [],
-    "metadata": [],
-    ...
+    "text": []
   }
 }
 ```
@@ -1771,6 +1888,9 @@ Binary image data with appropriate headers
 **Query Parameters**:
 - `page` (optional): Page number (1-indexed). Default: `1`.
 - `per_page` (optional): Items per page. Default: `config.default_pagination`.
+- `sort` (optional): Comma-separated sort fields. Accepted values: `id`, `name`, `priority`, `default`, `created_at`, `updated_at`. Default: `priority,name`.
+- `order` (optional): Comma-separated sort directions matching `sort`, or a single direction applied to every requested sort field. Accepted values: `asc`, `desc`. Default: `desc,asc`.
+- `exclude_defaults` (optional): When `true`, excludes system presets from the results. Default: `false`.
 
 **Response**:
 ```json
@@ -1801,6 +1921,9 @@ Binary image data with appropriate headers
 **Notes**:
 - `default: true` indicates this is a system default preset (cannot be modified or deleted)
 
+**Error Responses**:
+- `400 Bad Request` - Invalid data was provided.
+
 ---
 
 ### GET /api/presets/{id}
@@ -2200,12 +2323,46 @@ Binary image data with appropriate headers
 {
   "logs": [
     {
-      "timestamp": "2023-01-01T12:00:00Z",
-      "level": "INFO",
-      "message": "...",
-      ...
-    },
-    ...
+      "id": "<uuid>",
+      "datetime": "2026-05-18T12:00:00.000+00:00",
+      "level": "error",
+      "levelno": 40,
+      "logger": "ytptube",
+      "message": "Failed to download 'Example Video'.",
+      "exception": {
+        "type": "ValueError",
+        "message": "bad",
+        "file": "/app/library/downloads/queue_manager.py",
+        "line": 123,
+        "stack": [
+          {
+            "path": "/app/library/downloads/queue_manager.py",
+            "file": "queue_manager.py",
+            "module": "queue_manager",
+            "function": "start",
+            "line": 123
+          }
+        ]
+      },
+      "source": {
+        "path": "/app/library/downloads/queue_manager.py",
+        "file": "queue_manager.py",
+        "module": "queue_manager",
+        "function": "start",
+        "line": 123
+      },
+      "fields": {
+        "download": {
+          "download_id": "abc123",
+          "media_id": "video-id",
+          "title": "Example Video",
+          "url": "https://example.test/video",
+          "preset": "default",
+          "status": "error",
+          "has_cookies": false
+        }
+      }
+    }
   ],
   "offset": 0,
   "limit": 100,
@@ -2227,9 +2384,45 @@ Binary image data with appropriate headers
 **Event Payload**:
 ```json
 {
-  "id": "<sha256>",
-  "line": "<log line>",
-  "datetime": "2024-01-01T12:00:00.000000+00:00"
+  "id": "<uuid>",
+  "datetime": "2026-05-18T12:00:00.000+00:00",
+  "level": "error",
+  "levelno": 40,
+  "logger": "ytptube",
+  "message": "Failed to download 'Example Video'.",
+  "exception": {
+    "type": "ValueError",
+    "message": "bad",
+    "file": "/app/library/downloads/queue_manager.py",
+    "line": 123,
+    "stack": [
+      {
+        "path": "/app/library/downloads/queue_manager.py",
+        "file": "queue_manager.py",
+        "module": "queue_manager",
+        "function": "start",
+        "line": 123
+      }
+    ]
+  },
+  "source": {
+    "path": "/app/library/downloads/queue_manager.py",
+    "file": "queue_manager.py",
+    "module": "queue_manager",
+    "function": "start",
+    "line": 123
+  },
+  "fields": {
+    "download": {
+      "download_id": "abc123",
+      "media_id": "video-id",
+      "title": "Example Video",
+      "url": "https://example.test/video",
+      "preset": "default",
+      "status": "error",
+      "has_cookies": false
+    }
+  }
 }
 ```
 
@@ -2237,6 +2430,31 @@ Binary image data with appropriate headers
 
 ---
 
+### GET /api/logs/level
+**Purpose**: Read the active runtime log level.
+
+**Response**:
+```json
+{
+  "conf": "info",
+  "active": "info",
+  "levels": ["debug", "info", "warning", "error"]
+}
+```
+
+---
+
+### POST /api/logs/level/{level}
+**Purpose**: Change the active runtime log level.
+
+**Path Parameter**:
+- `level`: One of `debug`, `info`, `warning`, `error`.
+
+**Response**:
+- `204 No Content` on success.
+
+---
+
 ### GET /api/notifications/
 **Purpose**: Retrieve notification targets with pagination.
 
@@ -2417,60 +2635,139 @@ or an error:
 ---
 
 ### GET /api/system/configuration
-**Purpose**: Retrieve comprehensive system configuration including app settings, presets, download fields, queue status, and folder structure.
+**Purpose**: Retrieve system configuration.
 
 **Response**:
 ```json
 {
-  "app": {
-    "version": "...",
-    "download_path": "/path/to/downloads",
-    "base_path": "/",
-    ...
-  },
-  "presets": [
-    {
-      "id": 1,
-      "name": "default",
-      "description": "...",
-      ...
-    }
-  ],
-  "dl_fields": [
-    {
-      "id": 1,
-      "name": "Title",
-      "field": "title",
-      "kind": "text",
-      ...
-    }
-  ],
+  "app": {...},
+  "presets": [...],
+  "dl_fields": [...],
   "paused": false,
-  "folders": [
-    {"name": "folder1", "path": "folder1"},
-    {"name": "folder2", "path": "folder2"}
-  ],
-  "history_count": 150,
-  "queue": [
-    {
-      "id": "abc123",
-      "url": "https://example.com/video",
-      "status": "pending",
-      ...
-    }
-  ]
+  "history_count": 150
+}
+```
+
+---
+
+### GET /api/system/folders
+**Purpose**: List child directories for a given relative path within the download directory.
+
+**Query Parameters**:
+- `path=<relative-path>` (optional, default: root) - Relative path within the download directory.
+
+**Response**:
+```json
+{
+  "path": "videos",
+  "folders": ["archive", "shorts", "2024"]
 }
 ```
 
 **Notes**:
-- This endpoint combines multiple data sources into a single response for efficient initialization
-- The `folders` array includes available download folders up to the configured depth limit
-- The `queue` array contains active download items
+- Results are cached server-side for a short time.
+- Non-existent paths return an empty folder list.
+
+---
+
+### GET /api/system/diagnostics
+**Purpose**: View system information.
+
+**Response**:
+```json
+{
+  "status": "error",
+  "generated_at": 1713000000,
+  "summary": {
+    "total": 10,
+    "pass": 5,
+    "fail": 2,
+    "warn": 1,
+    "skip": 2,
+    "required_failed": 2
+  },
+  "runtime": {
+    "app_version": "1.0.0",
+    "app_branch": "main",
+    "app_commit_sha": "abcdef12",
+    "app_build_date": "20260526",
+    "started": 1712999900,
+    "uptime_seconds": 100,
+    "platform": "linux",
+    "platform_release": "6.8.0",
+    "platform_machine": "x86_64",
+    "python_version": "3.13.1",
+    "python_minimum": "3.13",
+    "is_native": false,
+    "console_enabled": false
+  },
+  "requirements": {
+    "python": {
+      "current": "3.13.1",
+      "required": "3.13",
+      "supported": true,
+      "note": ""
+    }
+  },
+  "checks": []
+}
+```
+
+**Notes**:
+- Unexpected collection errors are returned as an `error`.
+
+---
+
+### GET /api/system/limits
+**Purpose**: Get the system limits.
+
+**Response**:
+```json
+{
+  "downloads": {
+    "paused": false,
+    "global": {
+      "limit": 20,
+      "active": 3,
+      "available": 17,
+      "live_active": 1,
+      "queued": 8
+    },
+    "per_extractor": {
+      "default_limit": 2,
+      "items": [
+        {
+          "name": "youtube",
+          "limit": 3,
+          "source": "env_override",
+          "active": 2,
+          "queued": 4,
+          "available": 1
+        }
+      ]
+    }
+  },
+  "extraction": {
+    "concurrency": 4,
+    "timeout_seconds": 70,
+    "info_cache_ttl_seconds": 10800
+  },
+  "live": {
+    "prevent_premiere": true,
+    "premiere_buffer_minutes": 5
+  }
+}
+```
+
+**Notes**:
+- `downloads.global` counts only non-live downloads against the worker limit.
+- `downloads.global.live_active` is reported separately because live downloads bypass the global and per-extractor worker limits.
+- `downloads.per_extractor.items[*].source` is `default` unless an override was provided through `YTP_MAX_WORKERS_FOR_<EXTRACTOR>`.
 
 ---
 
 ### POST /api/system/terminal
-**Purpose**: Stream yt-dlp CLI output via Server-Sent Events (SSE). Requires `YTP_CONSOLE_ENABLED=true`.
+**Purpose**: Start a yt-dlp terminal session. Requires `YTP_CONSOLE_ENABLED=true`.
 
 **Body**:
 ```json
@@ -2479,9 +2776,147 @@ or an error:
 }
 ```
 
+**Response**:
+```json
+{
+  "session_id": "3a8c5f7e2d3b4a8f9c0d1e2f3a4b5c6d",
+  "command": "--help",
+  "status": "starting",
+  "created_at": 1713000000.0,
+  "started_at": 1713000000.0,
+  "finished_at": null,
+  "expires_at": null,
+  "exit_code": null,
+  "last_sequence": 0
+}
+```
+
+**Notes**:
+- Starts the command in an application-owned background task.
+- Only one active terminal session is allowed at a time.
+- The command continues running if the frontend disconnects or reloads.
+
+- `403 Forbidden` if console is disabled.
+- `400 Bad Request` if the request body is invalid.
+- `409 Conflict` if another terminal session is already active.
+
+---
+
+### GET /api/system/terminal
+**Purpose**: List recent terminal sessions.
+
+**Response**:
+```json
+{
+  "items": [
+    {
+      "session_id": "3a8c5f7e2d3b4a8f9c0d1e2f3a4b5c6d",
+      "command": "--help",
+      "status": "completed",
+      "created_at": 1713000000.0,
+      "started_at": 1713000000.0,
+      "finished_at": 1713000004.0,
+      "expires_at": 1713086404.0,
+      "available_until": 1713086404.0,
+      "exit_code": 0,
+      "last_sequence": 15
+    }
+  ]
+}
+```
+
+**Notes**:
+- Expired sessions are removed automatically later.
+
+- `403 Forbidden` if console is disabled.
+
+---
+
+### GET /api/system/terminal/active
+**Purpose**: Return the currently active terminal session metadata, or `null` if no session is active.
+
+**Response**:
+```json
+null
+```
+or
+```json
+{
+  "session_id": "3a8c5f7e2d3b4a8f9c0d1e2f3a4b5c6d",
+  "command": "--help",
+  "status": "running",
+  "created_at": 1713000000.0,
+  "started_at": 1713000000.0,
+  "finished_at": null,
+  "expires_at": null,
+  "exit_code": null,
+  "last_sequence": 12
+}
+```
+
+- `403 Forbidden` if console is disabled.
+
+---
+
+### GET /api/system/terminal/{session_id}
+**Purpose**: Return metadata for a specific terminal session.
+
+**Response**:
+```json
+{
+  "session_id": "3a8c5f7e2d3b4a8f9c0d1e2f3a4b5c6d",
+  "command": "--help",
+  "status": "completed",
+  "created_at": 1713000000.0,
+  "started_at": 1713000000.0,
+  "finished_at": 1713000004.0,
+  "expires_at": 1713086404.0,
+  "exit_code": 0,
+  "last_sequence": 15
+}
+```
+
+- `403 Forbidden` if console is disabled.
+- `404 Not Found` if the session does not exist or has already expired.
+
+---
+
+### DELETE /api/system/terminal/{session_id}
+**Purpose**: Request cancellation for the active terminal session.
+
+**Response**:
+```json
+{
+  "message": "Terminal session cancellation requested.",
+  "session_id": "3a8c5f7e2d3b4a8f9c0d1e2f3a4b5c6d"
+}
+```
+
+**Notes**:
+- This only applies to the currently active session.
+- Cancelled sessions are marked as `interrupted`.
+
+- `403 Forbidden` if console is disabled.
+- `404 Not Found` if the session does not exist or has already expired.
+- `409 Conflict` if the session exists but is no longer active.
+
+---
+
+### GET /api/system/terminal/{session_id}/stream
+**Purpose**: Replay a terminal session transcript over SSE and tail live output when the session is still running.
+
+**Query Parameters**:
+- `since` (optional): Resume after the provided integer event id.
+
+**Headers**:
+- `Last-Event-ID` (optional): Resume after the provided integer event id.
+
+If both `since` and `Last-Event-ID` are present, the larger value is used.
+
 **Response**:
 - `Content-Type: text/event-stream`
-- Emits `output` events for stdout/stderr and a final `close` event when the process exits.
+- Replays transcript events with monotonic integer SSE `id` values.
+- Emits `output` events for stdout/stderr and a final `close` event when available.
 
 **Event Payloads**:
 ```json
@@ -2492,7 +2927,8 @@ or an error:
 ```
 
 - `403 Forbidden` if console is disabled.
-- `400 Bad Request` if the request body is invalid.
+- `400 Bad Request` if the replay cursor is invalid.
+- `404 Not Found` if the session does not exist or has already expired.
 
 ---
 
@@ -3003,6 +3439,24 @@ Emitted when a download item is deleted from the queue or history.
 
 ---
 
+##### `item_bulk_deleted`
+
+Emitted when multiple items are cleared in bulk operation.
+
+**Event**:
+```json
+{
+  "event": "item_bulk_deleted",
+  "data": {
+    "count": 10001,
+    "status": "finished,skip",
+    "ids": ["id1", "id2", "..."]  // optional, included when clear is driven by explicit ids
+  }
+}
+```
+
+---
+
 ##### `item_moved`
 
 Emitted when a download item is moved between queue and history.
@@ -3014,7 +3468,7 @@ Emitted when a download item is moved between queue and history.
   "data": {
     "id": "abc123",
     "from": "queue",
-    "to": "done"
+    "to": "history"
   }
 }
 ```

FAQ.md

@@ -3,6 +3,9 @@
 Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line, 
 or the `environment:` section in `compose.yaml` file.
 
+<details>
+<summary>Click to expand</summary>
+
 | Environment Variable            | Description                                                         | Default               |
 | ------------------------------- | ------------------------------------------------------------------- | --------------------- |
 | TZ                              | The timezone to use for the application                             | `(not_set)`           |
@@ -30,7 +33,6 @@ or the `environment:` section in `compose.yaml` file.
 | YTP_DEBUG                       | Whether to turn on debug mode                                       | `false`               |
 | YTP_DEBUGPY_PORT                | The port to use for the debugpy debugger                            | `5678`                |
 | YTP_EXTRACT_INFO_TIMEOUT        | The timeout for extracting video information                        | `70`                  |
-| YTP_UI_UPDATE_TITLE             | Whether to update the title of the page with the current stats      | `true`                |
 | YTP_PIP_PACKAGES                | A space separated list of pip packages to install                   | `(not_set)`           |
 | YTP_PIP_IGNORE_UPDATES          | Do not update the custom pip packages                               | `false`               |
 | YTP_PICTURES_BACKENDS           | A comma separated list of pictures urls to use                      | `(not_set)`           |
@@ -38,16 +40,17 @@ or the `environment:` section in `compose.yaml` file.
 | YTP_YTDLP_AUTO_UPDATE           | Whether to enable the auto update for yt-dlp                        | `true`                |
 | YTP_YTDLP_DEBUG                 | Whether to turn debug logging for the internal `yt-dlp` package     | `false`               |
 | YTP_YTDLP_VERSION               | The version of yt-dlp to use. Defaults to latest version            | `(not_set)`           |
+| YTP_BROWSER_URL                 | Remote browser endpoint for the bundled `browser` extractor         | `(not_set)`           |
 | YTP_FLARESOLVERR_URL            | FlareSolverr endpoint URL.                                          | `(not_set)`           |
 | YTP_FLARESOLVERR_MAX_TIMEOUT    | Max FlareSolverr challenge timeout in seconds                       | `120`                 |
 | YTP_FLARESOLVERR_CLIENT_TIMEOUT | HTTP client timeout (seconds) when calling FlareSolverr             | `120`                 |
 | YTP_FLARESOLVERR_CACHE_TTL      | The cache TTL (in seconds) for FlareSolverr solutions               | `600`                 |
 | YTP_BASE_PATH                   | Set this if you are serving YTPTube from sub-folder                 | `/`                   |
 | YTP_PREVENT_LIVE_PREMIERE       | Prevents the initial youtube premiere stream from being downloaded  | `false`               |
+| YTP_QUEUE_DISPLAY_LIMIT         | Max queued downloads returned to the UI. `0` = unlimited            | `100`                 |
 | YTP_LIVE_PREMIERE_BUFFER        | buffer time in minutes to add to video duration                     | `5`                   |
 | YTP_TASKS_HANDLER_TIMER         | The cron expression for the tasks handler timer                     | `15 */1 * * *`        |
 | YTP_TEMP_DISABLED               | Disable temp files handling.                                        | `false`               |
-| YTP_DOWNLOAD_PATH_DEPTH         | How many subdirectories to show in auto complete.                   | `1`                   |
 | YTP_ALLOW_INTERNAL_URLS         | Allow requests to internal URLs                                     | `false`               |
 | YTP_SIMPLE_MODE                 | Switch default interface to Simple mode.                            | `false`               |
 | YTP_STATIC_UI_PATH              | Path to custom static UI files.                                     | `(not_set)`           |
@@ -57,19 +60,23 @@ or the `environment:` section in `compose.yaml` file.
 | YTP_IGNORE_ARCHIVED_ITEMS       | Don't report archived items in the download history.                | `false`               |
 | YTP_CHECK_FOR_UPDATES           | Whether to check for application updates.                           | `true`                |
 | YTP_EXTRACT_INFO_CONCURRENCY    | The number of concurrent extract info operations.                   | `4`                   |
+| YTP_THUMB_CONCURRENCY           | The number of concurrent ffmpeg thumbnail generations allowed.      | `2`                   |
+| YTP_THUMB_GENERATE              | Enable ffmpeg thumbnail generation when no local thumbnail exists.  | `true`                |
+| YTP_THUMB_SIDECAR               | Save generated thumbnails next to media instead of temp cache.      | `false`               |
+| YTP_DISABLE_EXEC                | Strip some dangerous yt-dlp options.                                | `false`               |
+</details>
 
 > [!NOTE]
-> To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`.
-> The extractor name must be in uppercase, to know the extractor name, check the log for the specific extractor used for the download.
-> The limit should not exceed the `YTP_MAX_WORKERS` value as it will be ignored.
-
-> [!IMPORTANT]
-> The env variable `YTP_SIMPLE_MODE` only control what being displayed for first time visitor, the users can still switch between the two modes via the WebUI settings page.
-
-## Notes about YTP_AUTO_CLEAR_HISTORY_DAYS
-
-- `0` days means no automatic clearing of the download history. lowest value that will trigger the clearing is `1` day.
-- This setting will **NOT** delete the downloaded files, it will only clear the history from the database.
+> To raise the worker limit for a specific extractor, set an env variable using this format: `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`
+> The extractor name must be uppercase. You can find the extractor name in the download logs. This value cannot be 
+> higher than `YTP_MAX_WORKERS`; higher values are ignored.
+>
+> `YTP_SIMPLE_MODE=true` only applies when the browser has no saved layout choice yet. Users can still choose a layout in 
+> WebUI Settings. `/?simple=1` forces and saves Simple for that browser.
+> 
+> `YTP_AUTO_CLEAR_HISTORY_DAYS`  `0` days means no automatic clearing of the download history. lowest value that will 
+> trigger the clearing is `1` day. This setting will **NOT** delete the downloaded files, it will only clear the 
+> history from the database.
 
 # Browser extensions & bookmarklets
 
@@ -116,6 +123,35 @@ As this is a simple basic authentication, if your browser doesn't show the promp
 
 `http://username:password@your_ytptube_url:port`
 
+# Security recommendations
+
+YTPTube is designed for LAN and home-lab use behind a firewall or reverse proxy. The web interface and API are
+unauthenticated by default because in a trusted network, auth adds friction without meaningful benefit. However,
+if you expose YTPTube to the internet directly or via port forwarding **YOU MUST enable authentication**.
+
+### Without auth, anyone who can reach the API can:
+
+- Download arbitrary content through your IP and server.
+- Delete or modify your downloaded files and database.
+- Run arbitrary `yt-dlp` options, including `--exec`, which executes shell commands inside the container.
+
+This is not a vulnerability, it's the intended design. The `cli` field passes options directly to `yt-dlp`,
+a tool that by design can execute commands. Auth is the mechanism that controls who gets to use that power.
+
+**If you expose YTPTube to untrusted networks**, do one of the following:
+
+1. **Enable authentication** — set both `YTP_AUTH_USERNAME` and `YTP_AUTH_PASSWORD`.
+2. **Put it behind a reverse proxy** with its own authentication layer (see [Run behind reverse proxy](#run-behind-reverse-proxy)).
+3. **Keep it on a private network** with no public exposure.
+
+YTPTube already gates other powerful features behind explicit opt-in: the built-in terminal, file browser actions and internal
+URL requests for example. The `cli` field is no different, its power is by design, and access control is your responsibility.
+
+> [!NOTE]
+> If you choose to run without authentication but still want to reduce at least some impact, you can set
+> `YTP_DISABLE_EXEC=true`. This strips some dangerous options at run time. However, understand that this is not a 
+> substitute for auth an unauthenticated API is still fully open for all other operations.
+
 # I cant download anything
 
 If you are receiving errors like:
@@ -168,6 +204,28 @@ YTP_YTDLP_VERSION=2025.07.21 or master or nightly
 
 Then restart the container to apply the changes.
 
+# Custom output template placeholders
+
+YTPTube supports custom `ytp_*` placeholders in `yt-dlp` output template via the following syntax `%(ytp_*:<args>)s`.
+
+## Currently available extra placeholders are:
+
+- `ytp_random`: random mixed letters and digits,
+  - `N` A number is required to specify the length of the random string, for example `%(ytp_random:8)s` will generate a random string of 8 characters. 
+  - if the args followed by `:s` it will generate random letters only, if followed by `:d` it will generate random digits only.
+
+## Examples of the custom placeholders in action:
+
+- Template: `%(title)s [%(ytp_random:8)s].%(ext)s`
+  - Example result: `My Video [A7k2Pq9Z].mp4`
+- Template: `%(uploader)s/%(ytp_random:6:d)s - %(title)s.%(ext)s`
+  - Example result: `MyChannel/483920 - My Video.mp4`
+- Template: `%(playlist)s/%(ytp_random:10:s)s/%(title)s.%(ext)s`
+  - Example result: `Favorites/QwErTyUiOp/My Video.mp4`
+
+> [!NOTE]
+> `%(ytp_` placeholders are a YTPTube extension and not avaliable via console or directly via yt-dlp.
+
 # How can I monitor sites without RSS feeds?
 
 YTPTube includes a **generic task handler** that turns JSON definitions into site-specific scrapers. You can use it
@@ -593,3 +651,108 @@ services:
 ```
 
 For more information please visit [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) project.
+
+# How to use the browser extractor?
+
+YTPTube ships with a bundled `browser` extractor plugin for `yt-dlp`. It opens the target page in a remote browser,
+waits for the page network activity, then tries to extract media urls from the captured requests and media elements.
+
+This is mainly useful for sites where the regular extractor or generic page parsing does not expose the final media url,
+but a real browser session does.
+
+## Environment variables
+
+You can define the remote browser endpoint globally with the following environment variable.
+
+### Selenium example
+
+```env
+YTP_BROWSER_URL=selenium+http://selenium:4444/wd/hub
+```
+
+### Playwright example
+
+For a native Playwright server, point to the Playwright websocket endpoint:
+
+```env
+YTP_BROWSER_URL=playwright+ws://playwright:3000/
+```
+
+### Playwright CDP example
+
+To connect Playwright over the Chrome DevTools Protocol, use either the shorthand `playwright+cdp://` form or an explicit transport form:
+
+```env
+YTP_BROWSER_URL=playwright+cdp://chrome:9222/
+# same as playwright+cdp+http://chrome:9222/
+```
+
+- `YTP_BROWSER_URL` is the full backend selector and endpoint in one value.
+- Supported forms are:
+  - `selenium+http://...`
+  - `selenium+https://...`
+  - `playwright+ws://...`
+  - `playwright+wss://...`
+  - `playwright+cdp://...` which is treated as `playwright+cdp+http://...`
+  - `playwright+cdp+http://...`
+  - `playwright+cdp+https://...`
+  - `playwright+cdp+ws://...`
+  - `playwright+cdp+wss://...`
+
+## yt-dlp usage
+
+If you want to set the browser extractor options directly on the yt-dlp side, you can also use `--extractor-args` with `generic:url=...`:
+
+### Selenium example
+
+```bash
+--use-extractors "generic" --extractor-args "generic:url=selenium+http://selenium:4444/wd/hub"
+```
+
+### Playwright example
+
+```bash
+--use-extractors "generic" --extractor-args "generic:url=playwright+ws://playwright:3000/"
+```
+
+### Playwright CDP example
+
+```bash
+--use-extractors "generic" --extractor-args "generic:url=playwright+cdp://chrome:9222/"
+```
+
+The explicit `--extractor-args` value takes priority over `YTP_BROWSER_URL`.
+
+## Example compose setup
+
+```yaml
+services:
+  ytptube:
+    user: "${UID:-1000}:${UID:-1000}"
+    image: ghcr.io/arabcoders/ytptube:latest
+    container_name: ytptube
+    restart: unless-stopped
+    environment:
+      - YTP_BROWSER_URL=selenium+http://selenium:4444/wd/hub # or playwright+ws://playwright:3000/ or playwright+cdp://chrome:9222/
+    ports:
+      - "8081:8081"
+    volumes:
+      - ./config:/config:rw
+      - ./downloads:/downloads:rw
+
+  selenium:
+    image: selenium/standalone-chrome:latest
+    container_name: selenium
+    restart: unless-stopped
+    shm_size: 2gb
+    
+  playwright:
+    image: mcr.microsoft.com/playwright:v1.59.0-noble
+    container_name: playwright
+    restart: unless-stopped
+    command: /bin/sh -c "npx -y playwright@1.59.0 run-server --port 3000 --host 0.0.0.0"
+```
+
+> [!NOTE]
+> The browser extractor is slower than the normal extractor flow and should only be used when a site actually needs a real browser session.
+> playwright require same version for both the server and the client, so make sure to use the same version in the container and in your local environment if you want to test it locally.

README.md

@@ -1,8 +1,9 @@
 # YTPTube
 
-![Build Status](https://github.com/ArabCoders/ytptube/actions/workflows/main.yml/badge.svg)
+![Build Status](https://github.com/arabcoders/ytptube/actions/workflows/main.yml/badge.svg)
 ![MIT License](https://img.shields.io/github/license/arabcoders/ytptube.svg)
-![Docker pull](https://ghcr-badge.elias.eu.org/shield/arabcoders/ytptube/ytptube)
+![Docker Pull](https://img.shields.io/docker/pulls/arabcoders/ytptube.svg)
+![gchr Pull](https://ghcr-badge.elias.eu.org/shield/arabcoders/ytptube/ytptube)
 
 **YTPTube** is a web-based GUI for [yt-dlp](https://github.com/yt-dlp/yt-dlp), designed to make downloading videos from 
 video platforms easier and user-friendly. It supports downloading playlists, channels, live streams and 
@@ -37,25 +38,34 @@ Example of the Simple mode interface.
 * Conditions feature to apply custom options based on `yt-dlp` returned info.
 * Custom browser extensions, bookmarklets and iOS shortcuts to send links to YTPTube instance.
 * A bundled executable version for Windows, macOS and Linux. `MacOS version is untested`.
+* Use playwright or selenium for extractors that require a browser. see [related FAQ](FAQ.md#how-to-use-the-browser-extractor).
 
 Please read the [FAQ](FAQ.md) for more information.
 
 # Installation
 
+> [!IMPORTANT]
+> By default YTPTube runs without authentication. If you expose it to the internet, **enable auth**. See [security recommendations](FAQ.md#security-recommendations).
+
 ## Run using docker command
 
 ```bash
-mkdir -p ./{config,downloads/files,downloads/tmp} && docker run -itd --rm --user "${UID}:${UID}" --name ytptube \
+mkdir -p ./{config,downloads/{files,tmp}} && docker run -itd --rm --user "${UID}:${UID}" --name ytptube \
 -e YTP_TEMP_PATH=/downloads/tmp -e YTP_DOWNLOAD_PATH=/downloads/files \
 -p 8081:8081 -v ./config:/config:rw -v ./downloads:/downloads:rw \
 ghcr.io/arabcoders/ytptube:latest
 ```
 
-Then you can access the WebUI at `http://localhost:8081`.
+## Run using podman
 
-> [!NOTE]
-> If you are using `podman` instead of `docker`, you can use the same command, but you need to change the user to `0:0`
-> it will appears to be running as root, but it will run as the user who started the container.
+```bash
+mkdir -p ./{config,downloads/{files,tmp}} && podman run -itd --rm --userns=keep-id --name ytptube \
+-e YTP_TEMP_PATH=/downloads/tmp -e YTP_DOWNLOAD_PATH=/downloads/files \
+-p 8081:8081 -v ./config:/config:rw -v ./downloads:/downloads:rw \
+arabcoders/ytptube:latest
+```
+
+Then you can access the WebUI at `http://localhost:8081`.
 
 ## Using compose file
 
@@ -65,6 +75,8 @@ The following is an example of a `compose.yaml` file that can be used to run YTP
 services:
   ytptube:
     user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id.
+    # comment out the above line and uncomment the below line if you are using podman-compose.
+    #userns_mode: keep-id
     image: ghcr.io/arabcoders/ytptube:latest
     container_name: ytptube
     restart: unless-stopped
@@ -79,18 +91,14 @@ services:
 ```
 
 > [!IMPORTANT]
-> Make sure to change the `user` line to match your user id and group id.
+> Make sure to change the `user` line to match your user id and group id in docker setups, or use `userns_mode: keep-id` in podman setups.
 
 ```bash
-mkdir -p ./{config,downloads/files,downloads/tmp} && docker compose -f compose.yaml up -d
+mkdir -p ./{config,downloads/{files,tmp}} && docker compose -f compose.yaml up -d
 ```
 
 Then you can access the WebUI at `http://localhost:8081`.
 
-> [!NOTE]
-> you can use podman-compose instead of docker-compose, as it supports the same syntax. However, you should change the 
-> user to `0:0` it will appears to be running as root, but it will run as the user who started the container.
-
 ## Unraid
 
 For `Unraid` users You can install the `Community Applications` plugin, and search for **ytptube** it comes 
@@ -104,15 +112,14 @@ For simple API documentation, you can refer to the [API documentation](API.md).
 
 This project is not affiliated with yt-dlp or any other service.
 
-It’s a personal project designed to make downloading videos from the internet more convenient. It’s not intended for 
-piracy or any unlawful use.
+This is a personal project designed to make downloading videos from the internet more convenient for me. It is not 
+intended for piracy or any unlawful use. This project was built primarily for my own use and preferences.
 
-This project was built primarily for my own needs and preferences. The UI might not be the most polished or visually 
-refined, but I’m happy with it as it is. You can, however, create and load your own UI for complete customization. I 
-plan to refactor the UI/UX in the future using [Nuxt/ui](https://ui.nuxt.com/).
+AI-assisted tools are used in this project. If you are uncomfortable with this, you should not use this project.
 
-Contributions are welcome, but I may decline changes that don’t align with my vision for the project. Unsolicited pull 
-requests may be ignored. For suggestions or feature requests, please open a discussion or join the Discord server.
+Contributions are welcome, but I may decline changes that do not interest me or do not align with my vision for this 
+project. Unsolicited pull requests will be closed. For suggestions or feature requests, please open a discussion or 
+join the Discord server.
 
 # Social contact
 

app/conftest.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import os
+import tempfile
+
+from app.tests.helpers import cleanup_test_run_root, get_test_run_root, get_test_system_temp_root
+
+
+def pytest_configure(config) -> None:
+    temp_root = get_test_system_temp_root()
+    for env_name in ("TMPDIR", "TEMP", "TMP"):
+        os.environ[env_name] = str(temp_root)
+
+    tempfile.tempdir = None
+
+    if getattr(config.option, "basetemp", None) is None:
+        config.option.basetemp = str(get_test_run_root() / "pytest")
+
+    os.environ["YTP_FILE_LOGGING"] = "false"
+
+
+def pytest_unconfigure(config) -> None:
+    del config
+    os.environ.pop("YTP_FILE_LOGGING", None)
+    cleanup_test_run_root()

app/features/conditions/migration.py

@@ -1,18 +1,18 @@
 from __future__ import annotations
 
 import json
-import logging
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, cast
 
 from app.features.conditions.models import ConditionModel
 from app.features.core.migration import Migration as FeatureMigration
 from app.library.config import Config
+from app.library.log import get_logger
 
 if TYPE_CHECKING:
     from app.features.conditions.repository import ConditionsRepository
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 class Migration(FeatureMigration):
@@ -36,7 +36,11 @@ class Migration(FeatureMigration):
         try:
             items: list[dict] | None = json.loads(self._source_file.read_text())
         except Exception as exc:
-            LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
+            LOG.exception(
+                "Failed to read conditions migration file '%s'. Ignoring.",
+                self._source_file,
+                extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
+            )
             await self._move_file(self._source_file)
             return
 
@@ -54,7 +58,11 @@ class Migration(FeatureMigration):
                 await self._repo.create(normalized)
                 inserted += 1
             except Exception as exc:
-                LOG.exception("Failed to insert condition '%s': %s", normalized.name, exc)
+                LOG.exception(
+                    "Failed to insert condition '%s'.",
+                    normalized.name,
+                    extra={"condition_name": normalized.name, "exception_type": type(exc).__name__},
+                )
 
         LOG.info("Migrated %s condition(s) from %s.", inserted, self._source_file)
         await self._move_file(self._source_file)

app/features/conditions/repository.py

@@ -1,31 +1,50 @@
 from __future__ import annotations
 
-import logging
 from typing import TYPE_CHECKING, Any
 
 from app.features.conditions.migration import Migration
 from app.library.Singleton import Singleton
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator, Iterable
+    from collections.abc import Callable, Iterable
+    from contextlib import AbstractAsyncContextManager
 
     from sqlalchemy.engine.result import Result
     from sqlalchemy.ext.asyncio import AsyncSession
     from sqlalchemy.sql.elements import ColumnElement
     from sqlalchemy.sql.selectable import Select
 
+    SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
+
 from sqlalchemy import delete, func, or_, select
 
 from app.features.conditions.models import ConditionModel
 from app.features.core.deps import get_session
+from app.library.log import get_logger
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
+
+
+def _model_from_payload(payload: dict[str, Any]) -> ConditionModel:
+    model = ConditionModel()
+    for key, value in payload.items():
+        if not hasattr(model, key):
+            msg = f"'{key}' is an invalid keyword argument for ConditionModel"
+            raise TypeError(msg)
+        setattr(model, key, value)
+    return model
+
+
+def _coerce_model(payload: ConditionModel | dict[str, Any]) -> ConditionModel:
+    if isinstance(payload, ConditionModel):
+        return payload
+    return _model_from_payload(payload)
 
 
 class ConditionsRepository(metaclass=Singleton):
-    def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
+    def __init__(self, session: SessionFactory | None = None) -> None:
         self._migrated = False
-        self.session: AsyncGenerator[AsyncSession] = session or get_session
+        self.session: SessionFactory = session or get_session
 
     async def run_migrations(self) -> None:
         if self._migrated:
@@ -38,7 +57,7 @@ class ConditionsRepository(metaclass=Singleton):
     def get_instance() -> ConditionsRepository:
         return ConditionsRepository()
 
-    async def list(self) -> list[ConditionModel]:
+    async def all(self) -> list[ConditionModel]:
         async with self.session() as session:
             result: Result[tuple[ConditionModel]] = await session.execute(
                 select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc())
@@ -93,11 +112,11 @@ class ConditionsRepository(metaclass=Singleton):
             result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1))
             return result.scalar_one_or_none()
 
-    async def create(self, payload: ConditionModel | dict) -> ConditionModel:
+    async def create(self, payload: ConditionModel | dict[str, Any]) -> ConditionModel:
         async with self.session() as session:
-            model: ConditionModel = ConditionModel(**payload) if isinstance(payload, dict) else payload
+            model = _coerce_model(payload)
             if model.id is not None:
-                model.id = None
+                model.id = None  # ty: ignore
 
             if await self.get_by_name(name=model.name) is not None:
                 msg: str = f"Condition with name '{model.name}' already exists."
@@ -160,13 +179,11 @@ class ConditionsRepository(metaclass=Singleton):
             await session.commit()
             return model
 
-    async def replace_all(self, items: Iterable[dict | ConditionModel]) -> list[ConditionModel]:
+    async def replace_all(self, items: Iterable[dict[str, Any] | ConditionModel]) -> list[ConditionModel]:
         async with self.session() as session:
             try:
                 await session.execute(delete(ConditionModel))
-                models: list[ConditionModel] = [
-                    ConditionModel(**item) if isinstance(item, dict) else item for item in items
-                ]
+                models: list[ConditionModel] = [_coerce_model(item) for item in items]
                 session.add_all(models)
                 await session.commit()
             except Exception:

app/features/conditions/router.py

@@ -1,4 +1,4 @@
-import logging
+import asyncio
 from collections import OrderedDict
 from typing import Any
 
@@ -14,9 +14,11 @@ from app.library.cache import Cache
 from app.library.config import Config
 from app.library.encoder import Encoder
 from app.library.Events import EventBus, Events
+from app.library.log import get_logger
 from app.library.router import route
+from app.library.Utils import validate_url
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 def _model(model: Any) -> Condition:
@@ -83,6 +85,11 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf
     if not (cond := params.get("condition")):
         return web.json_response({"error": "condition is required."}, status=web.HTTPBadRequest.status_code)
 
+    try:
+        await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
+    except ValueError as e:
+        return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code)
+
     try:
         preset: str = params.get("preset", config.default_preset)
         key: str = cache.hash(url + str(preset))
@@ -108,18 +115,44 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf
         else:
             data = cache.get(key)
     except Exception as e:
-        LOG.exception(e)
+        LOG.exception(
+            "Failed to extract video info for condition check '%s'.",
+            cond,
+            extra={
+                "route": "conditions.match",
+                "condition": cond,
+                "url": url,
+                "preset": preset,
+                "exception_type": type(e).__name__,
+            },
+        )
         return web.json_response(
             data={"error": f"Failed to extract video info. '{e!s}'"},
             status=web.HTTPInternalServerError.status_code,
         )
 
+    if not isinstance(data, dict):
+        return web.json_response(
+            data={"error": "Failed to extract video info."},
+            status=web.HTTPInternalServerError.status_code,
+        )
+
     try:
         from app.features.ytdlp.mini_filter import match_str
 
         status: bool = match_str(cond, data)
     except Exception as e:
-        LOG.exception(e)
+        LOG.exception(
+            "Failed to evaluate condition '%s'.",
+            cond,
+            extra={
+                "route": "conditions.match",
+                "condition": cond,
+                "url": url,
+                "preset": preset,
+                "exception_type": type(e).__name__,
+            },
+        )
         return web.json_response(
             data={"error": str(e)},
             status=web.HTTPBadRequest.status_code,

app/features/conditions/service.py

@@ -1,14 +1,40 @@
-import logging
+from collections.abc import Iterable
 
 from aiohttp import web
 
 from app.features.conditions.models import ConditionModel
 from app.features.conditions.repository import ConditionsRepository
+from app.features.conditions.schemas import Condition
 from app.features.ytdlp.mini_filter import match_str
 from app.library.Events import EventBus, Events
+from app.library.log import get_logger
 from app.library.Singleton import Singleton
 
-LOG: logging.Logger = logging.getLogger("feature.conditions")
+LOG = get_logger()
+
+
+def _ignored_identifiers(ignore_conditions: Iterable[str | int | float] | None) -> tuple[set[str], bool]:
+    ignored: set[str] = set()
+    ignore_all = False
+
+    if not ignore_conditions:
+        return ignored, ignore_all
+
+    for value in ignore_conditions:
+        if isinstance(value, bool) or not isinstance(value, (str, int, float)):
+            continue
+
+        identifier = str(value).strip()
+        if not identifier:
+            continue
+
+        if "*" == identifier:
+            ignore_all = True
+            continue
+
+        ignored.add(identifier)
+
+    return ignored, ignore_all
 
 
 class Conditions(metaclass=Singleton):
@@ -29,7 +55,7 @@ class Conditions(metaclass=Singleton):
         EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations")
 
     async def get_all(self) -> list[ConditionModel]:
-        return await self._repo.list()
+        return await self._repo.all()
 
     async def save(self, item: ConditionModel | dict) -> ConditionModel:
         """
@@ -54,7 +80,7 @@ class Conditions(metaclass=Singleton):
             if item.id is None or 0 == item.id:
                 model = await repo.create(item)
             else:
-                model = await repo.update(item.id, item.serialize())
+                model = await repo.update(item.id, Condition.model_validate(item).model_dump())
         except KeyError as exc:
             raise ValueError(str(exc)) from exc
 
@@ -87,12 +113,15 @@ class Conditions(metaclass=Singleton):
         repo = self._repo
         return await repo.get(identifier)
 
-    async def match(self, info: dict) -> ConditionModel | None:
+    async def match(
+        self, info: dict, ignore_conditions: Iterable[str | int | float] | None = None
+    ) -> ConditionModel | None:
         """
         Check if any condition matches the info dict.
 
         Args:
             info (dict): The info dict to check.
+            ignore_conditions (Iterable[str | int | float] | None): Condition ids or names to skip for this match.
 
         Returns:
             Condition|None: The condition if found, None otherwise.
@@ -101,8 +130,12 @@ class Conditions(metaclass=Singleton):
         if not info or not isinstance(info, dict) or len(info) < 1:
             return None
 
+        ignored_identifiers, ignore_all = _ignored_identifiers(ignore_conditions)
+        if ignore_all:
+            return None
+
         repo = self._repo
-        items: list[ConditionModel] = await repo.list()
+        items: list[ConditionModel] = await repo.all()
         if len(items) < 1:
             return None
 
@@ -110,18 +143,33 @@ class Conditions(metaclass=Singleton):
             if not item.enabled:
                 continue
 
+            if str(item.id) in ignored_identifiers or item.name in ignored_identifiers:
+                continue
+
             if not item.filter:
-                LOG.error(f"Filter is empty for '{item.name}'.")
+                LOG.error(
+                    "Filter is empty for '%s'.", item.name, extra={"condition_id": item.id, "condition_name": item.name}
+                )
                 continue
 
             try:
                 if not match_str(item.filter, info):
                     continue
 
-                LOG.debug(f"Matched '{item.id}: {item.name}' with filter '{item.filter}'.")
+                LOG.debug(
+                    "Matched '%s: %s' with filter '%s'.",
+                    item.id,
+                    item.name,
+                    item.filter,
+                    extra={"condition_id": item.id, "condition_name": item.name, "filter": item.filter},
+                )
                 return item
             except Exception as e:
-                LOG.error(f"Failed to evaluate '{item.id}: {item.name}'. '{e!s}'.")
+                LOG.exception(
+                    "Failed to evaluate condition '%s'.",
+                    item.name,
+                    extra={"condition_id": item.id, "condition_name": item.name, "exception_type": type(e).__name__},
+                )
                 continue
 
         return None
@@ -141,7 +189,7 @@ class Conditions(metaclass=Singleton):
         if not info or not isinstance(info, dict) or len(info) < 1:
             return None
 
-        if not (item := await self.get(identifier)) or not item.enabled or not item.filter:
+        if not (item := await self.get(str(identifier))) or not item.enabled or not item.filter:
             return None
 
         return item if match_str(item.filter, info) else None

app/features/conditions/tests/test_conditions.py

@@ -4,10 +4,18 @@ from __future__ import annotations
 
 import pytest
 import pytest_asyncio
+from types import SimpleNamespace
+import pytest
+from typing import Any
+from aiohttp import web
+from aiohttp.test_utils import make_mocked_request
 
-from app.features.conditions.models import ConditionModel
+from app.features.conditions.router import conditions_test
+from app.library.config import Config
+from app.library.encoder import Encoder
 from app.features.conditions.repository import ConditionsRepository
 from app.library.sqlite_store import SqliteStore
+from app.tests.helpers import make_in_memory_db_path
 
 
 @pytest_asyncio.fixture
@@ -15,7 +23,7 @@ async def repo():
     ConditionsRepository._reset_singleton()
     SqliteStore._reset_singleton()
 
-    store = SqliteStore(db_path=":memory:")
+    store = SqliteStore(db_path=make_in_memory_db_path("conditions-repository"))
     await store.get_connection()
 
     # Create repository
@@ -24,38 +32,50 @@ async def repo():
     yield repository
 
     # Cleanup - close connections properly
-    if store._conn:
-        await store._conn.close()
-    if store._engine:
-        await store._engine.dispose()
+    await store.close()
 
     # Reset singletons
     ConditionsRepository._reset_singleton()
     SqliteStore._reset_singleton()
 
 
+def _json_request(path: str, payload: object) -> web.Request:
+    mock_request: Any = make_mocked_request("POST", path)
+
+    async def _json() -> object:
+        return payload
+
+    mock_request.json = _json
+    return mock_request
+
+
+class TestAllowInternalUrlsScope:
+    def setup_method(self) -> None:
+        Config._reset_singleton()
+
+    @pytest.mark.asyncio
+    async def test_rejects_internal_url(self) -> None:
+        config = Config.get_instance()
+        config.allow_internal_urls = False
+        encoder = Encoder()
+        cache = SimpleNamespace(hash=lambda value: value, has=lambda _key: False, set=lambda **_kwargs: None)
+
+        request = _json_request(
+            "/api/conditions/test/",
+            {"url": "http://127.0.0.1/test", "condition": "title", "preset": config.default_preset},
+        )
+
+        response = await conditions_test(request, encoder, cache, config)
+
+        assert response.status == web.HTTPBadRequest.status_code
+        body = response.body
+        assert isinstance(body, bytes)
+        assert b"internal urls" in body.lower()
+
+
 class TestConditionsRepository:
     """Test suite for ConditionsRepository database operations."""
 
-    @pytest.mark.asyncio
-    async def test_repository_singleton(self, repo):
-        """Verify repository follows singleton pattern."""
-        instance1 = ConditionsRepository.get_instance()
-        instance2 = ConditionsRepository.get_instance()
-        assert instance1 is instance2, "Should return same singleton instance"
-
-    @pytest.mark.asyncio
-    async def test_list_empty(self, repo):
-        """List returns empty when no conditions exist."""
-        conditions = await repo.list()
-        assert conditions == [], "Should return empty list when no conditions"
-
-    @pytest.mark.asyncio
-    async def test_count_empty(self, repo):
-        """Count returns 0 when no conditions exist."""
-        count = await repo.count()
-        assert count == 0, "Should return 0 when no conditions exist"
-
     @pytest.mark.asyncio
     async def test_create_condition(self, repo):
         """Create condition with valid data."""
@@ -189,7 +209,7 @@ class TestConditionsRepository:
         await repo.create({"name": "A", "filter": "test", "priority": 1})
         await repo.create({"name": "C", "filter": "test", "priority": 2})
 
-        items = await repo.list()
+        items = await repo.all()
 
         assert items[0].name == "C", "Highest priority should be first"
         assert items[1].name == "A", "Same priority sorted alphabetically"
@@ -221,6 +241,6 @@ class TestConditionsRepository:
 
         assert len(result) == 2, "Should create 2 new conditions"
 
-        all_items = await repo.list()
+        all_items = await repo.all()
         assert len(all_items) == 2, "Should only have new conditions"
         assert all_items[0].name in ["New 1", "New 2"], "Should only have new items"

app/features/core/migration.py

@@ -1,15 +1,16 @@
 from __future__ import annotations
 
 import abc
-import logging
 import time
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from app.library.log import get_logger
+
 if TYPE_CHECKING:
     from app.library.config import Config
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 class Migration(abc.ABC):
@@ -26,7 +27,11 @@ class Migration(abc.ABC):
         try:
             await self.migrate()
         except Exception as exc:
-            LOG.exception("Feature migration '%s' failed: %s", self.name, exc)
+            LOG.exception(
+                "Feature migration '%s' failed.",
+                self.name,
+                extra={"feature": self.name, "exception_type": type(exc).__name__},
+            )
             return False
 
         return True

app/features/core/models.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from datetime import UTC, datetime
 
 from sqlalchemy import DateTime as SQLADateTime
-from sqlalchemy import TypeDecorator
+from sqlalchemy import Dialect, TypeDecorator
 from sqlalchemy.orm import DeclarativeBase
 
 
@@ -20,16 +20,18 @@ class UTCDateTime(TypeDecorator):
     impl = SQLADateTime
     cache_ok = True
 
-    def process_bind_param(self, value: datetime | None, _dialect) -> datetime | None:
+    def process_bind_param(self, value: datetime | None, dialect: Dialect) -> datetime | None:
         """Convert datetime to UTC before storing."""
+        _ = dialect
         if value is not None:
             if value.tzinfo is None:
                 return value.replace(tzinfo=UTC)
             return value.astimezone(UTC).replace(tzinfo=None)
         return value
 
-    def process_result_value(self, value: datetime | None, _dialect) -> datetime | None:
+    def process_result_value(self, value: datetime | None, dialect: Dialect) -> datetime | None:
         """Ensure datetime is timezone-aware (UTC) when loading."""
+        _ = dialect
         if value is not None and value.tzinfo is None:
             return value.replace(tzinfo=UTC)
         return value

app/features/dl_fields/migration.py

@@ -1,18 +1,18 @@
 from __future__ import annotations
 
 import json
-import logging
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, cast
 
 from app.features.core.migration import Migration as FeatureMigration
 from app.features.dl_fields.schemas import DLField
 from app.library.config import Config
+from app.library.log import get_logger
 
 if TYPE_CHECKING:
     from app.features.dl_fields.repository import DLFieldsRepository
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 class Migration(FeatureMigration):
@@ -36,7 +36,11 @@ class Migration(FeatureMigration):
         try:
             items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text())
         except Exception as exc:
-            LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
+            LOG.exception(
+                "Failed to read download fields migration file '%s'. Ignoring.",
+                self._source_file,
+                extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
+            )
             await self._move_file(self._source_file)
             return
 
@@ -54,7 +58,11 @@ class Migration(FeatureMigration):
                 await self._repo.create(normalized)
                 inserted += 1
             except Exception as exc:
-                LOG.exception("Failed to insert dl field '%s': %s", normalized["name"], exc)
+                LOG.exception(
+                    "Failed to insert download field '%s'.",
+                    normalized["name"],
+                    extra={"field_name": normalized["name"], "exception_type": type(exc).__name__},
+                )
 
         LOG.info("Migrated %s dl field(s) from %s.", inserted, self._source_file)
         await self._move_file(self._source_file)

app/features/dl_fields/repository.py

@@ -1,30 +1,49 @@
 from __future__ import annotations
 
-import logging
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 from sqlalchemy import delete, func, or_, select
 
 from app.features.core.deps import get_session
 from app.features.dl_fields.migration import Migration
 from app.features.dl_fields.models import DLFieldModel
+from app.library.log import get_logger
 from app.library.Singleton import Singleton
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator, Iterable
+    from collections.abc import Callable, Iterable
+    from contextlib import AbstractAsyncContextManager
 
     from sqlalchemy.engine.result import Result
     from sqlalchemy.ext.asyncio import AsyncSession
     from sqlalchemy.sql.elements import ColumnElement
     from sqlalchemy.sql.selectable import Select
 
-LOG: logging.Logger = logging.getLogger(__name__)
+    SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
+
+LOG = get_logger()
+
+
+def _model_from_payload(payload: dict[str, Any]) -> DLFieldModel:
+    model = DLFieldModel()
+    for key, value in payload.items():
+        if not hasattr(model, key):
+            msg = f"'{key}' is an invalid keyword argument for DLFieldModel"
+            raise TypeError(msg)
+        setattr(model, key, value)
+    return model
+
+
+def _coerce_model(payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
+    if isinstance(payload, DLFieldModel):
+        return payload
+    return _model_from_payload(payload)
 
 
 class DLFieldsRepository(metaclass=Singleton):
-    def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
+    def __init__(self, session: SessionFactory | None = None) -> None:
         self._migrated = False
-        self.session: AsyncGenerator[AsyncSession] = session or get_session
+        self.session: SessionFactory = session or get_session
 
     async def run_migrations(self) -> None:
         if self._migrated:
@@ -37,7 +56,7 @@ class DLFieldsRepository(metaclass=Singleton):
     def get_instance() -> DLFieldsRepository:
         return DLFieldsRepository()
 
-    async def list(self) -> list[DLFieldModel]:
+    async def all(self) -> list[DLFieldModel]:
         async with self.session() as session:
             result: Result[tuple[DLFieldModel]] = await session.execute(
                 select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc())
@@ -90,9 +109,9 @@ class DLFieldsRepository(metaclass=Singleton):
             result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1))
             return result.scalar_one_or_none()
 
-    async def create(self, payload: DLFieldModel | dict) -> DLFieldModel:
+    async def create(self, payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
         async with self.session() as session:
-            model: DLFieldModel = DLFieldModel(**payload) if isinstance(payload, dict) else payload
+            model = _coerce_model(payload)
 
             if await self.get_by_name(name=model.name) is not None:
                 msg: str = f"DL field with name '{model.name}' already exists."
@@ -153,13 +172,16 @@ class DLFieldsRepository(metaclass=Singleton):
             await session.commit()
             return model
 
-    async def replace_all(self, items: Iterable[dict | DLFieldModel]) -> list[DLFieldModel]:
+    async def replace_all(self, items: Iterable[dict[str, Any] | DLFieldModel]) -> list[DLFieldModel]:
         async with self.session() as session:
             try:
                 await session.execute(delete(DLFieldModel))
-                models: list[DLFieldModel] = [
-                    DLFieldModel(**item) if isinstance(item, dict) else item for item in items
-                ]
+                models: list[DLFieldModel] = []
+                for item in items:
+                    if isinstance(item, dict):
+                        models.append(DLFieldModel(**cast("dict[str, Any]", item)))
+                    else:
+                        models.append(item)
                 session.add_all(models)
                 await session.commit()
             except Exception:

app/features/dl_fields/router.py

@@ -1,4 +1,3 @@
-import logging
 from typing import Any
 
 from aiohttp import web
@@ -11,9 +10,10 @@ from app.features.dl_fields.schemas import DLField, DLFieldList, DLFieldPatch
 from app.features.dl_fields.service import DLFields
 from app.library.encoder import Encoder
 from app.library.Events import EventBus, Events
+from app.library.log import get_logger
 from app.library.router import route
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 def _model(model: Any) -> DLField:

app/features/dl_fields/service.py

@@ -1,18 +1,18 @@
 from __future__ import annotations
 
-import logging
 from typing import TYPE_CHECKING, Any
 
 from app.features.dl_fields.models import DLFieldModel
 from app.features.dl_fields.repository import DLFieldsRepository
 from app.features.dl_fields.schemas import DLField
 from app.library.Events import EventBus, Events
+from app.library.log import get_logger
 from app.library.Singleton import Singleton
 
 if TYPE_CHECKING:
     from aiohttp import web
 
-LOG: logging.Logger = logging.getLogger("feature.dl_fields")
+LOG = get_logger()
 
 
 class DLFields(metaclass=Singleton):
@@ -33,10 +33,10 @@ class DLFields(metaclass=Singleton):
         EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations")
 
     async def get_all(self) -> list[DLFieldModel]:
-        return await self._repo.list()
+        return await self._repo.all()
 
     async def get_all_serialized(self) -> list[dict[str, Any]]:
-        items = await self._repo.list()
+        items = await self._repo.all()
         return [DLField.model_validate(item).model_dump() for item in items]
 
     async def save(self, item: DLField | dict) -> DLFieldModel:

app/features/dl_fields/tests/test_dl_fields.py

@@ -8,26 +8,24 @@ import pytest_asyncio
 from app.features.dl_fields.repository import DLFieldsRepository
 from app.features.dl_fields.service import DLFields
 from app.library.sqlite_store import SqliteStore
+from app.tests.helpers import make_in_memory_db_path
 
 
 @pytest_asyncio.fixture
-async def repo(tmp_path):
+async def repo():
     """Provide a fresh repository instance with initialized database for each test."""
     DLFieldsRepository._reset_singleton()
     DLFields._reset_singleton()
     SqliteStore._reset_singleton()
 
-    store = SqliteStore(db_path=":memory:")
+    store = SqliteStore(db_path=make_in_memory_db_path("dl-fields-service"))
     await store.get_connection()
 
     repository = DLFieldsRepository.get_instance()
 
     yield repository
 
-    if store._conn:
-        await store._conn.close()
-    if store._engine:
-        await store._engine.dispose()
+    await store.close()
 
     DLFieldsRepository._reset_singleton()
     DLFields._reset_singleton()

app/features/dl_fields/tests/test_dl_fields_repository.py

@@ -5,28 +5,25 @@ from __future__ import annotations
 import pytest
 import pytest_asyncio
 
-from app.features.dl_fields.models import DLFieldModel
 from app.features.dl_fields.repository import DLFieldsRepository
 from app.library.sqlite_store import SqliteStore
+from app.tests.helpers import make_in_memory_db_path
 
 
 @pytest_asyncio.fixture
-async def repo(tmp_path):
+async def repo():
     """Provide a fresh repository instance with initialized database for each test."""
     DLFieldsRepository._reset_singleton()
     SqliteStore._reset_singleton()
 
-    store = SqliteStore(db_path=str(":memory:"))
+    store = SqliteStore(db_path=make_in_memory_db_path("dl-fields-repository"))
     await store.get_connection()
 
     repository = DLFieldsRepository.get_instance()
 
     yield repository
 
-    if store._conn:
-        await store._conn.close()
-    if store._engine:
-        await store._engine.dispose()
+    await store.close()
 
     DLFieldsRepository._reset_singleton()
     SqliteStore._reset_singleton()
@@ -35,25 +32,6 @@ async def repo(tmp_path):
 class TestDLFieldsRepository:
     """Test suite for DLFieldsRepository database operations."""
 
-    @pytest.mark.asyncio
-    async def test_repository_singleton(self, repo):
-        """Verify repository follows singleton pattern."""
-        instance1 = DLFieldsRepository.get_instance()
-        instance2 = DLFieldsRepository.get_instance()
-        assert instance1 is instance2, "Should return same singleton instance"
-
-    @pytest.mark.asyncio
-    async def test_list_empty(self, repo):
-        """List returns empty when no fields exist."""
-        fields = await repo.list()
-        assert fields == [], "Should return empty list when no fields"
-
-    @pytest.mark.asyncio
-    async def test_count_empty(self, repo):
-        """Count returns 0 when no fields exist."""
-        count = await repo.count()
-        assert count == 0, "Should return 0 when no fields exist"
-
     @pytest.mark.asyncio
     async def test_create_field(self, repo):
         """Create field with valid data."""
@@ -226,7 +204,7 @@ class TestDLFieldsRepository:
         await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1})
         await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0})
 
-        items = await repo.list()
+        items = await repo.all()
 
         assert items[0].name == "c", "Lowest order should be first"
         assert items[1].name == "a", "Same order sorted alphabetically"
@@ -265,6 +243,6 @@ class TestDLFieldsRepository:
 
         assert len(result) == 2, "Should create 2 new fields"
 
-        all_items = await repo.list()
+        all_items = await repo.all()
         assert len(all_items) == 2, "Should only have new fields"
         assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items"

app/features/notifications/migration.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import json
-import logging
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, cast
 
@@ -9,11 +8,12 @@ from app.features.core.migration import Migration as FeatureMigration
 from app.features.notifications.schemas import NotificationEvents
 from app.features.presets.service import Presets
 from app.library.config import Config
+from app.library.log import get_logger
 
 if TYPE_CHECKING:
     from app.features.notifications.repository import NotificationsRepository
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 class Migration(FeatureMigration):
@@ -37,7 +37,11 @@ class Migration(FeatureMigration):
         try:
             items: list[dict] | None = json.loads(self._source_file.read_text())
         except Exception as exc:
-            LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
+            LOG.exception(
+                "Failed to read notifications migration file '%s'. Ignoring.",
+                self._source_file,
+                extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
+            )
             await self._move_file(self._source_file)
             return
 
@@ -56,7 +60,11 @@ class Migration(FeatureMigration):
                 await self._repo.create(normalized)
                 inserted += 1
             except Exception as exc:
-                LOG.exception("Failed to insert notification '%s': %s", normalized.get("name"), exc)
+                LOG.exception(
+                    "Failed to insert notification target '%s'.",
+                    normalized.get("name"),
+                    extra={"target_name": normalized.get("name"), "exception_type": type(exc).__name__},
+                )
 
         LOG.info("Migrated %s notification target(s) from %s.", inserted, self._source_file)
         await self._move_file(self._source_file)

app/features/notifications/repository.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import logging
 from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import func, or_, select
@@ -8,23 +7,43 @@ from sqlalchemy import func, or_, select
 from app.features.core.deps import get_session
 from app.features.notifications.migration import Migration
 from app.features.notifications.models import NotificationModel
+from app.library.log import get_logger
 from app.library.Singleton import Singleton
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator
+    from collections.abc import Callable
+    from contextlib import AbstractAsyncContextManager
 
     from sqlalchemy.engine.result import Result
     from sqlalchemy.ext.asyncio import AsyncSession
     from sqlalchemy.sql.elements import ColumnElement
     from sqlalchemy.sql.selectable import Select
 
-LOG: logging.Logger = logging.getLogger(__name__)
+    SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
+
+LOG = get_logger()
+
+
+def _model_from_payload(payload: dict[str, Any]) -> NotificationModel:
+    model = NotificationModel()
+    for key, value in payload.items():
+        if not hasattr(model, key):
+            msg = f"'{key}' is an invalid keyword argument for NotificationModel"
+            raise TypeError(msg)
+        setattr(model, key, value)
+    return model
+
+
+def _coerce_model(payload: NotificationModel | dict[str, Any]) -> NotificationModel:
+    if isinstance(payload, NotificationModel):
+        return payload
+    return _model_from_payload(payload)
 
 
 class NotificationsRepository(metaclass=Singleton):
-    def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
+    def __init__(self, session: SessionFactory | None = None) -> None:
         self._migrated = False
-        self.session: AsyncGenerator[AsyncSession] = session or get_session
+        self.session: SessionFactory = session or get_session
 
     async def run_migrations(self) -> None:
         if self._migrated:
@@ -37,7 +56,7 @@ class NotificationsRepository(metaclass=Singleton):
     def get_instance() -> NotificationsRepository:
         return NotificationsRepository()
 
-    async def list(self) -> list[NotificationModel]:
+    async def all(self) -> list[NotificationModel]:
         async with self.session() as session:
             result: Result[tuple[NotificationModel]] = await session.execute(
                 select(NotificationModel).order_by(NotificationModel.name.asc())
@@ -92,11 +111,11 @@ class NotificationsRepository(metaclass=Singleton):
             result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1))
             return result.scalar_one_or_none()
 
-    async def create(self, payload: NotificationModel | dict) -> NotificationModel:
+    async def create(self, payload: NotificationModel | dict[str, Any]) -> NotificationModel:
         async with self.session() as session:
-            model: NotificationModel = NotificationModel(**payload) if isinstance(payload, dict) else payload
+            model = _coerce_model(payload)
             if model.id is not None:
-                model.id = None
+                model.id = None  # ty: ignore
 
             if await self.get_by_name(name=model.name) is not None:
                 msg: str = f"Notification target with name '{model.name}' already exists."

app/features/notifications/router.py

@@ -1,4 +1,3 @@
-import logging
 from typing import Any
 
 from aiohttp import web
@@ -11,9 +10,10 @@ from app.features.notifications.schemas import Notification, NotificationEvents,
 from app.features.notifications.service import Notifications
 from app.library.encoder import Encoder
 from app.library.Events import EventBus, Events
+from app.library.log import get_logger
 from app.library.router import route
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 def _model(model: Any) -> Notification:

app/features/notifications/schemas.py

@@ -16,6 +16,7 @@ class NotificationEvents:
     ITEM_COMPLETED: str = Events.ITEM_COMPLETED
     ITEM_CANCELLED: str = Events.ITEM_CANCELLED
     ITEM_DELETED: str = Events.ITEM_DELETED
+    ITEM_BULK_DELETED: str = Events.ITEM_BULK_DELETED
     ITEM_PAUSED: str = Events.ITEM_PAUSED
     ITEM_RESUMED: str = Events.ITEM_RESUMED
     ITEM_MOVED: str = Events.ITEM_MOVED

app/features/notifications/service.py

@@ -1,10 +1,8 @@
 from __future__ import annotations
 
 import asyncio
-import logging
-import traceback
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 from app.features.notifications.models import NotificationModel
 from app.features.notifications.repository import NotificationsRepository
@@ -23,6 +21,7 @@ from app.library.encoder import Encoder
 from app.library.Events import Event, EventBus, Events
 from app.library.httpx_client import async_client
 from app.library.ItemDTO import Item, ItemDTO
+from app.library.log import get_logger
 from app.library.Singleton import Singleton
 
 if TYPE_CHECKING:
@@ -31,7 +30,7 @@ if TYPE_CHECKING:
     import httpx
     from aiohttp import web
 
-LOG: logging.Logger = logging.getLogger("feature.notifications")
+LOG = get_logger()
 
 
 class Notifications(metaclass=Singleton):
@@ -65,10 +64,10 @@ class Notifications(metaclass=Singleton):
             await self._repo.run_migrations()
 
         EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations")
-        EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{__class__.__name__}.emit")
+        EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{type(self).__name__}.emit")
 
-    async def list(self) -> list[NotificationModel]:
-        return await self._repo.list()
+    async def all(self) -> list[NotificationModel]:
+        return await self._repo.all()
 
     async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]:
         return await self._repo.list_paginated(page, per_page)
@@ -93,8 +92,8 @@ class Notifications(metaclass=Singleton):
     async def delete(self, identifier: int | str) -> NotificationModel:
         return await self._repo.delete(identifier)
 
-    async def send(self, ev: Event, wait: bool = True) -> list[dict] | list[Awaitable[dict]]:
-        targets = await self._repo.list()
+    async def send(self, ev: Event, wait: bool = True) -> list[dict | Awaitable[dict]]:
+        targets = await self._repo.all()
         if len(targets) < 1:
             return []
 
@@ -127,7 +126,7 @@ class Notifications(metaclass=Singleton):
         if wait:
             return await asyncio.gather(*tasks)
 
-        return tasks
+        return cast("list[dict | Awaitable[dict]]", tasks)
 
     def emit(self, e: Event, _, **__) -> None:
         if not NotificationEvents.is_valid(e.event):
@@ -272,14 +271,27 @@ class Notifications(metaclass=Singleton):
 
             if not status:
                 msg = "Apprise failed to send notification."
-                raise RuntimeError(msg)  # noqa: TRY301
+                self._raise_apprise_error(msg)
         except Exception as exc:
-            LOG.exception(exc)
-            LOG.error("Error sending Apprise notification: %s", exc)
+            LOG.exception(
+                "Failed to send Apprise notification for event '%s'.",
+                ev.event,
+                extra={
+                    "event_id": ev.id,
+                    "event": ev.event,
+                    "target_count": len(targets),
+                    "targets": [t.name for t in targets],
+                    "exception_type": type(exc).__name__,
+                },
+            )
             return {"error": str(exc), "event": ev.event, "id": ev.id, "targets": [t.name for t in targets]}
 
         return {}
 
+    @staticmethod
+    def _raise_apprise_error(msg: str) -> None:
+        raise RuntimeError(msg)
+
     async def _send(self, target: Notification, ev: Event) -> dict:
         try:
             LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name)
@@ -335,14 +347,17 @@ class Notifications(metaclass=Singleton):
 
             return resp_data
         except Exception as exc:
-            err_msg = str(exc) or type(exc).__name__
-            tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
-            LOG.error(
-                "Error sending notification event '%s: %s' to '%s'. '%s'. %s",
+            LOG.exception(
+                "Failed to send notification event '%s: %s' to '%s'.",
                 ev.event,
                 ev.id,
                 target.name,
-                err_msg,
-                tb,
+                extra={
+                    "event_id": ev.id,
+                    "event": ev.event,
+                    "target_name": target.name,
+                    "url": target.request.url,
+                    "exception_type": type(exc).__name__,
+                },
             )
             return {"url": target.request.url, "status": 500, "text": str(ev)}

app/features/notifications/tests/test_notifications_repository.py

@@ -7,25 +7,23 @@ import pytest_asyncio
 
 from app.features.notifications.repository import NotificationsRepository
 from app.library.sqlite_store import SqliteStore
+from app.tests.helpers import make_in_memory_db_path
 
 
 @pytest_asyncio.fixture
-async def repo(tmp_path):
+async def repo():
     """Provide a fresh repository instance with initialized database for each test."""
     NotificationsRepository._reset_singleton()
     SqliteStore._reset_singleton()
 
-    store = SqliteStore(db_path=":memory:")
+    store = SqliteStore(db_path=make_in_memory_db_path("notifications-repository"))
     await store.get_connection()
 
     repository = NotificationsRepository.get_instance()
 
     yield repository
 
-    if store._conn:
-        await store._conn.close()
-    if store._engine:
-        await store._engine.dispose()
+    await store.close()
 
     NotificationsRepository._reset_singleton()
     SqliteStore._reset_singleton()
@@ -37,7 +35,7 @@ class TestNotificationsRepository:
     @pytest.mark.asyncio
     async def test_list_empty(self, repo):
         """List returns empty when no notifications exist."""
-        notifications = await repo.list()
+        notifications = await repo.all()
         assert notifications == [], "Should return empty list when no notifications exist"
 
     @pytest.mark.asyncio

app/features/presets/migration.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import json
-import logging
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
@@ -9,11 +8,12 @@ from app.features.core.migration import Migration as FeatureMigration
 from app.features.presets.schemas import Preset
 from app.features.presets.utils import preset_name
 from app.library.config import Config
+from app.library.log import get_logger
 
 if TYPE_CHECKING:
     from app.features.presets.repository import PresetsRepository
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 class Migration(FeatureMigration):
@@ -32,7 +32,11 @@ class Migration(FeatureMigration):
         try:
             items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text())
         except Exception as exc:
-            LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
+            LOG.exception(
+                "Failed to read presets migration file '%s'. Ignoring.",
+                self._source_file,
+                extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
+            )
             await self._move_file(self._source_file)
             return
 
@@ -42,7 +46,7 @@ class Migration(FeatureMigration):
             return
 
         inserted = 0
-        seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.list()}
+        seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.all()}
 
         for index, item in enumerate(items):
             if not (normalized := self._normalize(item, index, seen_names)):
@@ -51,7 +55,11 @@ class Migration(FeatureMigration):
                 await self._repo.create(normalized)
                 inserted += 1
             except Exception as exc:
-                LOG.exception("Failed to insert preset '%s': %s", normalized["name"], exc)
+                LOG.exception(
+                    "Failed to insert preset '%s'.",
+                    normalized["name"],
+                    extra={"preset": normalized["name"], "exception_type": type(exc).__name__},
+                )
 
         LOG.info("Migrated %s preset(s) from %s.", inserted, self._source_file)
         await self._move_file(self._source_file)

app/features/presets/repository.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import logging
 from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import func, or_, select
@@ -12,11 +11,13 @@ from app.features.presets.models import PresetModel
 from app.features.presets.utils import preset_name, seed_defaults
 from app.library.config import Config
 from app.library.Events import Event, EventBus, Events
+from app.library.log import get_logger
 from app.library.Services import Services
 from app.library.Singleton import Singleton
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator
+    from collections.abc import Callable
+    from contextlib import AbstractAsyncContextManager
 
     from aiohttp import web
     from sqlalchemy.engine.result import Result
@@ -24,13 +25,58 @@ if TYPE_CHECKING:
     from sqlalchemy.sql.elements import ColumnElement
     from sqlalchemy.sql.selectable import Select
 
-LOG: logging.Logger = logging.getLogger(__name__)
+    SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
+
+LOG = get_logger()
+
+
+def _payload_data(payload: PresetModel | dict[str, Any]) -> dict[str, Any]:
+    if isinstance(payload, dict):
+        data: dict[str, Any] = {}
+        for key, value in payload.items():
+            if not isinstance(key, str):
+                msg = "Preset payload keys must be strings."
+                raise TypeError(msg)
+            data[key] = value
+        return data
+
+    return {
+        "name": payload.name,
+        "description": payload.description,
+        "folder": payload.folder,
+        "template": payload.template,
+        "cookies": payload.cookies,
+        "cli": payload.cli,
+        "default": payload.default,
+        "priority": payload.priority,
+        "created_at": payload.created_at,
+        "updated_at": payload.updated_at,
+    }
 
 
 class PresetsRepository(metaclass=Singleton):
-    def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
+    SORT_FIELDS: dict[str, Any] = {
+        "id": PresetModel.id,
+        "name": PresetModel.name,
+        "priority": PresetModel.priority,
+        "default": PresetModel.default,
+        "created_at": PresetModel.created_at,
+        "updated_at": PresetModel.updated_at,
+    }
+    SORT_DIRECTIONS: tuple[str, str] = ("asc", "desc")
+    DEFAULT_SORT_ORDER: tuple[tuple[str, str], ...] = (("priority", "desc"), ("name", "asc"))
+    FIELD_DEFAULT_DIRECTIONS: dict[str, str] = {
+        "id": "asc",
+        "name": "asc",
+        "priority": "desc",
+        "default": "desc",
+        "created_at": "desc",
+        "updated_at": "desc",
+    }
+
+    def __init__(self, session: SessionFactory | None = None) -> None:
         self._migrated = False
-        self.session: AsyncGenerator[AsyncSession] = session or get_session
+        self.session: SessionFactory = session or get_session
 
     async def run_migrations(self) -> None:
         if self._migrated:
@@ -51,15 +97,15 @@ class PresetsRepository(metaclass=Singleton):
                 LOG.debug("Refreshing presets cache due to configuration update.")
                 await self._update_cache()
 
-        Services.get_instance().add(__class__.__name__, self)
+        Services.get_instance().add(PresetsRepository.__name__, self)
         EventBus.get_instance().subscribe(
-            Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
+            Events.STARTED, handle_event, f"{PresetsRepository.__name__}.run_migrations"
         ).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache")
 
     async def _update_cache(self) -> None:
         from app.features.presets.service import Presets
 
-        await Presets.get_instance().refresh_cache(await self.list())
+        await Presets.get_instance().refresh_cache(await self.all())
 
     @staticmethod
     def get_instance() -> PresetsRepository:
@@ -75,33 +121,115 @@ class PresetsRepository(metaclass=Singleton):
             LOG.error("Default preset '%s' not found, using 'default' preset.", default_name)
             config.default_preset = "default"
 
-    async def list(self) -> list[PresetModel]:
+    async def all(self) -> list[PresetModel]:
         async with self.session() as session:
             result: Result[tuple[PresetModel]] = await session.execute(
                 select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc())
             )
             return list(result.scalars().all())
 
-    async def list_paginated(self, page: int, per_page: int) -> tuple[list[PresetModel], int, int, int]:
+    @classmethod
+    def parse_sorting(cls, sort: str | None = None, order: str | None = None) -> tuple[tuple[str, str], ...]:
+        sort_value = (sort or "").strip()
+        order_value = (order or "").strip()
+
+        if not sort_value and not order_value:
+            return cls.DEFAULT_SORT_ORDER
+
+        if order_value and not sort_value:
+            msg = "sort is required when order is provided."
+            raise ValueError(msg)
+
+        fields: list[str] = []
+        for raw_field in sort_value.split(","):
+            field = raw_field.strip().lower()
+            if not field:
+                continue
+            if field not in cls.SORT_FIELDS:
+                msg = f"sort must use supported fields: {', '.join(cls.SORT_FIELDS)}."
+                raise ValueError(msg)
+            if field not in fields:
+                fields.append(field)
+
+        if not fields:
+            msg = "sort must include at least one field."
+            raise ValueError(msg)
+
+        directions: list[str] = []
+        if order_value:
+            for raw_direction in order_value.split(","):
+                direction = raw_direction.strip().lower()
+                if not direction:
+                    continue
+                if direction not in cls.SORT_DIRECTIONS:
+                    msg = "order must be 'asc' or 'desc'."
+                    raise ValueError(msg)
+                directions.append(direction)
+
+            if not directions:
+                msg = "order must include at least one direction."
+                raise ValueError(msg)
+
+            if len(directions) not in {1, len(fields)}:
+                msg = "order must provide one direction or match the number of sort fields."
+                raise ValueError(msg)
+
+        if not directions:
+            return tuple((field, cls.FIELD_DEFAULT_DIRECTIONS.get(field, "asc")) for field in fields)
+
+        if len(directions) == 1:
+            return tuple((field, directions[0]) for field in fields)
+
+        return tuple((field, directions[index]) for index, field in enumerate(fields))
+
+    @classmethod
+    def _apply_sort_direction(cls, field: str, direction: str) -> Any:
+        column = cls.SORT_FIELDS[field]
+        return column.asc() if direction == "asc" else column.desc()
+
+    @classmethod
+    def _build_order_by(cls, sort: str | None = None, order: str | None = None) -> list[Any]:
+        sorting = cls.parse_sorting(sort, order)
+
+        order_by: list[Any] = [cls._apply_sort_direction(field, direction) for field, direction in sorting]
+
+        if all(field != "id" for field, _ in sorting):
+            order_by.append(PresetModel.id.asc())
+
+        return order_by
+
+    async def list_paginated(
+        self,
+        page: int,
+        per_page: int,
+        sort: str | None = None,
+        order: str | None = None,
+        exclude_defaults: bool = False,
+    ) -> tuple[list[PresetModel], int, int, int]:
+        order_by = self._build_order_by(sort, order)
+
         async with self.session() as session:
-            total: int = await self.count()
+            total: int = await self.count(exclude_defaults=exclude_defaults)
             total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1
 
             if page > total_pages and total > 0:
                 page = total_pages
 
-            query: Select[tuple[PresetModel]] = (
-                select(PresetModel)
-                .order_by(PresetModel.priority.desc(), PresetModel.name.asc())
-                .limit(per_page)
-                .offset((page - 1) * per_page)
-            )
+            query: Select[tuple[PresetModel]] = select(PresetModel)
+            if exclude_defaults:
+                query = query.where(PresetModel.default.is_(False))
+
+            query = query.order_by(*order_by).limit(per_page).offset((page - 1) * per_page)
             result: Result[tuple[PresetModel]] = await session.execute(query)
             return list(result.scalars().all()), total, page, total_pages
 
-    async def count(self) -> int:
+    async def count(self, exclude_defaults: bool = False) -> int:
         async with self.session() as session:
-            result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(PresetModel))
+            query = select(func.count()).select_from(PresetModel)
+            if exclude_defaults:
+                query = query.where(PresetModel.default.is_(False))
+
+            result: Result[tuple[int]] = await session.execute(query)
             return int(result.scalar_one())
 
     async def get(self, identifier: int | str) -> PresetModel | None:
@@ -134,11 +262,12 @@ class PresetsRepository(metaclass=Singleton):
             result: Result[tuple[PresetModel]] = await session.execute(query.limit(1))
             return result.scalar_one_or_none()
 
-    async def create(self, payload: PresetModel | dict) -> PresetModel:
+    async def create(self, payload: PresetModel | dict[str, Any]) -> PresetModel:
         async with self.session() as session:
-            model: PresetModel = PresetModel(**payload) if isinstance(payload, dict) else payload
-            if model.id is not None:
-                model.id = None
+            data = _payload_data(payload)
+
+            data.pop("id", None)
+            model = PresetModel(**data)
 
             model.name = preset_name(model.name)
 
@@ -163,12 +292,10 @@ class PresetsRepository(metaclass=Singleton):
             result: Result[tuple[PresetModel]] = await session.execute(select(PresetModel).where(clause).limit(1))
             model: PresetModel | None = result.scalar_one_or_none()
 
-            if None is model:
+            if model is None:
                 msg: str = f"Preset '{identifier}' not found."
                 raise KeyError(msg)
 
-            assert None is not model
-
             payload.pop("id", None)
             payload.pop("created_at", None)
             payload.pop("updated_at", None)
@@ -177,9 +304,10 @@ class PresetsRepository(metaclass=Singleton):
                 if hasattr(model, key):
                     setattr(model, key, value)
 
-            model.name = preset_name(model.name)
-            if await self.get_by_name(name=model.name, exclude_id=model.id) is not None:
-                msg = f"Preset with name '{model.name}' already exists."
+            normalized_name = preset_name(model.name)
+            model.name = normalized_name
+            if await self.get_by_name(name=normalized_name, exclude_id=model.id) is not None:
+                msg = f"Preset with name '{normalized_name}' already exists."
                 raise ValueError(msg)
 
             await session.commit()
@@ -196,12 +324,11 @@ class PresetsRepository(metaclass=Singleton):
                 clause = PresetModel.name == preset_name(identifier)
 
             result: Result[tuple[PresetModel]] = await session.execute(select(PresetModel).where(clause).limit(1))
-            if not (model := result.scalar_one_or_none()):
+            model: PresetModel | None = result.scalar_one_or_none()
+            if model is None:
                 msg: str = f"Preset '{identifier}' not found."
                 raise KeyError(msg)
 
-            assert None is not model
-
             await session.delete(model)
             await session.commit()
             return model

app/features/presets/router.py

@@ -23,8 +23,18 @@ def _serialize(model: Any) -> dict:
 
 @route("GET", "api/presets/", "presets")
 async def presets_list(request: Request, encoder: Encoder, repo: PresetsRepository) -> Response:
-    page, per_page = normalize_pagination(request)
-    items, total, current_page, total_pages = await repo.list_paginated(page, per_page)
+    try:
+        page, per_page = normalize_pagination(request)
+        items, total, current_page, total_pages = await repo.list_paginated(
+            page=page,
+            per_page=per_page,
+            sort=request.query.get("sort"),
+            order=request.query.get("order"),
+            exclude_defaults=bool(request.query.get("exclude_defaults", False)),
+        )
+    except ValueError as exc:
+        return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
+
     return web.json_response(
         data=PresetList(
             items=[_model(model) for model in items],

app/features/presets/service.py

@@ -12,7 +12,7 @@ class Presets(metaclass=Singleton):
     def __init__(self, repo: PresetsRepository | None = None) -> None:
         self._repo: PresetsRepository = repo or PresetsRepository.get_instance()
         self._cache: list[tuple[int, str, Preset]] = []
-        Services.get_instance().add(__class__.__name__, self)
+        Services.get_instance().add(type(self).__name__, self)
 
     @staticmethod
     def get_instance() -> Presets:

app/features/presets/tests/test_presets_repository.py

@@ -1,10 +1,18 @@
 from __future__ import annotations
 
+import json
+from unittest.mock import MagicMock
+
 import pytest
 import pytest_asyncio
+from aiohttp import web
+from aiohttp.web import Request
 
 from app.features.presets.repository import PresetsRepository
+from app.features.presets.router import presets_list
+from app.library.encoder import Encoder
 from app.library.sqlite_store import SqliteStore
+from app.tests.helpers import make_in_memory_db_path
 
 
 @pytest_asyncio.fixture
@@ -12,16 +20,13 @@ async def repo():
     PresetsRepository._reset_singleton()
     SqliteStore._reset_singleton()
 
-    store = SqliteStore(db_path=":memory:")
+    store = SqliteStore(db_path=make_in_memory_db_path("presets-repository"))
     await store.get_connection()
 
     repository = PresetsRepository.get_instance()
     yield repository
 
-    if store._conn:
-        await store._conn.close()
-    if store._engine:
-        await store._engine.dispose()
+    await store.close()
 
     PresetsRepository._reset_singleton()
     SqliteStore._reset_singleton()
@@ -52,7 +57,7 @@ class TestPresetsRepository:
         await repo.create({"name": "A", "priority": 1})
         await repo.create({"name": "C", "priority": 2})
 
-        items = await repo.list()
+        items = await repo.all()
 
         assert items[0].name == "c", "Highest priority should be first"
         assert items[1].name == "a", "Same priority should sort by name"
@@ -69,3 +74,106 @@ class TestPresetsRepository:
         assert total == 5, "Should report total count"
         assert page == 1, "Should be on page 1"
         assert total_pages == 3, "Should have 3 pages total"
+        assert [item.priority for item in items] == [4, 3], "Should keep default priority-desc order"
+
+    @pytest.mark.asyncio
+    async def test_list_paginated_sorts_by_name_desc(self, repo):
+        await repo.create({"name": "Alpha", "priority": 1})
+        await repo.create({"name": "Gamma", "priority": 3})
+        await repo.create({"name": "Beta", "priority": 2})
+
+        items, _, _, _ = await repo.list_paginated(page=1, per_page=10, sort="name", order="desc")
+
+        assert [item.name for item in items] == ["gamma", "beta", "alpha"], "Should sort by requested field"
+
+    @pytest.mark.asyncio
+    async def test_list_paginated_excludes_defaults(self, repo):
+        await repo.create({"name": "System Default", "default": True, "priority": 10})
+        await repo.create({"name": "Custom Preset", "priority": 1})
+
+        items, total, page, total_pages = await repo.list_paginated(page=1, per_page=10, exclude_defaults=True)
+
+        assert [item.name for item in items] == ["custom_preset"], "Should exclude default presets"
+        assert total == 1, "Should count only custom presets"
+        assert page == 1, "Should keep current page when filtered results exist"
+        assert total_pages == 1, "Should compute pages from the filtered total"
+
+    @pytest.mark.asyncio
+    async def test_list_paginated_multi_sort(self, repo):
+        await repo.create({"name": "Charlie", "priority": 2})
+        await repo.create({"name": "Alpha", "priority": 1})
+        await repo.create({"name": "Bravo", "priority": 1})
+
+        items, _, _, _ = await repo.list_paginated(page=1, per_page=10, sort="priority,name", order="asc,desc")
+
+        assert [(item.priority, item.name) for item in items] == [
+            (1, "bravo"),
+            (1, "alpha"),
+            (2, "charlie"),
+        ], "Should support multiple sort fields and directions"
+
+    @pytest.mark.asyncio
+    async def test_list_paginated_bad_sort(self, repo):
+        with pytest.raises(ValueError, match="sort must use supported fields"):
+            await repo.list_paginated(page=1, per_page=10, sort="cli", order="asc")
+
+    @pytest.mark.asyncio
+    async def test_list_paginated_bad_order(self, repo):
+        with pytest.raises(ValueError, match="order must be 'asc' or 'desc'"):
+            await repo.list_paginated(page=1, per_page=10, sort="name", order="sideways")
+
+    @pytest.mark.asyncio
+    async def test_list_paginated_mismatched_order(self, repo):
+        with pytest.raises(ValueError, match="order must provide one direction or match the number of sort fields"):
+            await repo.list_paginated(page=1, per_page=10, sort="priority,name", order="asc,desc,asc")
+
+
+@pytest.mark.asyncio
+class TestPresetRoutes:
+    async def test_list_route_sort(self, repo):
+        await repo.create({"name": "Alpha", "priority": 1})
+        await repo.create({"name": "Bravo", "priority": 1})
+        await repo.create({"name": "Charlie", "priority": 2})
+
+        request = MagicMock(spec=Request)
+        request.query = {"page": "1", "per_page": "10", "sort": "priority,name", "order": "asc,desc"}
+
+        response = await presets_list(request, Encoder(), repo)
+        payload = json.loads(response.text)
+
+        assert response.status == web.HTTPOk.status_code, "Should return 200 for valid sorting"
+        assert [item["name"] for item in payload["items"]] == ["bravo", "alpha", "charlie"], "Should sort response"
+
+    async def test_list_route_bad_sort(self, repo):
+        request = MagicMock(spec=Request)
+        request.query = {"sort": "cli", "order": "asc"}
+
+        response = await presets_list(request, Encoder(), repo)
+        payload = json.loads(response.text)
+
+        assert response.status == web.HTTPBadRequest.status_code, "Should reject unsupported sort field"
+        assert "sort" in payload["error"], "Should explain invalid sort field"
+
+    async def test_list_route_bad_order(self, repo):
+        request = MagicMock(spec=Request)
+        request.query = {"sort": "name", "order": "sideways"}
+
+        response = await presets_list(request, Encoder(), repo)
+        payload = json.loads(response.text)
+
+        assert response.status == web.HTTPBadRequest.status_code, "Should reject unsupported sort direction"
+        assert "order" in payload["error"], "Should explain invalid sort direction"
+
+    async def test_list_route_exclude_defaults(self, repo):
+        await repo.create({"name": "System Default", "default": True, "priority": 10})
+        await repo.create({"name": "Custom Preset", "priority": 1})
+
+        request = MagicMock(spec=Request)
+        request.query = {"page": "1", "per_page": "10", "exclude_defaults": "true"}
+
+        response = await presets_list(request, Encoder(), repo)
+        payload = json.loads(response.text)
+
+        assert response.status == web.HTTPOk.status_code, "Should return 200 for valid default exclusion"
+        assert [item["name"] for item in payload["items"]] == ["custom_preset"], "Should exclude default presets"
+        assert payload["pagination"]["total"] == 1, "Should report filtered total"

app/features/presets/utils.py

@@ -1,10 +1,11 @@
-import logging
 import re
 from datetime import UTC, datetime
 
+from app.library.log import get_logger
+
 NAME_WHITESPACE_PATTERN = re.compile(r"\s+")
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 async def seed_defaults(repo) -> None:
@@ -48,7 +49,11 @@ async def seed_defaults(repo) -> None:
 
             await repo.update(existing.id, payload)
         except Exception as exc:
-            LOG.exception("Failed to seed default preset '%s': %s", preset.get("name"), exc)
+            LOG.exception(
+                "Failed to seed default preset '%s'.",
+                preset.get("name"),
+                extra={"preset": preset.get("name"), "exception_type": type(exc).__name__},
+            )
 
 
 def preset_name(value: str) -> str:

app/features/streaming/library/ffprobe.py

@@ -5,18 +5,16 @@ Python wrapper for ffprobe command line tool. ffprobe must exist in the path.
 import asyncio
 import functools
 import json
-import logging
 import operator
 import os
 import subprocess  # qa: ignore
 from pathlib import Path
 
-import anyio
-
 from app.features.streaming.types import FFProbeError
+from app.library.log import get_logger
 from app.library.Utils import timed_lru_cache
 
-LOG: logging.Logger = logging.getLogger("streaming.ffprobe")
+LOG = get_logger()
 
 
 class FFStream:
@@ -39,21 +37,26 @@ class FFStream:
 
     def __repr__(self):
         if "codec_long_name" not in self.__dict__:
-            self.codec_long_name = self.__dict__.get("codec_name", "")
+            self.__dict__["codec_long_name"] = self.__dict__.get("codec_name", "")
+
+        index = self.__dict__.get("index", "?")
+        codec_type = self.__dict__.get("codec_type", "unknown")
+        codec_long_name = self.__dict__.get("codec_long_name", "")
 
         if self.is_video():
-            return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, {self.framerate}, ({self.width}x{self.height})>"
+            return f"<Stream: #{index} [{codec_type}] {codec_long_name}, {self.__dict__.get('framerate')}, ({self.__dict__.get('width')}x{self.__dict__.get('height')})>"
 
         if self.is_audio():
             return (
-                f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, channels: {self.channels} ({self.channel_layout}), "
-                "{sample_rate}Hz> "
+                f"<Stream: #{index} [{codec_type}] {codec_long_name}, "
+                f"channels: {self.__dict__.get('channels')} ({self.__dict__.get('channel_layout')}), "
+                f"{self.__dict__.get('sample_rate')}Hz> "
             )
 
         if self.is_subtitle() or self.is_attachment():
-            return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}>"
+            return f"<Stream: #{index} [{codec_type}] {codec_long_name}>"
 
-        return f"<Stream: #{self.index} [{self.codec_type}]>"
+        return f"<Stream: #{index} [{codec_type}]>"
 
     def is_audio(self):
         """
@@ -252,14 +255,14 @@ async def ffprobe(file: Path | str) -> FFProbeResult:
         raise OSError(msg)
 
     try:
-        async with await anyio.open_file(os.devnull, "w") as tempf:
-            await asyncio.create_subprocess_exec(
-                "ffprobe",
-                "-h",
-                stdout=tempf,
-                stderr=tempf,
-                creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
-            )
+        proc = await asyncio.create_subprocess_exec(
+            "ffprobe",
+            "-h",
+            stdout=asyncio.subprocess.DEVNULL,
+            stderr=asyncio.subprocess.DEVNULL,
+            creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
+        )
+        await proc.wait()
     except FileNotFoundError as e:
         msg = "ffprobe not found."
         raise OSError(msg) from e

app/features/streaming/library/m3u8.py

@@ -39,7 +39,7 @@ class M3u8:
         m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
         m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
 
-        segmentSize: float = f"{self.duration:.6f}"
+        segmentSize: str = f"{self.duration:.6f}"
         splits: int = math.ceil(duration / self.duration)
 
         segmentParams: dict = {}

app/features/streaming/library/playlist.py

@@ -14,7 +14,7 @@ class Playlist:
         "The path where files are downloaded."
 
     async def make(self, file: Path) -> str:
-        ref: str = Path(str(file.relative_to(self.download_path)).strip("/"))
+        ref: Path = Path(str(file.relative_to(self.download_path)).strip("/"))
 
         try:
             ff = await ffprobe(file)

app/features/streaming/library/segment_encoders.py

@@ -1,7 +1,7 @@
-# flake8: noqa: ARG002
 from __future__ import annotations
 
 import os
+import shutil
 import subprocess
 import sys
 from functools import lru_cache
@@ -16,9 +16,13 @@ def detect_qsv_capabilities() -> dict[str, dict[str, bool]]:
     Returns a dict where keys are codec names (e.g. "h264", "hevc", "vp9") and
     values are dicts with keys "full" and "lp" booleans.
     """
+    vainfo = shutil.which("vainfo")
+    if not vainfo:
+        return {}
+
     try:
         result: subprocess.CompletedProcess[str] = subprocess.run(
-            ["vainfo"],  # noqa: S607
+            [vainfo],
             capture_output=True,
             text=True,
             check=False,
@@ -87,9 +91,13 @@ def ffmpeg_encoders() -> set[str]:
     """
     from app.library.config import SUPPORTED_CODECS
 
+    ffmpeg = shutil.which("ffmpeg")
+    if not ffmpeg:
+        return set()
+
     try:
         result: subprocess.CompletedProcess[str] = subprocess.run(
-            ["ffmpeg", "-hide_banner", "-loglevel", "error", "-encoders"],  # noqa: S607
+            [ffmpeg, "-hide_banner", "-loglevel", "error", "-encoders"],
             capture_output=True,
             text=True,
             check=False,
@@ -149,9 +157,11 @@ class _BaseBuilder:
     codec_name: str
 
     def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
+        _ = ctx
         return []
 
     def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
+        _ = ctx
         return [*args, "-codec:v", self.codec_name]
 
 
@@ -312,4 +322,4 @@ def encoder_fallback_chain(codec: str) -> tuple[str, ...]:
         "libx264": [],
     }
 
-    return chains.get(codec, chains["libx264"])
+    return tuple(chains.get(codec, chains["libx264"]))

app/features/streaming/library/segments.py

@@ -1,11 +1,10 @@
 import asyncio
-import hashlib
-import logging
 import os
-import subprocess  # type: ignore
+import subprocess
 import sys
 import tempfile
 from pathlib import Path
+from secrets import token_hex
 from typing import TYPE_CHECKING, ClassVar
 
 from aiohttp import web
@@ -19,6 +18,7 @@ from app.features.streaming.library.segment_encoders import (
     select_encoder,
 )
 from app.library.config import SUPPORTED_CODECS, Config
+from app.library.log import get_logger
 
 if TYPE_CHECKING:
     from asyncio.subprocess import Process
@@ -26,7 +26,7 @@ if TYPE_CHECKING:
     from .ffprobe import FFProbeResult
     from .segment_encoders import EncoderBuilder
 
-LOG: logging.Logger = logging.getLogger("player.segments")
+LOG = get_logger()
 
 
 class Segments:
@@ -68,21 +68,21 @@ class Segments:
         self.attempted: set[str] = set()
         "The set of attempted codecs."
 
-    async def build_ffmpeg_args(self, file: Path, s_codec: str) -> list[str]:
+    def _make_stream_input(self, file: Path) -> Path:
+        while True:
+            stream_input = Path(tempfile.gettempdir()).joinpath(f"ytptube_stream.{token_hex(8)}")
+            try:
+                stream_input.symlink_to(file, target_is_directory=False)
+                return stream_input
+            except FileExistsError:
+                continue
+
+    async def build_ffmpeg_args(self, file: Path, s_codec: str, *, stream_input: Path | None = None) -> list[str]:
         try:
             ff: FFProbeResult = await ffprobe(file)
         except UnicodeDecodeError:
             pass
-
-        tmpFile: Path = Path(tempfile.gettempdir()).joinpath(
-            f"ytptube_stream.{hashlib.sha256(str(file).encode()).hexdigest()}"
-        )
-
-        if not tmpFile.exists():
-            try:
-                tmpFile.symlink_to(file, target_is_directory=False)
-            except FileExistsError:
-                pass
+        input_path = stream_input or file
 
         startTime: str = f"{0:.6f}" if self.index == 0 else f"{self.duration * self.index:.6f}"
 
@@ -123,7 +123,7 @@ class Segments:
             # hardware/global input options must come before -i
             *input_args,
             "-i",
-            f"file:{tmpFile}",
+            f"file:{input_path}",
             "-map_metadata",
             "-1",
         ]
@@ -170,9 +170,15 @@ class Segments:
 
         stderr_task: asyncio.Task[None] = asyncio.create_task(_drain_stderr())
 
-        LOG.debug(f"Streaming '{file}' segment '{self.index}'. ffmpeg: {' '.join(args)}")
+        LOG.debug(
+            "Streaming segment %s for '%s'.",
+            self.index,
+            file,
+            extra={"file": str(file), "segment_index": self.index, "ffmpeg_args": args},
+        )
 
         try:
+            assert proc.stdout is not None
             while True:
                 chunk: bytes = await proc.stdout.read(1024 * 64)
                 if not chunk:
@@ -191,7 +197,10 @@ class Segments:
             try:
                 await asyncio.wait_for(proc.wait(), timeout=5)
             except TimeoutError:
-                LOG.error("ffmpeg process did not terminate in time. Killing it.")
+                LOG.warning(
+                    "Segment stream for '%s' did not stop ffmpeg in time after the client disconnected; killing it.",
+                    file,
+                )
                 proc.kill()
             raise
         except ConnectionResetError:
@@ -204,7 +213,10 @@ class Segments:
                 try:
                     rc: int = await asyncio.wait_for(proc.wait(), timeout=5)
                 except TimeoutError:
-                    LOG.error("ffmpeg process did not terminate in time after disconnect. Killing it.")
+                    LOG.warning(
+                        "Segment stream for '%s' did not stop ffmpeg in time after the client disconnected; killing it.",
+                        file,
+                    )
                     proc.kill()
                     try:
                         rc = await asyncio.wait_for(proc.wait(), timeout=5)
@@ -215,7 +227,7 @@ class Segments:
                 try:
                     rc = await asyncio.wait_for(proc.wait(), timeout=30)
                 except TimeoutError:
-                    LOG.error("ffmpeg process wait timed out. Killing it.")
+                    LOG.warning("Segment stream for '%s' did not stop ffmpeg in time; killing the process.", file)
                     proc.kill()
                     try:
                         rc = await asyncio.wait_for(proc.wait(), timeout=5)
@@ -229,33 +241,47 @@ class Segments:
         return (wrote_any, rc, client_disconnected_local, stderr_buf.decode("utf-8", errors="ignore").strip())
 
     async def stream(self, file: Path, resp: web.StreamResponse) -> None:
-        codec: str = Segments._cached_vcodec if Segments._cache_initialized else self.vcodec
+        codec: str = self.vcodec
+        if Segments._cache_initialized and Segments._cached_vcodec:
+            codec = Segments._cached_vcodec
         if not codec or codec not in SUPPORTED_CODECS:
             codec: str = select_encoder(self.vcodec or "")
 
-        LOG.debug(f"Selected video codec '{codec}' for segment streaming.")
+        LOG.debug("Selected video codec '%s' for segment streaming.", codec, extra={"codec": codec})
 
         if Segments._cached_vcodec and Segments._cache_initialized:
             codecs: list[str] = [Segments._cached_vcodec]
         else:
             codecs: list[str] = [codec, *list(encoder_fallback_chain(codec))]
 
-        for s_codec in codecs:
-            if s_codec in self.attempted:
-                continue
+        stream_input = self._make_stream_input(file)
+        try:
+            for s_codec in codecs:
+                if s_codec in self.attempted:
+                    continue
 
-            ffmpeg_args: list[str] = await self.build_ffmpeg_args(file, s_codec)
-            _, rc, client_disconnected, stderr_text = await self._run(resp, file, ffmpeg_args)
+                ffmpeg_args: list[str] = await self.build_ffmpeg_args(file, s_codec, stream_input=stream_input)
+                _, rc, client_disconnected, stderr_text = await self._run(resp, file, ffmpeg_args)
 
-            if 0 == rc:
-                Segments._cached_vcodec = s_codec
-                Segments._cache_initialized = True
-                return
+                if 0 == rc:
+                    Segments._cached_vcodec = s_codec
+                    Segments._cache_initialized = True
+                    return
 
-            if client_disconnected:
-                return
+                if client_disconnected:
+                    return
 
-            if 0 != rc:
-                err: str = stderr_text[:500] if stderr_text else "no error output"
-                LOG.warning(f"transcoding has failed (cmd={ffmpeg_args}) (rc={rc}): {err}. Trying fallbacks.")
-                self.attempted.add(s_codec)
+                if 0 != rc:
+                    err: str = stderr_text[:500] if stderr_text else "no error output"
+                    LOG.warning(
+                        "Retrying segment %s for '%s' because hardware encoder '%s' failed: %s.",
+                        self.index,
+                        file,
+                        s_codec,
+                        err,
+                        extra={"ffmpeg_args": ffmpeg_args, "returncode": rc, "stderr": err, "codec": s_codec},
+                    )
+                    self.attempted.add(s_codec)
+        finally:
+            if stream_input.is_symlink():
+                stream_input.unlink()

app/features/streaming/library/subtitle.py

@@ -1,14 +1,45 @@
-import logging
+from __future__ import annotations
+
+from dataclasses import dataclass
 from pathlib import Path
+from typing import Any
 
 import anyio
 import pysubs2
 from pysubs2.formats.substation import SubstationFormat
 from pysubs2.time import ms_to_times
 
-from app.library.Utils import ALLOWED_SUBS_EXTENSIONS
+from app.library.log import get_logger
+from app.library.Utils import ALLOWED_SUBS_EXTENSIONS, get_file_sidecar
 
-LOG: logging.Logger = logging.getLogger("player.subtitle")
+LOG = get_logger()
+
+SOURCE_FORMATS: tuple[str, ...] = ("vtt", "srt", "ass")
+DELIVERY_FORMATS: dict[str, str] = {
+    "vtt": "vtt",
+    "srt": "vtt",
+    "ass": "ass",
+}
+RENDERERS: dict[str, str] = {
+    "vtt": "native",
+    "srt": "native",
+    "ass": "assjs",
+}
+MEDIA_TYPES: dict[str, str] = {
+    "vtt": "text/vtt; charset=UTF-8",
+    "ass": "text/x-ssa; charset=UTF-8",
+}
+TEXT_ENCODINGS: tuple[str, ...] = ("utf-8-sig", "utf-16", "utf-16-le", "utf-16-be", "cp1252")
+
+
+@dataclass(frozen=True, slots=True)
+class SubtitleTrack:
+    file: Path
+    lang: str
+    name: str
+    source_format: str
+    delivery_format: str
+    renderer: str
 
 
 def ms_to_timestamp(ms: int) -> str:
@@ -18,18 +49,51 @@ def ms_to_timestamp(ms: int) -> str:
     return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}"
 
 
-SubstationFormat.ms_to_timestamp = ms_to_timestamp
+_substation_format: Any = SubstationFormat
+_substation_format.ms_to_timestamp = ms_to_timestamp
 
 
 class Subtitle:
+    @staticmethod
+    def normalize_format(source_format: str) -> str | None:
+        fmt = source_format.strip().lower().removeprefix(".")
+        if fmt not in SOURCE_FORMATS:
+            return None
+
+        return fmt
+
+    async def read_text(self, file: Path) -> str:
+        async with await anyio.open_file(file, "rb") as f:
+            subtitle_bytes = await f.read()
+
+        return self.decode_bytes(subtitle_bytes)
+
+    @staticmethod
+    def decode_bytes(subtitle_bytes: bytes) -> str:
+        for encoding in TEXT_ENCODINGS:
+            try:
+                return subtitle_bytes.decode(encoding)
+            except UnicodeDecodeError:
+                continue
+
+        return subtitle_bytes.decode("utf-8", errors="replace")
+
+    def media_type(self, file: Path) -> str:
+        fmt = self.normalize_format(file.suffix)
+        if fmt is None:
+            msg = f"File '{file}' subtitle type is not supported."
+            raise Exception(msg)
+
+        return MEDIA_TYPES[DELIVERY_FORMATS[fmt]]
+
     async def make(self, file: Path) -> str:
-        if file.suffix not in ALLOWED_SUBS_EXTENSIONS:
+        fmt = self.normalize_format(file.suffix)
+        if fmt is None or file.suffix not in ALLOWED_SUBS_EXTENSIONS:
             msg: str = f"File '{file}' subtitle type is not supported."
             raise Exception(msg)
 
-        if file.suffix == ".vtt":
-            async with await anyio.open_file(file) as f:
-                return await f.read()
+        if fmt == "vtt":
+            return await self.read_text(file)
 
         subs: pysubs2.SSAFile = pysubs2.load(path=str(file))
 
@@ -47,3 +111,45 @@ class Subtitle:
             pass
 
         return subs.to_string("vtt")
+
+    async def make_delivery(self, file: Path) -> tuple[str, str]:
+        fmt = self.normalize_format(file.suffix)
+        if fmt is None or file.suffix not in ALLOWED_SUBS_EXTENSIONS:
+            msg: str = f"File '{file}' subtitle type is not supported."
+            raise Exception(msg)
+
+        if fmt == "ass":
+            return await self.read_text(file), self.media_type(file)
+
+        return await self.make(file), self.media_type(file)
+
+
+def get_subtitle_tracks(file: Path) -> list[SubtitleTrack]:
+    sidecars = get_file_sidecar(file).get("subtitle", [])
+    indexed_tracks: list[tuple[int, SubtitleTrack]] = []
+
+    for index, item in enumerate(sidecars):
+        track_file = item.get("file")
+        if not isinstance(track_file, Path):
+            continue
+
+        fmt = Subtitle.normalize_format(track_file.suffix)
+        if fmt is None:
+            continue
+
+        indexed_tracks.append(
+            (
+                index,
+                SubtitleTrack(
+                    file=track_file,
+                    lang=str(item.get("lang") or "und"),
+                    name=str(item.get("name") or track_file.name),
+                    source_format=fmt,
+                    delivery_format=DELIVERY_FORMATS[fmt],
+                    renderer=RENDERERS[fmt],
+                ),
+            )
+        )
+
+    indexed_tracks.sort(key=lambda item: (SOURCE_FORMATS.index(item[1].source_format), item[0]))
+    return [track for _, track in indexed_tracks]

app/features/streaming/library/thumbnail.py

@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import subprocess
+from pathlib import Path
+
+from app.features.streaming.library.ffprobe import ffprobe
+from app.library.cache import Cache
+from app.library.config import Config
+from app.library.log import get_logger
+from app.library.Utils import FILES_TYPE, get_file_sidecar
+
+LOG = get_logger()
+
+IMAGE_TYPES: tuple[str, ...] = (".jpg", ".jpeg", ".png", ".webp")
+FOLDER_IMAGE_ORDER: tuple[str, ...] = ("thumbnail", "poster", "artwork", "cover", "fanart")
+THUMBNAIL_SAMPLE_FRAMES = 200
+THUMBNAIL_DEFAULT_SEEK_SECONDS = 3.0
+THUMBNAIL_MIN_SEEK_SECONDS = 1.0
+THUMBNAIL_MAX_SEEK_SECONDS = 15.0
+THUMBNAIL_END_MARGIN_SECONDS = 0.1
+THUMBNAIL_MISS_TTL = 3600.0
+
+_LOCK = asyncio.Lock()
+_IN_PROCESS: dict[str, asyncio.Task[Path | None]] = {}
+
+
+_SEM: asyncio.Semaphore | None = None
+_SEM_LIMIT: int | None = None
+
+
+def _get_semaphore() -> asyncio.Semaphore:
+    global _SEM, _SEM_LIMIT  # noqa: PLW0603
+
+    limit: int = max(1, int(Config.get_instance().thumb_concurrency))
+    if _SEM is None or _SEM_LIMIT != limit:
+        _SEM = asyncio.Semaphore(limit)
+        _SEM_LIMIT = limit
+        LOG.info("Configured thumbnail generation to run %s job(s) at a time.", limit, extra={"limit": limit})
+
+    return _SEM
+
+
+def _is_same_stem_image(media_file: Path, image_file: Path) -> bool:
+    if image_file.parent != media_file.parent:
+        return False
+
+    for image_type in FILES_TYPE:
+        if "image" != image_type.get("type"):
+            continue
+
+        rx = image_type.get("rx")
+        if rx and rx.search(image_file.name):
+            break
+    else:
+        return False
+
+    return image_file.stem == media_file.stem
+
+
+def pick_local_thumb(media_file: Path) -> Path | None:
+    sidecar: dict[str, list[dict[str, str]]] = get_file_sidecar(media_file)
+    images: list[dict[str, str]] = sidecar.get("image", [])
+    local_images: list[Path] = [Path(item["file"]) for item in images if isinstance(item.get("file"), Path)]
+
+    for image_file in local_images:
+        if image_file.suffix.lower() in IMAGE_TYPES and _is_same_stem_image(media_file, image_file):
+            return image_file
+
+    by_name: dict[str, Path] = {
+        image_file.stem.lower(): image_file for image_file in local_images if image_file.suffix.lower() in IMAGE_TYPES
+    }
+    for name in FOLDER_IMAGE_ORDER:
+        if image_file := by_name.get(name):
+            return image_file
+
+    return None
+
+
+def _seek_seconds(ff_info) -> float | None:
+    duration: float = 0.0
+    try:
+        duration = float(ff_info.metadata.get("duration") or 0.0)
+    except (TypeError, ValueError):
+        duration = 0.0
+
+    if duration <= 0:
+        return THUMBNAIL_DEFAULT_SEEK_SECONDS
+
+    if duration <= THUMBNAIL_END_MARGIN_SECONDS:
+        return None
+
+    return min(
+        max(duration * 0.10, THUMBNAIL_MIN_SEEK_SECONDS),
+        THUMBNAIL_MAX_SEEK_SECONDS,
+        duration - THUMBNAIL_END_MARGIN_SECONDS,
+    )
+
+
+def _build_ffmpeg_args(media_file: Path, output_file: Path, *, seek_seconds: float | None) -> list[str]:
+    args: list[str] = [
+        "ffmpeg",
+        "-nostdin",
+        "-hide_banner",
+        "-loglevel",
+        "error",
+        "-y",
+    ]
+
+    if seek_seconds is not None and seek_seconds > 0:
+        args.extend(["-ss", f"{seek_seconds:.3f}"])
+
+    args.extend(
+        [
+            "-i",
+            str(media_file),
+            "-vf",
+            f"thumbnail={THUMBNAIL_SAMPLE_FRAMES},scale=1280:-1:force_original_aspect_ratio=decrease",
+            "-frames:v",
+            "1",
+            "-q:v",
+            "3",
+            "-an",
+            str(output_file),
+        ]
+    )
+    return args
+
+
+async def _run_ffmpeg(media_file: Path, output_file: Path) -> Path | None:
+    ff_info = await ffprobe(media_file)
+    if not ff_info.has_video():
+        LOG.debug(
+            "Skipping thumbnail generation for '%s' because no video stream exists.",
+            media_file,
+            extra={"media_file": str(media_file)},
+        )
+        return None
+
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+    temp_file = output_file.with_suffix(".tmp.jpg")
+
+    if temp_file.exists():
+        temp_file.unlink(missing_ok=True)
+
+    seek_seconds: float | None = _seek_seconds(ff_info)
+    attempts: list[float | None] = [seek_seconds]
+    if seek_seconds is not None and seek_seconds > 0:
+        attempts.append(None)
+
+    sem = _get_semaphore()
+    if sem.locked():
+        limit = _SEM_LIMIT or 1
+        LOG.debug(
+            "Waiting for a thumbnail generation slot for '%s' (limit=%s).",
+            media_file,
+            limit,
+            extra={"media_file": str(media_file), "limit": limit},
+        )
+
+    async with sem:
+        last_error: str = "ffmpeg produced an empty thumbnail file"
+        for idx, attempt_seek in enumerate(attempts, start=1):
+            temp_file.unlink(missing_ok=True)
+            args: list[str] = _build_ffmpeg_args(media_file, temp_file, seek_seconds=attempt_seek)
+            LOG.debug(
+                "Generating thumbnail for '%s'. attempt=%s/%s seek=%s",
+                media_file,
+                idx,
+                len(attempts),
+                "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
+                extra={
+                    "media_file": str(media_file),
+                    "attempt": idx,
+                    "attempt_count": len(attempts),
+                    "seek": "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
+                },
+            )
+
+            try:
+                proc = await asyncio.create_subprocess_exec(
+                    *args,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                    creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
+                )
+            except FileNotFoundError as exc:
+                msg = "ffmpeg not found."
+                raise OSError(msg) from exc
+
+            stdout, stderr = await proc.communicate()
+            if 0 == proc.returncode and temp_file.exists() and temp_file.stat().st_size > 0:
+                temp_file.replace(output_file)
+                LOG.info(
+                    "Generated thumbnail '%s' for '%s'.",
+                    output_file,
+                    media_file,
+                    extra={"media_file": str(media_file), "output_file": str(output_file)},
+                )
+                return output_file
+
+            if 0 != proc.returncode:
+                last_error: str = (
+                    f"ffmpeg thumbnail generation failed (rc={proc.returncode}).\n"
+                    f"stdout:\n{stdout.decode('utf-8', errors='replace')}\n"
+                    f"stderr:\n{stderr.decode('utf-8', errors='replace')}"
+                )
+            else:
+                last_error: str = "ffmpeg produced an empty thumbnail file"
+
+            LOG.debug(
+                "Thumbnail generation attempt failed for '%s'. seek=%s",
+                media_file,
+                "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
+                extra={
+                    "media_file": str(media_file),
+                    "seek": "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
+                },
+            )
+
+    temp_file.unlink(missing_ok=True)
+    raise OSError(last_error)
+
+
+async def ensure_thumb(media_file: Path, cache_root: Path, item_id: str | None = None) -> Path | None:
+    config: Config = Config.get_instance()
+    cache: Cache = Cache.get_instance()
+    if bool(config.thumb_generate) is not True:
+        return None
+
+    sidecar: bool = bool(config.thumb_sidecar)
+    if not sidecar and not item_id:
+        msg = "item_id is required when thumbnail sidecar is disabled."
+        raise ValueError(msg)
+
+    thumb_id: str = str(media_file.resolve()) if sidecar else str(item_id)
+    cache_file: Path = media_file.with_name(f"{media_file.name}.jpg") if sidecar else cache_root / f"{item_id}.jpg"
+
+    miss_key: str = f"thumbnail-miss:{thumb_id}"
+
+    if cache_file.exists() and cache_file.stat().st_size > 0:
+        cache.delete(miss_key)
+        return cache_file
+
+    if cache.has(miss_key):
+        return None
+
+    async with _LOCK:
+        if cache_file.exists() and cache_file.stat().st_size > 0:
+            cache.delete(miss_key)
+            return cache_file
+
+        if cache.has(miss_key):
+            return None
+
+        task = _IN_PROCESS.get(thumb_id)
+        if task is not None and not task.done():
+            LOG.debug("Waiting for thumbnail generation for '%s'.", media_file, extra={"media_file": str(media_file)})
+        else:
+            task = asyncio.create_task(_run_ffmpeg(media_file, cache_file), name=f"thumb-{item_id or media_file.stem}")
+            _IN_PROCESS[thumb_id] = task
+            LOG.debug(
+                "Starting thumbnail generation for '%s' -> '%s'.",
+                media_file,
+                cache_file,
+                extra={"media_file": str(media_file), "cache_file": str(cache_file)},
+            )
+
+    try:
+        result: Path | None = await task
+        if result is None:
+            cache.set(miss_key, value=True, ttl=THUMBNAIL_MISS_TTL)
+        else:
+            cache.delete(miss_key)
+        return result
+    except OSError:
+        cache.set(miss_key, value=True, ttl=THUMBNAIL_MISS_TTL)
+        raise
+    finally:
+        async with _LOCK:
+            current = _IN_PROCESS.get(thumb_id)
+            if current is task and task.done():
+                _IN_PROCESS.pop(thumb_id, None)

app/features/streaming/router.py

@@ -1,21 +1,22 @@
-import logging
 import time
 from datetime import UTC, datetime
 from pathlib import Path
 
 from aiohttp import web
 from aiohttp.web import Request, Response
+from aiohttp.web_response import StreamResponse
 
 from app.features.streaming.library.m3u8 import M3u8
 from app.features.streaming.library.playlist import Playlist
 from app.features.streaming.library.segments import Segments
-from app.features.streaming.library.subtitle import Subtitle
+from app.features.streaming.library.subtitle import Subtitle, get_subtitle_tracks
 from app.features.streaming.types import StreamingError
 from app.library.config import Config
+from app.library.log import get_logger
 from app.library.router import route
 from app.library.Utils import get_file
 
-LOG: logging.Logger = logging.getLogger("streaming")
+LOG = get_logger()
 
 
 @route("GET", "api/player/playlist/{file:.*}.m3u8", "playlist_create")
@@ -32,7 +33,7 @@ async def playlist_create(request: Request, config: Config, app: web.Application
         Response: The response object.
 
     """
-    file: str = request.match_info.get("file")
+    file: str | None = request.match_info.get("file")
 
     if not file:
         return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@@ -83,8 +84,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
         Response: The response object.
 
     """
-    file: str = request.match_info.get("file")
-    mode: str = request.match_info.get("mode")
+    file: str | None = request.match_info.get("file")
+    mode: str | None = request.match_info.get("mode")
 
     if mode not in ["video", "subtitle"]:
         return web.json_response(
@@ -94,13 +95,14 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
     if not file:
         return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
 
-    duration = request.query.get("duration", None)
+    duration: float | None = None
+    duration_arg = request.query.get("duration", None)
 
     if "subtitle" in mode:
-        if not duration:
+        if not duration_arg:
             return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
 
-        duration = float(duration)
+        duration = float(duration_arg)
 
     base_path: str = config.base_path.rstrip("/")
 
@@ -124,11 +126,19 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
             return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
 
         if "subtitle" in mode:
+            if duration is None:
+                return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
             text = await cls.make_subtitle(file=realFile, duration=duration)
         else:
             text = await cls.make_stream(file=realFile)
     except StreamingError as e:
-        LOG.exception(e)
+        LOG.exception(
+            "Failed to create %s streaming playlist for '%s': %s.",
+            mode,
+            file,
+            e,
+            extra={"route": "streaming.playlist", "file_path": file, "mode": mode, "exception_type": type(e).__name__},
+        )
         return web.json_response(data={"error": str(e)}, status=web.HTTPNotFound.status_code)
 
     return web.Response(
@@ -143,7 +153,7 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
 
 
 @route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream")
-async def segments_stream(request: Request, config: Config, app: web.Application) -> Response:
+async def segments_stream(request: Request, config: Config, app: web.Application) -> StreamResponse:
     """
     Get the segments.
 
@@ -156,9 +166,9 @@ async def segments_stream(request: Request, config: Config, app: web.Application
         Response: The response object.
 
     """
-    file: str = request.match_info.get("file")
-    segment: int = request.match_info.get("segment")
-    sd: int = request.query.get("sd")
+    file: str | None = request.match_info.get("file")
+    segment: str | None = request.match_info.get("segment")
+    sd: str | None = request.query.get("sd")
     vc: int = int(request.query.get("vc", 0))
     ac: int = int(request.query.get("ac", 0))
 
@@ -235,7 +245,7 @@ async def subtitles_get(request: Request, config: Config, app: web.Application)
         Response: The response object.
 
     """
-    file: str = request.match_info.get("file")
+    file: str | None = request.match_info.get("file")
 
     if not file:
         return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
@@ -277,3 +287,134 @@ async def subtitles_get(request: Request, config: Config, app: web.Application)
         },
         status=web.HTTPOk.status_code,
     )
+
+
+@route("GET", "api/player/subtitles/manifest/{file:.*}", "subtitles_manifest_get")
+async def subtitles_manifest_get(request: Request, config: Config, app: web.Application) -> Response:
+    """
+    Get subtitle track metadata for a media file.
+
+    Args:
+        request (Request): The request object.
+        config (Config): The configuration instance.
+        app (web.Application): The aiohttp application instance.
+
+    Returns:
+        Response: The response object.
+
+    """
+    file: str | None = request.match_info.get("file")
+
+    if not file:
+        return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
+
+    realFile, status = get_file(download_path=config.download_path, file=file)
+    if web.HTTPFound.status_code == status:
+        return Response(
+            status=status,
+            headers={
+                "Location": str(
+                    app.router["subtitles_manifest_get"].url_for(
+                        file=str(realFile).replace(config.download_path, "").strip("/")
+                    )
+                ),
+            },
+        )
+
+    if web.HTTPNotFound.status_code == status:
+        return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
+
+    tracks = [
+        {
+            "lang": track.lang,
+            "name": track.name,
+            "source_format": track.source_format,
+            "delivery_format": track.delivery_format,
+            "renderer": track.renderer,
+            "url": str(
+                app.router["subtitles_track_get"].url_for(
+                    source_format=track.source_format,
+                    file=str(track.file).replace(config.download_path, "").strip("/"),
+                )
+            ),
+        }
+        for track in get_subtitle_tracks(realFile)
+    ]
+
+    return web.json_response(data={"subtitles": tracks}, status=web.HTTPOk.status_code)
+
+
+@route("GET", "api/player/subtitles/{source_format}/{file:.*}", "subtitles_track_get")
+async def subtitles_track_get(request: Request, config: Config, app: web.Application) -> Response:
+    """
+    Get a subtitle file using its preferred delivery format.
+
+    Args:
+        request (Request): The request object.
+        config (Config): The configuration instance.
+        app (web.Application): The aiohttp application instance.
+
+    Returns:
+        Response: The response object.
+
+    """
+    file: str | None = request.match_info.get("file")
+    source_format: str | None = request.match_info.get("source_format")
+
+    if not file:
+        return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
+
+    fmt = Subtitle.normalize_format(source_format or "")
+    if fmt is None:
+        return web.json_response(
+            data={"error": "Only vtt, srt, and ass subtitle formats are supported."},
+            status=web.HTTPBadRequest.status_code,
+        )
+
+    realFile, status = get_file(download_path=config.download_path, file=file)
+    if web.HTTPFound.status_code == status:
+        return Response(
+            status=status,
+            headers={
+                "Location": str(
+                    app.router["subtitles_track_get"].url_for(
+                        source_format=fmt,
+                        file=str(realFile).replace(config.download_path, "").strip("/"),
+                    )
+                ),
+            },
+        )
+
+    if web.HTTPNotFound.status_code == status:
+        return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
+
+    if Subtitle.normalize_format(realFile.suffix) != fmt:
+        return web.json_response(
+            data={"error": f"Subtitle file '{file}' does not match requested source format '{fmt}'."},
+            status=web.HTTPBadRequest.status_code,
+        )
+
+    mtime = realFile.stat().st_mtime
+
+    if request.if_modified_since and request.if_modified_since.timestamp() == mtime:
+        lastMod = time.strftime("%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple())
+        return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod})
+
+    body, content_type = await Subtitle().make_delivery(file=realFile)
+    return web.Response(
+        body=body,
+        headers={
+            "Content-Type": content_type,
+            "X-Accel-Buffering": "no",
+            "Access-Control-Allow-Origin": "*",
+            "Pragma": "public",
+            "Cache-Control": f"public, max-age={time.time() + 31536000}",
+            "Last-Modified": time.strftime(
+                "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple()
+            ),
+            "Expires": time.strftime(
+                "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple()
+            ),
+        },
+        status=web.HTTPOk.status_code,
+    )

app/features/streaming/tests/test_ffprobe.py

@@ -1,16 +1,17 @@
-import tempfile
 from pathlib import Path
 from unittest.mock import AsyncMock, patch
 
 import pytest
 
+from app.tests.helpers import make_test_temp_dir
+
 
 class TestFFProbe:
     """Test the ffprobe module functionality."""
 
     def setup_method(self):
         """Set up test files."""
-        self.temp_dir = tempfile.mkdtemp()
+        self.temp_dir = str(make_test_temp_dir("ffprobe"))
         self.test_file = Path(self.temp_dir) / "test_video.mp4"
         self.test_file.touch()
 
@@ -83,13 +84,11 @@ class TestFFProbe:
 
                 # First call should execute the function
                 result1 = await ffprobe(str(self.test_file))
-                assert result1 is not None
                 assert isinstance(result1.metadata, dict)
                 first_call_count = call_count
 
                 # Second call with same argument should use cached result
                 result2 = await ffprobe(str(self.test_file))
-                assert result2 is not None
                 assert isinstance(result2.metadata, dict)
 
                 assert call_count == first_call_count, (
@@ -118,7 +117,7 @@ class TestFFProbe:
 
                 # Test with Path object
                 result = await ffprobe(self.test_file)
-                assert result is not None
+                assert result.metadata == {"duration": "10.0"}
 
     def test_ffprobe_result_properties(self):
         """Test FFProbeResult object properties."""

app/features/streaming/tests/test_m3u8.py

@@ -64,7 +64,7 @@ async def test_make_stream_basic_ok_codecs(tmp_path: Path, monkeypatch: pytest.M
 
 
 @pytest.mark.asyncio
-async def test_make_stream_transcode_flags_and_remainder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_make_stream_transcode_flags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     base = tmp_path / "dl"
     base.mkdir()
     media = base / "v.mp4"
@@ -107,7 +107,7 @@ async def test_make_stream_transcode_flags_and_remainder(tmp_path: Path, monkeyp
 
 
 @pytest.mark.asyncio
-async def test_make_stream_raises_without_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_make_stream_no_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     base = tmp_path / "dl"
     base.mkdir()
     media = base / "v.mp4"

app/features/streaming/tests/test_playlist.py

@@ -99,7 +99,7 @@ async def test_make_playlist_with_subtitles(tmp_path: Path, monkeypatch: pytest.
 
 
 @pytest.mark.asyncio
-async def test_make_playlist_raises_without_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_make_playlist_no_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     base = tmp_path / "downloads"
     base.mkdir()
     media = base / "file.mp4"

app/features/streaming/tests/test_segments.py

@@ -1,13 +1,13 @@
 import asyncio
-import hashlib
 import logging
-import tempfile
 from pathlib import Path
 from typing import Any
 
 import pytest
+from aiohttp import web
 
 from app.features.streaming.library.segments import Segments
+from app.tests.helpers import get_test_system_temp_root
 
 
 class DummyFF:
@@ -22,6 +22,10 @@ class DummyFF:
         return self._a
 
 
+def _ffmpeg_input_path(args: list[str]) -> Path:
+    return Path(args[args.index("-i") + 1].removeprefix("file:"))
+
+
 @pytest.mark.asyncio
 async def test_build_ffmpeg_args_video_and_audio(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     # Create a dummy media file
@@ -53,8 +57,7 @@ async def test_build_ffmpeg_args_video_and_audio(tmp_path: Path, monkeypatch: py
     assert captured_args, "ffmpeg was not invoked"
     args = captured_args[0]
 
-    # Compute expected symlink path used by Segments
-    tmpFile = Path(tempfile.gettempdir()).joinpath(f"ytptube_stream.{hashlib.sha256(str(media).encode()).hexdigest()}")
+    tmp_file = _ffmpeg_input_path(args)
 
     # Start time is duration * index with 6 decimals for non-zero index
     assert "-ss" in args
@@ -64,7 +67,9 @@ async def test_build_ffmpeg_args_video_and_audio(tmp_path: Path, monkeypatch: py
     assert args[args.index("-t") + 1] == f"{5.5:.6f}"
     # Input uses file:<symlink>
     assert "-i" in args
-    assert args[args.index("-i") + 1] == f"file:{tmpFile}"
+    assert tmp_file.parent == get_test_system_temp_root()
+    assert tmp_file.name.startswith("ytptube_stream.")
+    assert not tmp_file.exists()
     # Includes video and audio mapping and codecs
     assert "-map" in args
     assert "0:v:0" in args
@@ -144,18 +149,18 @@ class _FakeProc:
         self.killed = True
 
 
-class _FakeResp:
-    def __init__(self, fail_with: Exception | None = None) -> None:
+class _FakeResp(web.StreamResponse):
+    def __init__(self, fail_with: BaseException | None = None) -> None:
         self.data: bytearray = bytearray()
         self.eof = False
         self._exc = fail_with
 
-    async def write(self, data: bytes) -> None:
+    async def write(self, data: bytes | bytearray | memoryview, *_args: Any, **_kwargs: Any) -> None:
         if self._exc:
             raise self._exc
         self.data.extend(data)
 
-    async def write_eof(self) -> None:
+    async def write_eof(self, *_args: Any, **_kwargs: Any) -> None:
         self.eof = True
 
 
@@ -168,7 +173,7 @@ class _FakeProcFail(_FakeProc):
 
 
 @pytest.mark.asyncio
-async def test_build_ffmpeg_args_no_dri_falls_back_to_software(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_build_ffmpeg_args_no_dri(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     media = tmp_path / "file.mp4"
     media.write_bytes(b"data")
 
@@ -215,7 +220,7 @@ async def test_build_ffmpeg_args_no_dri_falls_back_to_software(tmp_path: Path, m
 
 
 @pytest.mark.asyncio
-async def test_stream_gpu_failure_falls_back_to_software(
+async def test_stream_gpu_fallback(
     monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture
 ) -> None:
     async def fake_ffprobe(_file: Path):
@@ -251,14 +256,14 @@ async def test_stream_gpu_failure_falls_back_to_software(
     # Encourage GPU preference
     seg.vcodec = ""  # empty -> try GPUs first
     resp = _FakeResp()
-    with caplog.at_level(logging.WARNING, logger="player.segments"):
+    with caplog.at_level(logging.WARNING, logger="ytptube"):
         await seg.stream(tmp_path / "file.mp4", resp)
 
     # Ensure fallback path streamed data
     assert len(resp.data) > 0
     # Ensure we logged the reason for GPU failure (message text may vary)
     assert any(
-        ("Hardware encoder failed" in r.message) or ("transcoding has failed" in r.message) for r in caplog.records
+        ("hardware encoder" in r.message.lower()) or ("transcoding has failed" in r.message) for r in caplog.records
     )
     assert any("nvenc failure" in r.message for r in caplog.records)
     # Verify second invocation switched codec to a safe fallback (software)
@@ -338,6 +343,9 @@ async def test_stream_gpu_fallback_switches_codec(monkeypatch: pytest.MonkeyPatc
 
 @pytest.mark.asyncio
 async def test_stream_normal_flow(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    media = tmp_path / "file.mp4"
+    media.write_bytes(b"data")
+
     async def fake_ffprobe(_file: Path):
         return DummyFF(v=True, a=True)
 
@@ -346,16 +354,21 @@ async def test_stream_normal_flow(monkeypatch: pytest.MonkeyPatch, tmp_path: Pat
     # Process that yields two chunks and then EOF
     proc = _FakeProc([b"abc", b"def", b""])
 
-    async def fake_create_subprocess_exec(*_args: Any, **_kwargs: Any):
+    captured_args: list[list[str]] = []
+
+    async def fake_create_subprocess_exec(*args: Any, **_kwargs: Any):
+        captured_args.append(list(args[1:]))
         return proc
 
     monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec)
 
     seg = Segments(download_path=str(tmp_path), index=0, duration=1.0, vconvert=True, aconvert=True)
     resp = _FakeResp()
-    await seg.stream(tmp_path / "file.mp4", resp)
+    await seg.stream(media, resp)
 
     assert bytes(resp.data) == b"abcdef"
+    assert not _ffmpeg_input_path(captured_args[0]).exists()
+    assert media.exists()
     # EOF behavior may differ; don't require True
 
 
@@ -368,7 +381,10 @@ async def test_stream_client_reset(monkeypatch: pytest.MonkeyPatch, tmp_path: Pa
 
     proc = _FakeProc([b"abc", b"def"])  # will attempt to write and fail
 
-    async def fake_create_subprocess_exec(*_args: Any, **_kwargs: Any):
+    captured_args: list[list[str]] = []
+
+    async def fake_create_subprocess_exec(*args: Any, **_kwargs: Any):
+        captured_args.append(list(args[1:]))
         return proc
 
     monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec)
@@ -380,6 +396,7 @@ async def test_stream_client_reset(monkeypatch: pytest.MonkeyPatch, tmp_path: Pa
 
     # Should not write EOF due to client disconnect
     assert resp.eof is False
+    assert not _ffmpeg_input_path(captured_args[0]).exists()
 
 
 @pytest.mark.asyncio

app/features/streaming/tests/test_subtitle.py

@@ -4,7 +4,7 @@ from unittest.mock import patch
 
 import pytest
 
-from app.features.streaming.library.subtitle import Subtitle, ms_to_timestamp
+from app.features.streaming.library.subtitle import Subtitle, get_subtitle_tracks, ms_to_timestamp
 
 
 class TestMsToTimestamp:
@@ -24,12 +24,12 @@ class TestMsToTimestamp:
 
 @pytest.mark.asyncio
 async def test_make_unsupported_extension(tmp_path: Path) -> None:
-    srt = tmp_path / "sub.txt"
-    srt.write_text("not a subtitle")
+    file = tmp_path / "sub.txt"
+    file.write_text("not a subtitle")
 
-    sub = Subtitle()
+    subtitle = Subtitle()
     with pytest.raises(Exception, match="subtitle type is not supported"):
-        await sub.make(srt)
+        await subtitle.make(file)
 
 
 @pytest.mark.asyncio
@@ -38,11 +38,51 @@ async def test_make_vtt_reads_file(tmp_path: Path) -> None:
     content = "WEBVTT\n\n00:00:00.00 --> 00:00:01.00\nHello"
     vtt.write_text(content)
 
-    sub = Subtitle()
-    out = await sub.make(vtt)
+    subtitle = Subtitle()
+    out = await subtitle.make(vtt)
     assert out == content
 
 
+@pytest.mark.asyncio
+async def test_make_delivery_ass(tmp_path: Path) -> None:
+    ass = tmp_path / "file.ass"
+    content = "[Script Info]\nTitle: Demo\n"
+    ass.write_text(content, encoding="utf-8")
+
+    subtitle = Subtitle()
+    out, media_type = await subtitle.make_delivery(ass)
+    assert out == content
+    assert media_type == "text/x-ssa; charset=UTF-8"
+
+
+def test_tracks_order(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    media = tmp_path / "video.mkv"
+    media.write_text("x", encoding="utf-8")
+    ass_file = tmp_path / "video.ass"
+    ass_file.write_text("ass", encoding="utf-8")
+    vtt_file = tmp_path / "video.vtt"
+    vtt_file.write_text("WEBVTT\n\n", encoding="utf-8")
+    srt_file = tmp_path / "video.en.srt"
+    srt_file.write_text("1\n00:00:00,000 --> 00:00:01,000\nHello\n", encoding="utf-8")
+
+    monkeypatch.setattr(
+        "app.features.streaming.library.subtitle.get_file_sidecar",
+        lambda _file: {
+            "subtitle": [
+                {"file": ass_file, "lang": "en", "name": "ASS"},
+                {"file": srt_file, "lang": "en", "name": "SRT"},
+                {"file": vtt_file, "lang": "en", "name": "VTT"},
+            ]
+        },
+    )
+
+    tracks = get_subtitle_tracks(media)
+
+    assert [track.source_format for track in tracks] == ["vtt", "srt", "ass"]
+    assert [track.delivery_format for track in tracks] == ["vtt", "vtt", "ass"]
+    assert [track.renderer for track in tracks] == ["native", "native", "assjs"]
+
+
 class _DummySubs:
     def __init__(self, events):
         self.events = events
@@ -61,13 +101,13 @@ async def test_make_no_events_raises(tmp_path: Path) -> None:
 
     with patch("app.features.streaming.library.subtitle.pysubs2.load") as mock_load:
         mock_load.return_value = _DummySubs(events=[])
-        sub = Subtitle()
+        subtitle = Subtitle()
         with pytest.raises(Exception, match="No subtitle events were found"):
-            await sub.make(srt)
+            await subtitle.make(srt)
 
 
 @pytest.mark.asyncio
-async def test_make_single_event_returns_vtt(tmp_path: Path) -> None:
+async def test_make_single_vtt(tmp_path: Path) -> None:
     srt = tmp_path / "sub.ass"
     srt.write_text("dummy")
 
@@ -75,14 +115,14 @@ async def test_make_single_event_returns_vtt(tmp_path: Path) -> None:
     d = _DummySubs(events=[single])
 
     with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
-        sub = Subtitle()
-        out = await sub.make(srt)
+        subtitle = Subtitle()
+        out = await subtitle.make(srt)
         assert out == "OUT"
         assert d.snapshot == [1000], "Snapshot should contain the single event"
 
 
 @pytest.mark.asyncio
-async def test_make_two_events_pop_first_when_ends_equal(tmp_path: Path) -> None:
+async def test_make_two_events_same_end(tmp_path: Path) -> None:
     srt = tmp_path / "sub.srt"
     srt.write_text("dummy")
 
@@ -91,14 +131,14 @@ async def test_make_two_events_pop_first_when_ends_equal(tmp_path: Path) -> None
     d = _DummySubs(events=[e1, e2])
 
     with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
-        sub = Subtitle()
-        out = await sub.make(srt)
+        subtitle = Subtitle()
+        out = await subtitle.make(srt)
         assert out == "OUT"
         assert d.snapshot == [5000], "Since ends are equal, first should be popped => only last remains"
 
 
 @pytest.mark.asyncio
-async def test_make_two_events_no_pop_when_different(tmp_path: Path) -> None:
+async def test_make_two_events_keep_both(tmp_path: Path) -> None:
     srt = tmp_path / "sub.srt"
     srt.write_text("dummy")
 
@@ -107,7 +147,7 @@ async def test_make_two_events_no_pop_when_different(tmp_path: Path) -> None:
     d = _DummySubs(events=[e1, e2])
 
     with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
-        sub = Subtitle()
-        out = await sub.make(srt)
+        subtitle = Subtitle()
+        out = await subtitle.make(srt)
         assert out == "OUT"
         assert d.snapshot == [5000, 6000], "Both remain since ends differ"

app/features/streaming/tests/test_subtitle_routes.py

@@ -0,0 +1,132 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from aiohttp import web
+from aiohttp.test_utils import make_mocked_request
+
+from app.features.streaming import router
+from app.library.config import Config
+
+
+class _DummyRoute:
+    def __init__(self, pattern: str) -> None:
+        self.pattern = pattern
+
+    def url_for(self, **params: str) -> str:
+        path = self.pattern
+        for key, value in params.items():
+            path = path.replace(f"{{{key}:.*}}", value)
+            path = path.replace(f"{{{key}}}", value)
+        return path
+
+
+class _DummyRouter(dict[str, _DummyRoute]):
+    def __init__(self) -> None:
+        super().__init__(
+            {
+                "subtitles_manifest_get": _DummyRoute("/api/player/subtitles/manifest/{file:.*}"),
+                "subtitles_track_get": _DummyRoute("/api/player/subtitles/{source_format}/{file:.*}"),
+                "subtitles_get": _DummyRoute("/api/player/subtitle/{file:.*}.vtt"),
+            }
+        )
+
+
+def _make_request(path: str, *, match_info: dict[str, str]) -> web.Request:
+    return make_mocked_request("GET", path, app=SimpleNamespace(router=_DummyRouter()), match_info=match_info)
+
+
+@pytest.mark.asyncio
+async def test_subtitles_manifest_order(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    config = Config.get_instance()
+    config.download_path = str(tmp_path)
+
+    media = tmp_path / "video.mp4"
+    media.write_text("x", encoding="utf-8")
+
+    monkeypatch.setattr(
+        "app.features.streaming.router.get_file",
+        lambda **_kwargs: (media, web.HTTPOk.status_code),
+    )
+    monkeypatch.setattr(
+        "app.features.streaming.router.get_subtitle_tracks",
+        lambda _file: [
+            SimpleNamespace(
+                lang="en",
+                name="English VTT",
+                source_format="vtt",
+                delivery_format="vtt",
+                renderer="native",
+                file=tmp_path / "video.vtt",
+            ),
+            SimpleNamespace(
+                lang="en",
+                name="English ASS",
+                source_format="ass",
+                delivery_format="ass",
+                renderer="assjs",
+                file=tmp_path / "video.ass",
+            ),
+        ],
+    )
+
+    req = _make_request(
+        "/api/player/subtitles/manifest/video.mp4",
+        match_info={"file": "video.mp4"},
+    )
+    response = await router.subtitles_manifest_get(req, config, req.app)
+
+    assert response.status == web.HTTPOk.status_code
+    body = response.text
+    assert '"source_format": "vtt"' in body
+    assert '"renderer": "native"' in body
+    assert '"source_format": "ass"' in body
+    assert '"renderer": "assjs"' in body
+    assert body.index('"source_format": "vtt"') < body.index('"source_format": "ass"')
+
+
+@pytest.mark.asyncio
+async def test_subtitles_track_get_ass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    config = Config.get_instance()
+    config.download_path = str(tmp_path)
+
+    subtitle = tmp_path / "video.ass"
+    subtitle.write_text("[Script Info]\nTitle: Demo\n", encoding="utf-8")
+
+    monkeypatch.setattr(
+        "app.features.streaming.router.get_file",
+        lambda **_kwargs: (subtitle, web.HTTPOk.status_code),
+    )
+
+    req = _make_request(
+        "/api/player/subtitles/ass/video.ass",
+        match_info={"source_format": "ass", "file": "video.ass"},
+    )
+    response = await router.subtitles_track_get(req, config, req.app)
+
+    assert response.status == web.HTTPOk.status_code
+    assert response.text == "[Script Info]\nTitle: Demo\n"
+    assert response.headers["Content-Type"] == "text/x-ssa; charset=UTF-8"
+
+
+@pytest.mark.asyncio
+async def test_subtitles_track_bad_format(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    config = Config.get_instance()
+    config.download_path = str(tmp_path)
+
+    subtitle = tmp_path / "video.ass"
+    subtitle.write_text("[Script Info]\nTitle: Demo\n", encoding="utf-8")
+
+    monkeypatch.setattr(
+        "app.features.streaming.router.get_file",
+        lambda **_kwargs: (subtitle, web.HTTPOk.status_code),
+    )
+
+    req = _make_request(
+        "/api/player/subtitles/vtt/video.ass",
+        match_info={"source_format": "vtt", "file": "video.ass"},
+    )
+    response = await router.subtitles_track_get(req, config, req.app)
+
+    assert response.status == web.HTTPBadRequest.status_code
+    assert b"does not match requested source format" in response.body

app/features/streaming/tests/test_thumbnail.py

@@ -0,0 +1,358 @@
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+from typing import Any
+from unittest.mock import AsyncMock
+
+import pytest
+
+from app.tests.helpers import temporary_test_dir
+
+
+@pytest.mark.asyncio
+async def test_pick_same_stem() -> None:
+    from app.features.streaming.library.thumbnail import pick_local_thumb
+
+    with temporary_test_dir("thumb-local") as temp_dir:
+        media = temp_dir / "video.mp4"
+        poster = temp_dir / "poster.jpg"
+        thumb = temp_dir / "video.jpg"
+
+        media.write_text("video")
+        poster.write_text("poster")
+        thumb.write_text("thumb")
+
+        assert pick_local_thumb(media) == thumb
+
+
+@pytest.mark.asyncio
+async def test_singleflight(monkeypatch: pytest.MonkeyPatch) -> None:
+    from app.features.streaming.library import thumbnail
+
+    thumbnail._IN_PROCESS.clear()
+    thumbnail.Cache.get_instance().clear()
+
+    with temporary_test_dir("thumb-singleflight") as temp_dir:
+        media = temp_dir / "video.mp4"
+        cache_root = temp_dir / "cache"
+        media.write_text("video")
+        item_id = "item-1"
+
+        calls = {"count": 0}
+
+        async def fake_run_ffmpeg(_file: Path, output_file: Path) -> Path:
+            calls["count"] += 1
+            await asyncio.sleep(0.01)
+            output_file.parent.mkdir(parents=True, exist_ok=True)
+            output_file.write_text("image")
+            return output_file
+
+        monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
+
+        first, second = await asyncio.gather(
+            thumbnail.ensure_thumb(media, cache_root, item_id=item_id),
+            thumbnail.ensure_thumb(media, cache_root, item_id=item_id),
+        )
+
+        assert first == second
+        assert first is not None
+        assert calls["count"] == 1
+        assert thumbnail._IN_PROCESS == {}
+
+
+@pytest.mark.asyncio
+async def test_cache_hit(monkeypatch: pytest.MonkeyPatch) -> None:
+    from app.features.streaming.library import thumbnail
+
+    thumbnail._IN_PROCESS.clear()
+    thumbnail.Cache.get_instance().clear()
+
+    with temporary_test_dir("thumb-cache") as temp_dir:
+        media = temp_dir / "video.mp4"
+        cache_root = temp_dir / "cache"
+        media.write_text("video")
+        item_id = "item-1"
+
+        cache_file = cache_root / f"{item_id}.jpg"
+        cache_file.parent.mkdir(parents=True, exist_ok=True)
+        cache_file.write_text("image")
+
+        async def fake_run_ffmpeg(_file: Path, _output_file: Path) -> Path:
+            raise AssertionError("ffmpeg should not run when cache exists")
+
+        monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
+
+        result = await thumbnail.ensure_thumb(media, cache_root, item_id=item_id)
+
+        assert result == cache_file
+
+
+@pytest.mark.asyncio
+async def test_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
+    from app.features.streaming.library import thumbnail
+
+    thumbnail._IN_PROCESS.clear()
+    thumbnail.Cache.get_instance().clear()
+
+    with temporary_test_dir("thumb-disabled") as temp_dir:
+        media = temp_dir / "video.mp4"
+        media.write_text("video")
+
+        monkeypatch.setattr(
+            thumbnail.Config,
+            "get_instance",
+            staticmethod(lambda: type("Cfg", (), {"thumb_generate": False, "thumb_sidecar": False})()),
+        )
+
+        result = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id="item-1")
+
+        assert result is None
+
+
+@pytest.mark.asyncio
+async def test_sidecar_mode(monkeypatch: pytest.MonkeyPatch) -> None:
+    from app.features.streaming.library import thumbnail
+
+    thumbnail._IN_PROCESS.clear()
+    thumbnail._SEM = None
+    thumbnail._SEM_LIMIT = None
+    thumbnail.Cache.get_instance().clear()
+
+    with temporary_test_dir("thumb-sidecar") as temp_dir:
+        media = temp_dir / "video.mp4"
+        media.write_text("video")
+
+        sidecar = media.with_name(f"{media.name}.jpg")
+
+        monkeypatch.setattr(
+            thumbnail.Config,
+            "get_instance",
+            staticmethod(
+                lambda: type(
+                    "Cfg",
+                    (),
+                    {"thumb_generate": True, "thumb_sidecar": True, "thumb_concurrency": 1},
+                )()
+            ),
+        )
+
+        async def fake_run_ffmpeg(_file: Path, out_file: Path) -> Path:
+            out_file.write_text("image")
+            return out_file
+
+        monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
+
+        result = await thumbnail.ensure_thumb(media, temp_dir / "cache")
+
+        assert result == sidecar
+        assert sidecar.exists()
+
+
+@pytest.mark.asyncio
+async def test_miss_cache(monkeypatch: pytest.MonkeyPatch) -> None:
+    from app.features.streaming.library import thumbnail
+
+    thumbnail._IN_PROCESS.clear()
+    thumbnail.Cache.get_instance().clear()
+
+    with temporary_test_dir("thumb-miss") as temp_dir:
+        media = temp_dir / "video.mp4"
+        media.write_text("video")
+        item_id = "item-1"
+
+        monkeypatch.setattr(
+            thumbnail.Config,
+            "get_instance",
+            staticmethod(
+                lambda: type(
+                    "Cfg",
+                    (),
+                    {"thumb_generate": True, "thumb_sidecar": False, "thumb_concurrency": 1},
+                )()
+            ),
+        )
+
+        calls = {"count": 0}
+
+        async def fake_run_ffmpeg(_file: Path, _out_file: Path) -> Path | None:
+            calls["count"] += 1
+            return None
+
+        monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
+
+        first = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id=item_id)
+        second = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id=item_id)
+
+        assert first is None
+        assert second is None
+        assert calls["count"] == 1
+
+
+def test_sem_limit(monkeypatch: pytest.MonkeyPatch) -> None:
+    from app.features.streaming.library import thumbnail
+
+    thumbnail._SEM = None
+    thumbnail._SEM_LIMIT = None
+
+    monkeypatch.setattr(
+        thumbnail.Config,
+        "get_instance",
+        staticmethod(
+            lambda: type("Cfg", (), {"thumb_concurrency": 3, "thumb_generate": True, "thumb_sidecar": False})()
+        ),
+    )
+
+    sem = thumbnail._get_semaphore()
+
+    assert sem._value == 3
+    assert thumbnail._SEM_LIMIT == 3
+
+
+def test_seek_bounds() -> None:
+    from app.features.streaming.library.ffprobe import FFProbeResult
+    from app.features.streaming.library.thumbnail import _seek_seconds
+
+    ff_info = FFProbeResult()
+    ff_info.metadata = {"duration": "120.0"}
+    assert _seek_seconds(ff_info) == 12.0
+
+    ff_info.metadata = {"duration": "2.0"}
+    assert _seek_seconds(ff_info) == 1.0
+
+    ff_info.metadata = {"duration": "0.05"}
+    assert _seek_seconds(ff_info) is None
+
+    ff_info.metadata = {}
+    assert _seek_seconds(ff_info) == 3.0
+
+
+@pytest.mark.asyncio
+async def test_cache_name_uses_item_id(monkeypatch: pytest.MonkeyPatch) -> None:
+    from app.features.streaming.library import thumbnail
+
+    thumbnail._IN_PROCESS.clear()
+    thumbnail.Cache.get_instance().clear()
+
+    with temporary_test_dir("thumb-id-name") as temp_dir:
+        media = temp_dir / "video.mp4"
+        cache_root = temp_dir / "cache"
+        media.write_text("video")
+
+        async def fake_run_ffmpeg(_file: Path, output_file: Path) -> Path:
+            output_file.parent.mkdir(parents=True, exist_ok=True)
+            output_file.write_text("image")
+            return output_file
+
+        monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
+
+        result = await thumbnail.ensure_thumb(media, cache_root, item_id="item-1")
+
+        assert result == cache_root / "item-1.jpg"
+
+
+@pytest.mark.asyncio
+async def test_retry_no_seek(monkeypatch: pytest.MonkeyPatch) -> None:
+    from app.features.streaming.library.ffprobe import FFProbeResult
+    from app.features.streaming.library import thumbnail
+
+    with temporary_test_dir("thumb-retry") as temp_dir:
+        media = temp_dir / "video.mp4"
+        output = temp_dir / ".jpg"
+        media.write_text("video")
+
+        ff_info: Any = FFProbeResult()
+        ff_info.metadata = {"duration": "120.0"}
+        ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
+
+        monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))
+
+        calls: list[list[str]] = []
+
+        class DummyProc:
+            def __init__(self, attempt: int, out_path: Path) -> None:
+                self.returncode = 1 if attempt == 1 else 0
+                self._out_path = out_path
+
+            async def communicate(self) -> tuple[bytes, bytes]:
+                if self.returncode == 0:
+                    self._out_path.write_text("image")
+                    return b"", b""
+                return b"", b"seek failed"
+
+        async def fake_create_subprocess_exec(*args, **kwargs):
+            del kwargs
+            calls.append([str(arg) for arg in args])
+            return DummyProc(len(calls), Path(str(args[-1])))
+
+        monkeypatch.setattr(thumbnail.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
+
+        result = await thumbnail._run_ffmpeg(media, output)
+
+        assert result == output
+        assert len(calls) == 2
+        assert "-ss" in calls[0]
+        assert calls[0][calls[0].index("-ss") + 1] == "12.000"
+        assert "-ss" not in calls[1]
+        assert calls[0][calls[0].index("-vf") + 1] == "thumbnail=200,scale=1280:-1:force_original_aspect_ratio=decrease"
+        assert calls[0][-1].endswith(".tmp.jpg")
+
+
+@pytest.mark.asyncio
+async def test_limit_wait(monkeypatch: pytest.MonkeyPatch) -> None:
+    from app.features.streaming.library import thumbnail
+    from app.features.streaming.library.ffprobe import FFProbeResult
+
+    thumbnail._SEM = None
+    thumbnail._SEM_LIMIT = None
+    thumbnail._IN_PROCESS.clear()
+    monkeypatch.setattr(
+        thumbnail.Config,
+        "get_instance",
+        staticmethod(
+            lambda: type("Cfg", (), {"thumb_concurrency": 1, "thumb_generate": True, "thumb_sidecar": False})()
+        ),
+    )
+
+    with temporary_test_dir("thumb-limit") as temp_dir:
+        media1 = temp_dir / "video1.mp4"
+        media2 = temp_dir / "video2.mp4"
+        out1 = temp_dir / "out1.jpg"
+        out2 = temp_dir / "out2.jpg"
+        media1.write_text("video")
+        media2.write_text("video")
+
+        ff_info: Any = FFProbeResult()
+        ff_info.metadata = {"duration": "60.0"}
+        ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
+        monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))
+
+        active = {"count": 0, "max": 0}
+
+        class DummyProc:
+            def __init__(self, out_path: Path) -> None:
+                self.returncode = 0
+                self._out_path = out_path
+
+            async def communicate(self) -> tuple[bytes, bytes]:
+                active["count"] += 1
+                active["max"] = max(active["max"], active["count"])
+                await asyncio.sleep(0.01)
+                self._out_path.write_text("image")
+                active["count"] -= 1
+                return b"", b""
+
+        async def fake_create_subprocess_exec(*args, **kwargs):
+            del kwargs
+            return DummyProc(Path(str(args[-1])))
+
+        monkeypatch.setattr(thumbnail.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
+
+        first, second = await asyncio.gather(
+            thumbnail._run_ffmpeg(media1, out1),
+            thumbnail._run_ffmpeg(media2, out2),
+        )
+
+        assert first == out1
+        assert second == out2
+        assert active["max"] == 1

app/features/tasks/definitions/handlers/_base_handler.py

@@ -1,4 +1,3 @@
-# flake8: noqa: ARG004
 from typing import TYPE_CHECKING, Any
 
 from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult
@@ -12,6 +11,7 @@ if TYPE_CHECKING:
 class BaseHandler:
     @staticmethod
     async def can_handle(task: HandleTask) -> bool:
+        _ = task
         return False
 
     @staticmethod
@@ -24,6 +24,7 @@ class BaseHandler:
 
     @staticmethod
     def parse(url: str) -> Any | None:
+        _ = url
         return None
 
     @staticmethod

app/features/tasks/definitions/handlers/generic.py

@@ -6,12 +6,12 @@ import asyncio
 import fnmatch
 import hashlib
 import json
-import logging
 import re
 from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any
 from urllib.parse import urljoin
 
+import httpx
 import jmespath
 from parsel import Selector
 
@@ -25,16 +25,16 @@ from app.features.ytdlp.utils import get_archive_id
 from app.library.cache import Cache
 from app.library.config import Config
 from app.library.httpx_client import Globals, build_request_headers, get_async_client, resolve_curl_transport
+from app.library.log import get_logger
 
 from ._base_handler import BaseHandler
 
 if TYPE_CHECKING:
     from pathlib import Path
 
-    import httpx
     from parsel.selector import SelectorList
 
-LOG: logging.Logger = logging.getLogger("handlers.generic")
+LOG = get_logger()
 CACHE: Cache = Cache()
 
 
@@ -64,12 +64,15 @@ class GenericTaskHandler(BaseHandler):
             from app.features.tasks.definitions.utils import model_to_schema
 
             repo = TaskDefinitionsRepository.get_instance()
-            models = await repo.list()
+            models = await repo.all()
 
             cls._definitions = [model_to_schema(model) for model in models]
             return cls._definitions
         except Exception as exc:
-            LOG.error(f"Failed to load task definitions from database: {exc}")
+            LOG.exception(
+                "Failed to load generic task definitions.",
+                extra={"error": str(exc), "exception_type": type(exc).__name__},
+            )
             return []
 
     @classmethod
@@ -102,7 +105,15 @@ class GenericTaskHandler(BaseHandler):
                     if pattern_str and re.match(pattern_str, url):
                         return definition
             except Exception as exc:
-                LOG.error(f"Error while matching definition '{definition.name}': {exc}")
+                LOG.exception(
+                    "Failed to match a generic task definition.",
+                    extra={
+                        "definition": definition.name,
+                        "url": url,
+                        "error": str(exc),
+                        "exception_type": type(exc).__name__,
+                    },
+                )
 
         return None
 
@@ -120,13 +131,18 @@ class GenericTaskHandler(BaseHandler):
         """
         definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
         if definition:
-            LOG.debug(f"'{task.name}': Matched generic task definition '{definition.name}'.")
+            LOG.debug(
+                "Task '%s' matched a generic task definition.",
+                task.name,
+                extra={"task_name": task.name, "url": task.url, "definition": definition.name},
+            )
             return True
 
         return False
 
     @staticmethod
-    async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:  # noqa: ARG004
+    async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
+        _ = config
         definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
         if not definition:
             return TaskFailure(message="No generic task definition matched the provided URL.")
@@ -134,14 +150,34 @@ class GenericTaskHandler(BaseHandler):
         ytdlp_opts: dict[str, Any] = task.get_ytdlp_opts().get_all()
         target_url: str = definition.definition.request.url or task.url
 
-        LOG.debug(f"{task.name!r}: Fetching '{target_url}' using engine '{definition.definition.engine.type}'.")
+        LOG.debug(
+            "Fetching content for task '%s'.",
+            task.name,
+            extra={
+                "task_name": task.name,
+                "definition": definition.name,
+                "url": target_url,
+                "engine": definition.definition.engine.type,
+            },
+        )
 
         try:
             body_text, json_data = await GenericTaskHandler._fetch_content(
                 url=target_url, definition=definition, ytdlp_opts=ytdlp_opts
             )
+        except httpx.HTTPError as exc:
+            return TaskFailure(message="Failed to fetch target URL.", error=str(exc))
         except Exception as exc:
-            LOG.exception(exc)
+            LOG.exception(
+                "Failed to fetch content for task '%s'.",
+                task.name,
+                extra={
+                    "task_name": task.name,
+                    "definition": definition.name,
+                    "url": target_url,
+                    "exception_type": type(exc).__name__,
+                },
+            )
             return TaskFailure(message="Failed to fetch target URL.", error=str(exc))
 
         if "json" == definition.definition.response.type and json_data is None:
@@ -179,7 +215,9 @@ class GenericTaskHandler(BaseHandler):
                         continue
                 else:
                     LOG.warning(
-                        f"[{definition.name}]: '{task.name}': Unable to generate static archive id for '{url}' in feed. Doing real request to fetch yt-dlp archive id."
+                        "Task '%s' could not generate a static archive ID. Fetching it with yt-dlp.",
+                        task.name,
+                        extra={"definition": definition.name, "task_name": task.name, "url": url},
                     )
 
                     (info, _) = await fetch_info(
@@ -187,18 +225,23 @@ class GenericTaskHandler(BaseHandler):
                         url=url,
                         no_archive=True,
                         no_log=True,
+                        budget_sleep=True,
                     )
 
                     if not info:
                         LOG.error(
-                            f"[{definition.name}]: '{task.name}': Failed to extract info for URL '{url}' to generate archive ID. Skipping."
+                            "Task '%s' failed to extract info to generate an archive ID. Skipping item.",
+                            task.name,
+                            extra={"definition": definition.name, "task_name": task.name, "url": url},
                         )
                         CACHE.set(cache_key, None)
                         continue
 
                     if not info.get("id") or not info.get("extractor_key"):
                         LOG.error(
-                            f"[{definition.name}]: '{task.name}': Incomplete info extracted for URL '{url}' to generate archive ID. Skipping."
+                            "Task '%s' returned incomplete info while generating an archive ID. Skipping item.",
+                            task.name,
+                            extra={"definition": definition.name, "task_name": task.name, "url": url},
                         )
                         CACHE.set(cache_key, None)
                         continue
@@ -294,7 +337,17 @@ class GenericTaskHandler(BaseHandler):
             try:
                 json_data: dict[str, Any] = response.json()
             except Exception as exc:
-                LOG.error(f"Failed to decode JSON response from '{url}': {exc}")
+                LOG.exception(
+                    "Task definition '%s' returned invalid JSON for '%s'.",
+                    definition.name,
+                    url,
+                    extra={
+                        "definition": definition.name,
+                        "url": url,
+                        "error": str(exc),
+                        "exception_type": type(exc).__name__,
+                    },
+                )
                 return response.text, None
 
             return response.text, json_data
@@ -324,7 +377,11 @@ class GenericTaskHandler(BaseHandler):
             from selenium.webdriver.support import expected_conditions as EC
             from selenium.webdriver.support.ui import WebDriverWait
         except ImportError as exc:
-            LOG.error(f"Selenium engine requested but selenium is not installed: {exc!s}.")
+            LOG.exception(
+                "Task definition '%s' requested Selenium, but Selenium is not installed.",
+                definition.name,
+                extra={"definition": definition.name, "error": str(exc), "exception_type": type(exc).__name__},
+            )
             return (None, None)
 
         options_map: dict[str, Any] = definition.definition.engine.options
@@ -337,7 +394,12 @@ class GenericTaskHandler(BaseHandler):
         browser: str = str(options_map.get("browser", "chrome")).lower()
 
         if "chrome" != browser:
-            LOG.error(f"Unsupported selenium browser '{browser}'. Only 'chrome' is supported.")
+            LOG.error(
+                "Task definition '%s' requested unsupported Selenium browser '%s'.",
+                definition.name,
+                browser,
+                extra={"definition": definition.name, "browser": browser},
+            )
             return (None, None)
 
         arguments: list[str] | str = options_map.get("arguments", ["--headless", "--disable-gpu"])
@@ -419,7 +481,9 @@ class GenericTaskHandler(BaseHandler):
 
         link_values: list[str] = extracted.get("link", [])
         if not link_values:
-            LOG.debug(f"Definition '{definition.name}' produced no link values.")
+            LOG.debug(
+                "Definition '%s' produced no link values.", definition.name, extra={"definition": definition.name}
+            )
             return []
 
         total_items: int = len(link_values)
@@ -454,7 +518,11 @@ class GenericTaskHandler(BaseHandler):
         base_url: str,
     ) -> list[dict[str, str]]:
         if json_data is None:
-            LOG.debug(f"Definition '{definition.name}' expects JSON but no data was parsed.")
+            LOG.debug(
+                "Definition '%s' expects JSON but no data was parsed.",
+                definition.name,
+                extra={"definition": definition.name},
+            )
             return []
 
         if definition.definition.parse.get("items"):
@@ -493,7 +561,11 @@ class GenericTaskHandler(BaseHandler):
         container_type = container.get("type", "css")
         container_selector = container.get("selector") or container.get("expression") or ""
         if not container_selector:
-            LOG.error(f"Container missing selector/expression. Definition '{definition.name}'.")
+            LOG.error(
+                "Task definition '%s' is missing an item container selector.",
+                definition.name,
+                extra={"definition": definition.name},
+            )
             return []
         container_fields = container.get("fields", {})
 
@@ -547,7 +619,11 @@ class GenericTaskHandler(BaseHandler):
         container_fields = container.get("fields", {})
 
         if "jsonpath" != container_type:
-            LOG.error(f"JSON response requires container selector type 'jsonpath'. Definition '{definition.name}'.")
+            LOG.error(
+                "JSON response requires container selector type 'jsonpath'. Definition '%s'.",
+                definition.name,
+                extra={"definition": definition.name, "container_type": container_type},
+            )
             return []
 
         nodes: Any = GenericTaskHandler._json_search(json_data, container_selector)
@@ -603,7 +679,16 @@ class GenericTaskHandler(BaseHandler):
             try:
                 pattern: re.Pattern[str] = re.compile(rule.expression, re.MULTILINE | re.DOTALL)
             except re.error as exc:
-                LOG.error(f"Invalid regex expression '{rule.expression}': {exc}")
+                LOG.exception(
+                    "Invalid regex expression '%s'.",
+                    rule.expression,
+                    extra={
+                        "field": field,
+                        "expression": rule.expression,
+                        "error": str(exc),
+                        "exception_type": type(exc).__name__,
+                    },
+                )
                 return values
 
             for match in pattern.finditer(target):
@@ -614,7 +699,12 @@ class GenericTaskHandler(BaseHandler):
 
             return values
 
-        LOG.error(f"Unsupported extraction type '{rule.type}' for JSON data in field '{field}'.")
+        LOG.error(
+            "Unsupported extraction type '%s' for JSON data in field '%s'.",
+            rule.type,
+            field,
+            extra={"field": field, "rule_type": rule.type},
+        )
         return values
 
     @staticmethod
@@ -622,7 +712,11 @@ class GenericTaskHandler(BaseHandler):
         try:
             return jmespath.search(expression, data)
         except Exception as exc:
-            LOG.error(f"JSONPath search failed for expression '{expression}': {exc}")
+            LOG.exception(
+                "JSONPath search failed for expression '%s'.",
+                expression,
+                extra={"expression": expression, "error": str(exc), "exception_type": type(exc).__name__},
+            )
             return None
 
     @staticmethod
@@ -657,7 +751,16 @@ class GenericTaskHandler(BaseHandler):
             try:
                 pattern: re.Pattern[str] = re.compile(rule.expression, re.MULTILINE | re.DOTALL)
             except re.error as exc:
-                LOG.error(f"Invalid regex expression '{rule.expression}': {exc}")
+                LOG.exception(
+                    "Invalid regex expression '%s'.",
+                    rule.expression,
+                    extra={
+                        "field": field,
+                        "expression": rule.expression,
+                        "error": str(exc),
+                        "exception_type": type(exc).__name__,
+                    },
+                )
                 return values
 
             for match in pattern.finditer(html):
@@ -669,7 +772,7 @@ class GenericTaskHandler(BaseHandler):
             return values
 
         if "jsonpath" == rule.type:
-            LOG.error("Extraction type 'jsonpath' is only valid for JSON responses.")
+            LOG.error("Field '%s' uses 'jsonpath' on a non-JSON response.", field, extra={"field": field})
             return values
 
         selection: SelectorList[Selector] = (
@@ -701,7 +804,12 @@ class GenericTaskHandler(BaseHandler):
             try:
                 return match.group(attribute)
             except (IndexError, KeyError):
-                LOG.debug(f"Regex group '{attribute}' not found in pattern '{match.re.pattern}'.")
+                LOG.debug(
+                    "Regex group '%s' not found in pattern '%s'.",
+                    attribute,
+                    match.re.pattern,
+                    extra={"attribute": attribute, "pattern": match.re.pattern},
+                )
                 return None
 
         if match.groupdict():

app/features/tasks/definitions/handlers/rss.py

@@ -1,20 +1,23 @@
 import hashlib
-import logging
 import re
 from typing import TYPE_CHECKING, Any
 from xml.etree.ElementTree import Element
 
+import httpx
+
 from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
 from app.features.ytdlp.extractor import fetch_info
 from app.features.ytdlp.utils import get_archive_id
 from app.library.cache import Cache
+from app.library.config import Config
+from app.library.log import get_logger
 
 from ._base_handler import BaseHandler
 
 if TYPE_CHECKING:
     from xml.etree.ElementTree import Element
 
-LOG: logging.Logger = logging.getLogger("handlers.rss")
+LOG = get_logger()
 CACHE: Cache = Cache()
 
 
@@ -26,7 +29,11 @@ class RssGenericHandler(BaseHandler):
 
     @staticmethod
     async def can_handle(task: HandleTask) -> bool:
-        LOG.debug(f"'{task.name}': Checking if task URL is parsable RSS feed: {task.url}")
+        LOG.debug(
+            "Checking if task '%s' uses a parsable RSS feed.",
+            task.name,
+            extra={"task_name": task.name, "url": task.url},
+        )
         return RssGenericHandler.parse(task.url) is not None
 
     @staticmethod
@@ -50,7 +57,11 @@ class RssGenericHandler(BaseHandler):
         from defusedxml.ElementTree import fromstring
 
         feed_url: str = parsed["url"]
-        LOG.debug(f"'{task.name}': Fetching RSS/Atom feed from {feed_url}")
+        LOG.debug(
+            "Fetching RSS/Atom feed for task '%s'.",
+            task.name,
+            extra={"task_name": task.name, "feed_url": feed_url},
+        )
 
         response = await RssGenericHandler.request(url=feed_url, ytdlp_opts=params)
         response.raise_for_status()
@@ -71,7 +82,12 @@ class RssGenericHandler(BaseHandler):
         # Try to parse as Atom feed first
         entries = root.findall("atom:entry", ns)
         if entries:
-            LOG.debug(f"'{task.name}': Detected Atom feed format with {len(entries)} entries")
+            LOG.debug(
+                "'%s': Detected Atom feed format with %s entries",
+                task.name,
+                len(entries),
+                extra={"task_name": task.name, "feed_url": feed_url, "entry_count": len(entries)},
+            )
             for entry in entries:
                 link_elem: Element | None = entry.find("atom:link[@rel='alternate']", ns)
                 if link_elem is None:
@@ -82,7 +98,11 @@ class RssGenericHandler(BaseHandler):
                     url = link_elem.get("href", "")
 
                 if not url:
-                    LOG.warning(f"'{task.name}': Atom entry missing URL. Skipping.")
+                    LOG.warning(
+                        "'%s': Atom entry missing URL. Skipping.",
+                        task.name,
+                        extra={"task_name": task.name, "feed_url": feed_url},
+                    )
                     continue
 
                 title_elem: Element | None = entry.find("atom:title", ns)
@@ -96,7 +116,12 @@ class RssGenericHandler(BaseHandler):
         else:
             # Try to parse as RSS feed
             rss_items = root.findall(".//item")
-            LOG.debug(f"'{task.name}': Detected RSS feed format with {len(rss_items)} items")
+            LOG.debug(
+                "'%s': Detected RSS feed format with %s items",
+                task.name,
+                len(rss_items),
+                extra={"task_name": task.name, "feed_url": feed_url, "entry_count": len(rss_items)},
+            )
 
             for item in rss_items:
                 # Try different link element names (link, url, media:content)
@@ -117,7 +142,11 @@ class RssGenericHandler(BaseHandler):
                             url = enclosure_elem.get("url", "")
 
                 if not url:
-                    LOG.warning(f"'{task.name}': RSS item missing URL. Skipping.")
+                    LOG.warning(
+                        "'%s': RSS item missing URL. Skipping.",
+                        task.name,
+                        extra={"task_name": task.name, "feed_url": feed_url},
+                    )
                     continue
 
                 title_elem = item.find("title")
@@ -132,12 +161,14 @@ class RssGenericHandler(BaseHandler):
         return feed_url, items, real_count
 
     @staticmethod
-    async def extract(task: HandleTask) -> TaskResult | TaskFailure:
+    async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
+        _ = config
         """
         Extract items from an RSS/Atom feed.
 
         Args:
             task (Task): The task containing the feed URL.
+            config (Config | None): Optional handler configuration.
 
         Returns:
             TaskResult | TaskFailure: Extraction result with parsed items or failure information.
@@ -151,8 +182,19 @@ class RssGenericHandler(BaseHandler):
 
         try:
             feed_url, items, real_count = await RssGenericHandler._get(task, params, parsed)
+        except httpx.HTTPError as exc:
+            return TaskFailure(message="Failed to fetch RSS/Atom feed.", error=str(exc))
         except Exception as exc:
-            LOG.exception(exc)
+            LOG.exception(
+                "Failed to fetch RSS/Atom feed for task '%s'.",
+                task.name,
+                extra={
+                    "task_id": task.id,
+                    "task_name": task.name,
+                    "url": task.url,
+                    "exception_type": type(exc).__name__,
+                },
+            )
             return TaskFailure(message="Failed to fetch RSS/Atom feed.", error=str(exc))
 
         task_items: list[TaskItem] = []
@@ -172,12 +214,17 @@ class RssGenericHandler(BaseHandler):
                 if CACHE.has(cache_key):
                     archive_id = CACHE.get(cache_key)
                     if not archive_id:
-                        LOG.debug(f"'{task.name}': Cached failure for URL '{url}'. Skipping.")
+                        LOG.debug(
+                            "Task '%s' has a cached archive ID lookup failure. Skipping item.",
+                            task.name,
+                            extra={"task_name": task.name, "url": url},
+                        )
                         continue
                 else:
                     LOG.warning(
-                        f"'{task.name}': Unable to generate static archive ID for '{url}' in feed. "
-                        "Doing real request to fetch yt-dlp archive ID."
+                        "Task '%s' could not generate a static archive ID. Fetching it with yt-dlp.",
+                        task.name,
+                        extra={"task_name": task.name, "url": url},
                     )
 
                     (info, _) = await fetch_info(
@@ -185,18 +232,23 @@ class RssGenericHandler(BaseHandler):
                         url=url,
                         no_archive=True,
                         no_log=True,
+                        budget_sleep=True,
                     )
 
                     if not info:
                         LOG.error(
-                            f"'{task.name}': Failed to extract info for URL '{url}' to generate archive ID. Skipping."
+                            "Task '%s' failed to extract info to generate an archive ID. Skipping item.",
+                            task.name,
+                            extra={"task_name": task.name, "url": url},
                         )
                         CACHE.set(cache_key, None)
                         continue
 
                     if not info.get("id") or not info.get("extractor_key"):
                         LOG.error(
-                            f"'{task.name}': Incomplete info extracted for URL '{url}' to generate archive ID. Skipping."
+                            "Task '%s' returned incomplete info while generating an archive ID. Skipping item.",
+                            task.name,
+                            extra={"task_name": task.name, "url": url},
                         )
                         CACHE.set(cache_key, None)
                         continue

app/features/tasks/definitions/handlers/tver.py

@@ -1,12 +1,15 @@
-import logging
 import re
 
+import httpx
+
 from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
 from app.features.ytdlp.utils import get_archive_id
+from app.library.config import Config
+from app.library.log import get_logger
 
 from ._base_handler import BaseHandler
 
-LOG: logging.Logger = logging.getLogger("handlers.tver")
+LOG = get_logger()
 
 
 class TverHandler(BaseHandler):
@@ -22,7 +25,11 @@ class TverHandler(BaseHandler):
 
     @staticmethod
     async def can_handle(task: HandleTask) -> bool:
-        LOG.debug(f"Checking if task '{task.name}' is using parsable Tver series URL: {task.url}")
+        LOG.debug(
+            "Checking if task '%s' uses a parsable Tver series URL.",
+            task.name,
+            extra={"task_name": task.name, "url": task.url},
+        )
         return TverHandler.parse(task.url) is not None
 
     @staticmethod
@@ -53,7 +60,11 @@ class TverHandler(BaseHandler):
 
             return {"platform_uid": platform_uid, "platform_token": platform_token}
         except Exception as exc:
-            LOG.warning(f"Failed to create tver session: {exc}")
+            LOG.warning(
+                "Failed to create Tver session.",
+                extra={"error": str(exc), "exception_type": type(exc).__name__},
+                exc_info=True,
+            )
             return None
 
     @staticmethod
@@ -81,7 +92,11 @@ class TverHandler(BaseHandler):
 
         feed_url = TverHandler.SERIES_API.format(id=series_id)
 
-        LOG.debug(f"Fetching '{task.name}' episodes from tver series {series_id}.")
+        LOG.debug(
+            "Fetching Tver episodes for task '%s'.",
+            task.name,
+            extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
+        )
 
         response = await TverHandler.request(
             url=feed_url,
@@ -103,7 +118,11 @@ class TverHandler(BaseHandler):
         try:
             contents = data.get("result", {}).get("contents", [])
             if not contents:
-                LOG.warning(f"No contents found in tver series response for '{task.name}'.")
+                LOG.warning(
+                    "No contents found in Tver series response for '%s'.",
+                    task.name,
+                    extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
+                )
                 return feed_url, items, has_items
 
             season_block = contents[0] if contents else {}
@@ -114,9 +133,13 @@ class TverHandler(BaseHandler):
                     continue
 
                 content = episode_data.get("content", {})
-                episode_id = content.pop("id")
+                episode_id = content.pop("id", None)
                 if not episode_id:
-                    LOG.warning(f"Episode missing ID in '{task.name}' feed. Skipping.")
+                    LOG.warning(
+                        "Episode missing ID in '%s' feed. Skipping.",
+                        task.name,
+                        extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
+                    )
                     continue
 
                 url = f"https://tver.jp/episodes/{episode_id}"
@@ -127,7 +150,9 @@ class TverHandler(BaseHandler):
                 archive_id = id_dict.get("archive_id")
                 if not archive_id:
                     LOG.warning(
-                        f"Could not compute archive ID for episode '{episode_id}' in '{task.name}' feed. Skipping."
+                        "Task '%s' could not compute an archive ID for an episode. Skipping item.",
+                        task.name,
+                        extra={"task_name": task.name, "series_id": series_id, "episode_id": episode_id, "url": url},
                     )
                     continue
 
@@ -137,12 +162,24 @@ class TverHandler(BaseHandler):
                 )
 
         except Exception as exc:
-            LOG.warning(f"Error parsing tver episodes for '{task.name}': {exc}")
+            LOG.warning(
+                "Failed to parse Tver episodes for task '%s'.",
+                task.name,
+                extra={
+                    "task_id": task.id,
+                    "task_name": task.name,
+                    "series_id": series_id,
+                    "error": str(exc),
+                    "exception_type": type(exc).__name__,
+                },
+                exc_info=True,
+            )
 
         return feed_url, items, has_items
 
     @staticmethod
-    async def extract(task: HandleTask) -> TaskResult | TaskFailure:
+    async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
+        _ = config
         series_id: str | None = TverHandler.parse(task.url)
         if not series_id:
             return TaskFailure(message="Unrecognized Tver series URL.")
@@ -151,8 +188,20 @@ class TverHandler(BaseHandler):
 
         try:
             feed_url, items, has_items = await TverHandler._collect_feed(task, params, series_id)
+        except httpx.HTTPError as exc:
+            return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc))
         except Exception as exc:
-            LOG.exception(exc)
+            LOG.exception(
+                "Failed to fetch Tver feed for task '%s'.",
+                task.name,
+                extra={
+                    "task_id": task.id,
+                    "task_name": task.name,
+                    "url": task.url,
+                    "series_id": series_id,
+                    "exception_type": type(exc).__name__,
+                },
+            )
             return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc))
 
         task_items: list[TaskItem] = []
@@ -161,10 +210,11 @@ class TverHandler(BaseHandler):
             if not (url := entry.get("url")):
                 continue
 
-            archive_id: str = entry.get("archive_id")
-            task_items.append(
-                TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=entry.get("metadata", {}))
-            )
+            archive_id: str | None = entry.get("archive_id")
+            metadata = entry.get("metadata", {})
+            if not isinstance(metadata, dict):
+                metadata = {}
+            task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
 
         return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})
 

app/features/tasks/definitions/handlers/twitch.py

@@ -1,17 +1,20 @@
-import logging
 import re
 from typing import TYPE_CHECKING
 from xml.etree.ElementTree import Element
 
+import httpx
+
 from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
 from app.features.ytdlp.utils import get_archive_id
+from app.library.config import Config
+from app.library.log import get_logger
 
 from ._base_handler import BaseHandler
 
 if TYPE_CHECKING:
     from xml.etree.ElementTree import Element
 
-LOG: logging.Logger = logging.getLogger("handlers.twitch")
+LOG = get_logger()
 
 
 class TwitchHandler(BaseHandler):
@@ -21,7 +24,11 @@ class TwitchHandler(BaseHandler):
 
     @staticmethod
     async def can_handle(task: HandleTask) -> bool:
-        LOG.debug(f"Checking if task '{task.name}' is using parsable Twitch URL: {task.url}")
+        LOG.debug(
+            "Checking if task '%s' uses a parsable Twitch URL.",
+            task.name,
+            extra={"task_name": task.name, "url": task.url},
+        )
         return TwitchHandler.parse(task.url) is not None
 
     @staticmethod
@@ -34,7 +41,7 @@ class TwitchHandler(BaseHandler):
 
         feed_url: str = TwitchHandler.FEED.format(handle=handle_name)
 
-        LOG.debug(f"Fetching '{task.name}' feed.")
+        LOG.debug("Fetching '%s' feed.", task.name, extra={"task_name": task.name, "feed_url": feed_url})
         response = await TwitchHandler.request(url=feed_url, ytdlp_opts=params)
         response.raise_for_status()
 
@@ -46,14 +53,22 @@ class TwitchHandler(BaseHandler):
             link_elem: Element[str] | None = entry.find("link")
             url: str = link_elem.text.strip() if link_elem is not None and link_elem.text else ""
             if not url:
-                LOG.warning(f"Entry in '{task.name}' feed is missing URL. Skipping entry.")
+                LOG.warning(
+                    "Entry in '%s' feed is missing URL. Skipping entry.",
+                    task.name,
+                    extra={"task_name": task.name, "feed_url": feed_url},
+                )
                 continue
 
             match: re.Match[str] | None = re.search(
                 r"^https?://(?:www\.)?twitch\.tv/videos/(?P<id>\d+)(?:[/?].*)?$", url
             )
             if not match:
-                LOG.warning(f"URL in '{task.name}' feed does not look like a VOD link: {url}")
+                LOG.warning(
+                    "Task '%s' produced a feed entry that does not look like a Twitch VOD link. Skipping entry.",
+                    task.name,
+                    extra={"task_name": task.name, "url": url},
+                )
                 continue
 
             vid: str = match.group("id")
@@ -66,7 +81,11 @@ class TwitchHandler(BaseHandler):
             id_dict = get_archive_id(url)
             archive_id: str | None = id_dict.get("archive_id")
             if not archive_id:
-                LOG.warning(f"Could not compute archive ID for video '{vid}' in '{task.name}' feed. Skipping entry.")
+                LOG.warning(
+                    "Task '%s' could not compute an archive ID for a Twitch video. Skipping entry.",
+                    task.name,
+                    extra={"task_name": task.name, "video_id": vid, "url": url},
+                )
                 continue
 
             items.append({"id": vid, "url": url, "title": title, "archive_id": archive_id})
@@ -74,7 +93,8 @@ class TwitchHandler(BaseHandler):
         return feed_url, items, has_items
 
     @staticmethod
-    async def extract(task: HandleTask) -> TaskResult | TaskFailure:
+    async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
+        _ = config
         handle_name: str | None = TwitchHandler.parse(task.url)
         if not handle_name:
             return TaskFailure(message="Unrecognized Twitch channel URL.")
@@ -83,8 +103,19 @@ class TwitchHandler(BaseHandler):
 
         try:
             feed_url, items, has_items = await TwitchHandler._collect_feed(task, params, handle_name)
+        except httpx.HTTPError as exc:
+            return TaskFailure(message="Failed to fetch Twitch feed.", error=str(exc))
         except Exception as exc:
-            LOG.exception(exc)
+            LOG.exception(
+                "Failed to fetch Twitch feed for task '%s'.",
+                task.name,
+                extra={
+                    "task_id": task.id,
+                    "task_name": task.name,
+                    "url": task.url,
+                    "exception_type": type(exc).__name__,
+                },
+            )
             return TaskFailure(message="Failed to fetch Twitch feed.", error=str(exc))
 
         task_items: list[TaskItem] = []
@@ -93,7 +124,7 @@ class TwitchHandler(BaseHandler):
             if not (url := entry.get("url")):
                 continue
 
-            archive_id: str = entry.get("archive_id")
+            archive_id: str | None = entry.get("archive_id")
             task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id))
 
         return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})

app/features/tasks/definitions/handlers/youtube.py

@@ -1,17 +1,20 @@
-import logging
 import re
 from typing import TYPE_CHECKING, Any
 from xml.etree.ElementTree import Element
 
+import httpx
+
 from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
 from app.features.ytdlp.utils import get_archive_id
+from app.library.config import Config
+from app.library.log import get_logger
 
 from ._base_handler import BaseHandler
 
 if TYPE_CHECKING:
     from xml.etree.ElementTree import Element
 
-LOG: logging.Logger = logging.getLogger("handlers.youtube")
+LOG = get_logger()
 
 
 class YoutubeHandler(BaseHandler):
@@ -27,7 +30,11 @@ class YoutubeHandler(BaseHandler):
 
     @staticmethod
     async def can_handle(task: HandleTask) -> bool:
-        LOG.debug(f"'{task.name}': Checking if task URL is parsable YouTube URL: {task.url}")
+        LOG.debug(
+            "Checking if task '%s' uses a parsable YouTube URL.",
+            task.name,
+            extra={"task_name": task.name, "url": task.url},
+        )
         return YoutubeHandler.parse(task.url) is not None
 
     @staticmethod
@@ -47,7 +54,7 @@ class YoutubeHandler(BaseHandler):
         from defusedxml.ElementTree import fromstring
 
         feed_url: str = YoutubeHandler.FEED.format(type=parsed["type"], id=parsed["id"])
-        LOG.debug(f"'{task.name}': Fetching feed.")
+        LOG.debug("'%s': Fetching feed.", task.name, extra={"task_name": task.name, "feed_url": feed_url})
 
         response = await YoutubeHandler.request(url=feed_url, ytdlp_opts=params)
         response.raise_for_status()
@@ -65,7 +72,11 @@ class YoutubeHandler(BaseHandler):
             vid_elem: Element[str] | None = entry.find("yt:videoId", ns)
             vid: str = vid_elem.text if vid_elem is not None and vid_elem.text else ""
             if not vid:
-                LOG.warning(f"'{task.name}': Entry in the feed is missing a video ID. Skipping.")
+                LOG.warning(
+                    "'%s': Entry in the feed is missing a video ID. Skipping.",
+                    task.name,
+                    extra={"task_name": task.name, "feed_url": feed_url},
+                )
                 continue
 
             url: str = f"https://www.youtube.com/watch?v={vid}"
@@ -73,7 +84,11 @@ class YoutubeHandler(BaseHandler):
             id_dict: dict[str, str | None] = get_archive_id(url)
             archive_id: str | None = id_dict.get("archive_id")
             if not archive_id:
-                LOG.warning(f"'{task.name}': Could not compute archive ID for video '{vid}' in feed. Skipping.")
+                LOG.warning(
+                    "Task '%s' could not compute an archive ID for a YouTube video. Skipping item.",
+                    task.name,
+                    extra={"task_name": task.name, "video_id": vid, "url": url},
+                )
                 continue
 
             title_elem: Element[str] | None = entry.find("atom:title", ns)
@@ -89,7 +104,8 @@ class YoutubeHandler(BaseHandler):
         return feed_url, items, real_count
 
     @staticmethod
-    async def extract(task: HandleTask) -> TaskResult | TaskFailure:
+    async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
+        _ = config
         parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
         if not parsed:
             return TaskFailure(message="Unrecognized YouTube channel or playlist URL.")
@@ -98,8 +114,19 @@ class YoutubeHandler(BaseHandler):
 
         try:
             feed_url, items, real_count = await YoutubeHandler._get(task, params, parsed)
+        except httpx.HTTPError as exc:
+            return TaskFailure(message="Failed to fetch YouTube feed.", error=str(exc))
         except Exception as exc:
-            LOG.exception(exc)
+            LOG.exception(
+                "Failed to fetch YouTube feed for task '%s'.",
+                task.name,
+                extra={
+                    "task_id": task.id,
+                    "task_name": task.name,
+                    "url": task.url,
+                    "exception_type": type(exc).__name__,
+                },
+            )
             return TaskFailure(message="Failed to fetch YouTube feed.", error=str(exc))
 
         task_items: list[TaskItem] = []
@@ -108,7 +135,7 @@ class YoutubeHandler(BaseHandler):
             if not (url := entry.get("url")):
                 continue
 
-            archive_id: str = entry.get("archive_id")
+            archive_id: str | None = entry.get("archive_id")
             metadata: dict[str, Any] = {"published": entry.get("published")}
 
             task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))

app/features/tasks/definitions/migration.py

@@ -1,17 +1,17 @@
 from __future__ import annotations
 
 import json
-import logging
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from app.features.core.migration import Migration as FeatureMigration
 from app.library.config import Config
+from app.library.log import get_logger
 
 if TYPE_CHECKING:
     from app.features.tasks.definitions.repository import TaskDefinitionsRepository
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 class Migration(FeatureMigration):
@@ -48,7 +48,11 @@ class Migration(FeatureMigration):
                 await self._repo.create(normalized)
                 inserted += 1
             except Exception as exc:
-                LOG.exception("Failed to insert task definition '%s': %s", normalized.get("name"), exc)
+                LOG.exception(
+                    "Failed to insert task definition '%s'.",
+                    normalized.get("name"),
+                    extra={"definition": normalized.get("name"), "exception_type": type(exc).__name__},
+                )
             finally:
                 await self._move_file(path)
 
@@ -62,13 +66,21 @@ class Migration(FeatureMigration):
         try:
             content = path.read_text(encoding="utf-8")
         except Exception as exc:
-            LOG.error("Failed to read task definition '%s': %s", path, exc)
+            LOG.exception(
+                "Failed to read task definition '%s'.",
+                path,
+                extra={"path": str(path), "exception_type": type(exc).__name__},
+            )
             return None
 
         try:
             payload = json.loads(content)
         except Exception as exc:
-            LOG.error("Failed to parse JSON for '%s': %s", path, exc)
+            LOG.exception(
+                "Failed to parse JSON for task definition '%s'.",
+                path,
+                extra={"path": str(path), "exception_type": type(exc).__name__},
+            )
             return None
 
         if not isinstance(payload, dict):

app/features/tasks/definitions/repository.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import logging
 from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import func, or_, select
@@ -10,24 +9,28 @@ from app.features.core.schemas import CEFeature, ConfigEvent
 from app.features.tasks.definitions.migration import Migration
 from app.features.tasks.definitions.models import TaskDefinitionModel
 from app.library.Events import Event, EventBus, Events
+from app.library.log import get_logger
 from app.library.Services import Services
 from app.library.Singleton import Singleton
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator
+    from collections.abc import Callable
+    from contextlib import AbstractAsyncContextManager
 
     from sqlalchemy.engine.result import Result
     from sqlalchemy.ext.asyncio import AsyncSession
     from sqlalchemy.sql.elements import ColumnElement
     from sqlalchemy.sql.selectable import Select
 
-LOG: logging.Logger = logging.getLogger(__name__)
+    SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
+
+LOG = get_logger()
 
 
 class TaskDefinitionsRepository(metaclass=Singleton):
-    def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
+    def __init__(self, session: SessionFactory | None = None) -> None:
         self._migrated = False
-        self.session: AsyncGenerator[AsyncSession] = session or get_session
+        self.session: SessionFactory = session or get_session
 
     async def run_migrations(self) -> None:
         if self._migrated:
@@ -51,12 +54,12 @@ class TaskDefinitionsRepository(metaclass=Singleton):
                 LOG.debug("Refreshing task definitions due to configuration update.")
                 await GenericTaskHandler.refresh_definitions(force=True)
 
-        Services.get_instance().add(__class__.__name__, self)
+        Services.get_instance().add(TaskDefinitionsRepository.__name__, self)
         EventBus.get_instance().subscribe(
-            Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
+            Events.STARTED, handle_event, f"{TaskDefinitionsRepository.__name__}.run_migrations"
         ).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions")
 
-    async def list(self) -> list[TaskDefinitionModel]:
+    async def all(self) -> list[TaskDefinitionModel]:
         async with self.session() as session:
             result: Result[tuple[TaskDefinitionModel]] = await session.execute(
                 select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc())
@@ -115,9 +118,9 @@ class TaskDefinitionsRepository(metaclass=Singleton):
 
     async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel:
         async with self.session() as session:
-            model: TaskDefinitionModel = TaskDefinitionModel(**payload) if isinstance(payload, dict) else payload
+            model: TaskDefinitionModel = TaskDefinitionModel(**payload)
             if model.id is not None:
-                model.id = None
+                model.id = None  # ty: ignore
 
             if await self.get_by_name(name=model.name) is not None:
                 msg: str = f"Task definition with name '{model.name}' already exists."

app/features/tasks/definitions/results.py

@@ -48,10 +48,13 @@ class HandleTask(TaskSchema):
         if isinstance(ret, tuple):
             return ret
 
-        archive_file: Path = ret.get("file")
-        items: set[str] = ret.get("items", set())
+        archive_file = ret.get("file")
+        items = ret.get("items", set())
+        if not isinstance(archive_file, Path) or not isinstance(items, set):
+            return (False, "Failed to get archive information.")
+        archive_items = [item for item in items if isinstance(item, str)]
 
-        if len(items) < 1 or not archive_add(archive_file, list(items)):
+        if len(archive_items) < 1 or not archive_add(archive_file, archive_items):
             return (True, "No new items to mark as downloaded.")
 
         return (True, f"Task '{self.name}' items marked as downloaded.")
@@ -70,10 +73,13 @@ class HandleTask(TaskSchema):
         if isinstance(ret, tuple):
             return ret
 
-        archive_file: Path = ret.get("file")
-        items: set[str] = ret.get("items", set())
+        archive_file = ret.get("file")
+        items = ret.get("items", set())
+        if not isinstance(archive_file, Path) or not isinstance(items, set):
+            return (False, "Failed to get archive information.")
+        archive_items = [item for item in items if isinstance(item, str)]
 
-        if len(items) < 1 or not archive_delete(archive_file, list(items)):
+        if len(archive_items) < 1 or not archive_delete(archive_file, archive_items):
             return (True, "No items to remove from archive file.")
 
         return (True, f"Removed '{self.name}' items from archive file.")
@@ -107,6 +113,7 @@ class HandleTask(TaskSchema):
             no_archive=True,
             follow_redirect=False,
             sanitize_info=True,
+            budget_sleep=True,
         )
 
         if not ie_info or not isinstance(ie_info, dict):
@@ -133,7 +140,7 @@ class HandleTask(TaskSchema):
 
         archive_file: Path = Path(archive_file)
 
-        (ie_info, _) = await fetch_info(params, self.url, no_archive=True, follow_redirect=True)
+        (ie_info, _) = await fetch_info(params, self.url, no_archive=True, follow_redirect=True, budget_sleep=True)
         if not ie_info or not isinstance(ie_info, dict):
             return (False, "Failed to extract information from URL.")
 

app/features/tasks/definitions/router.py

@@ -1,4 +1,3 @@
-import logging
 from typing import Any
 
 from aiohttp import web
@@ -16,9 +15,10 @@ from app.features.tasks.definitions.schemas import (
 from app.features.tasks.definitions.utils import model_to_schema, schema_to_payload
 from app.library.encoder import Encoder
 from app.library.Events import EventBus, Events
+from app.library.log import get_logger
 from app.library.router import route
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 @route("GET", "api/tasks/definitions/", "task_definitions")
@@ -95,7 +95,11 @@ async def task_definitions_create(request: Request, encoder: Encoder, notify: Ev
     except ValueError as exc:
         return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
     except Exception as exc:
-        LOG.exception(exc)
+        LOG.exception(
+            "Failed to create task definition '%s'.",
+            getattr(definition_input, "name", None),
+            extra={"definition": getattr(definition_input, "name", None), "exception_type": type(exc).__name__},
+        )
         return web.json_response(
             data={"error": "Failed to create task definition."},
             status=web.HTTPInternalServerError.status_code,
@@ -144,7 +148,15 @@ async def task_definitions_update(request: Request, encoder: Encoder, notify: Ev
     except ValueError as exc:
         return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
     except Exception as exc:
-        LOG.exception(exc)
+        LOG.exception(
+            "Failed to update task definition '%s'.",
+            definition_input.name,
+            extra={
+                "definition_id": identifier,
+                "definition": definition_input.name,
+                "exception_type": type(exc).__name__,
+            },
+        )
         return web.json_response(
             data={"error": "Failed to update task definition."},
             status=web.HTTPInternalServerError.status_code,
@@ -202,7 +214,11 @@ async def task_definitions_patch(request: Request, encoder: Encoder, notify: Eve
     except ValueError as exc:
         return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
     except Exception as exc:
-        LOG.exception(exc)
+        LOG.exception(
+            "Failed to patch task definition '%s'.",
+            identifier,
+            extra={"definition_id": identifier, "exception_type": type(exc).__name__},
+        )
         return web.json_response(
             data={"error": "Failed to patch task definition."},
             status=web.HTTPInternalServerError.status_code,
@@ -228,7 +244,11 @@ async def task_definitions_delete(request: Request, encoder: Encoder, notify: Ev
     except KeyError as exc:
         return web.json_response(data={"error": str(exc)}, status=web.HTTPNotFound.status_code)
     except Exception as exc:
-        LOG.exception(exc)
+        LOG.exception(
+            "Failed to delete task definition '%s'.",
+            identifier,
+            extra={"definition_id": identifier, "exception_type": type(exc).__name__},
+        )
         return web.json_response(
             data={"error": "Failed to delete task definition."},
             status=web.HTTPInternalServerError.status_code,

app/features/tasks/definitions/service.py

@@ -3,18 +3,19 @@ from __future__ import annotations
 import asyncio
 import importlib
 import inspect
-import logging
 import pkgutil
 import random
 from datetime import UTC, datetime
 from typing import TYPE_CHECKING, Any
 
+from app.features.tasks.definitions.handlers._base_handler import BaseHandler
 from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
 from app.features.tasks.models import TaskModel
 from app.features.ytdlp.utils import archive_read
 from app.library.downloads.queue_manager import DownloadQueue
 from app.library.Events import EventBus, Events
 from app.library.ItemDTO import Item, ItemDTO
+from app.library.log import get_logger
 from app.library.Services import Services
 
 if TYPE_CHECKING:
@@ -22,12 +23,12 @@ if TYPE_CHECKING:
     from app.library.config import Config
     from app.library.Scheduler import Scheduler
 
-LOG: logging.Logger = logging.getLogger("definitions.service")
+LOG = get_logger()
 
 
 class TaskHandle:
     def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None:
-        self._handlers: list[type] = []
+        self._handlers: list[type[BaseHandler]] = []
         "The available handlers."
         self._repo: TasksRepository = tasks
         "The tasks manager."
@@ -35,7 +36,7 @@ class TaskHandle:
         "The scheduler."
         self._config: Config = config
         "The configuration."
-        self._task_name: str = f"{__class__.__name__}._dispatcher"
+        self._task_name: str = f"{TaskHandle.__name__}._dispatcher"
         "The task name for the scheduler."
         self._queued: dict[str, set[str]] = {}
         "Queued archive IDs per handler."
@@ -45,11 +46,11 @@ class TaskHandle:
         EventBus.get_instance().subscribe(
             Events.ITEM_ERROR,
             self._handle_item_error,
-            f"{__class__.__name__}.item_error",
+            f"{TaskHandle.__name__}.item_error",
         )
 
     def load(self) -> None:
-        self._handlers: list[type] = self._discover()
+        self._handlers: list[type[BaseHandler]] = self._discover()
 
         timer: str = self._config.tasks_handler_timer
         try:
@@ -58,20 +59,30 @@ class TaskHandle:
             CronSim(timer, datetime.now(UTC))
         except Exception as e:
             timer = "15 */1 * * *"
-            LOG.error(f"Invalid timer format. '{e!s}'. Defaulting to '{timer}'.")
+            LOG.error(
+                "Invalid task handler timer '%s'; using default '%s'.",
+                self._config.tasks_handler_timer,
+                timer,
+                extra={
+                    "timer": self._config.tasks_handler_timer,
+                    "default_timer": timer,
+                    "exception_type": type(e).__name__,
+                },
+            )
 
         self._scheduler.add(
             timer=timer,
             func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"),
-            id=f"{__class__.__name__}._dispatcher",
+            id=f"{TaskHandle.__name__}._dispatcher",
         )
 
     async def _dispatcher(self):
         s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []}
 
-        handler_groups: dict[str, list[tuple[HandleTask, type]]] = {}
+        handler_groups: dict[str, list[tuple[HandleTask, type[BaseHandler]]]] = {}
+        dispatches: list[tuple[HandleTask, type[BaseHandler], asyncio.Task[TaskResult | TaskFailure | None]]] = []
 
-        tasks: list[TaskModel] = await self._repo.list()
+        tasks: list[TaskModel] = await self._repo.all()
 
         for task_model in tasks:
             task: HandleTask = HandleTask.model_validate(task_model)
@@ -81,12 +92,16 @@ class TaskHandle:
                 continue
 
             if not task.get_ytdlp_opts().get_all().get("download_archive"):
-                LOG.debug(f"Task '{task.name}' does not have an archive file configured.")
+                LOG.debug(
+                    "Task '%s' does not have an archive file configured.",
+                    task.name,
+                    extra={"task_id": task.id, "task_name": task.name},
+                )
                 s["f"].append(task.name)
                 continue
 
             try:
-                handler: type | None = await self._find_handler(task)
+                handler: type[BaseHandler] | None = await self._find_handler(task)
                 if handler is None:
                     s["u"].append(task.name)
                     continue
@@ -95,9 +110,12 @@ class TaskHandle:
                 if handler_name not in handler_groups:
                     handler_groups[handler_name] = []
                 handler_groups[handler_name].append((task, handler))
-                s["h"].append(task.name)
             except Exception as e:
-                LOG.error(f"Failed to handle task '{task.name}'. '{e!s}'.")
+                LOG.exception(
+                    "Failed to find handler for task '%s'.",
+                    task.name,
+                    extra={"task_id": task.id, "task_name": task.name, "exception_type": type(e).__name__},
+                )
                 s["f"].append(task.name)
 
         for tasks_with_handlers in handler_groups.values():
@@ -112,15 +130,61 @@ class TaskHandle:
                         name=f"taskHandler-{task.id}",
                     )
                     t.add_done_callback(lambda fut, t=task: self._handle_exception(fut, t))
+                    dispatches.append((task, handler, t))
                 except Exception as e:
-                    LOG.error(f"Failed to dispatch task '{task.name}'. '{e!s}'.")
+                    LOG.exception(
+                        "Failed to schedule handler '%s' for task '%s'.",
+                        handler.__name__,
+                        task.name,
+                        extra={
+                            "task_id": task.id,
+                            "task_name": task.name,
+                            "handler_name": handler.__name__,
+                            "exception_type": type(e).__name__,
+                        },
+                    )
+                    s["f"].append(task.name)
+
+        if dispatches:
+            results = await asyncio.gather(*(t for _, _, t in dispatches), return_exceptions=True)
+
+            for (task, handler, _), result in zip(dispatches, results, strict=True):
+                if isinstance(result, TaskResult):
+                    s["h"].append(task.name)
+                    continue
+
+                if isinstance(result, TaskFailure):
+                    s["f"].append(task.name)
+                    continue
+
+                if result is None:
+                    LOG.error(
+                        "Handler '%s' returned no result for task '%s'.",
+                        handler.__name__,
+                        task.name,
+                        extra={"task_id": task.id, "task_name": task.name, "handler_name": handler.__name__},
+                    )
+
+                s["f"].append(task.name)
 
         if len(tasks) > 0:
             LOG.info(
-                f"Tasks handler summary: Handled: {len(s['h'])}, Unhandled: {len(s['u'])}, Disabled: {len(s['d'])}, Failed: {len(s['f'])}."
+                "Task dispatch finished: %s handled, %s unhandled, %s disabled, %s failed.",
+                len(s["h"]),
+                len(s["u"]),
+                len(s["d"]),
+                len(s["f"]),
+                extra={
+                    "handled_count": len(s["h"]),
+                    "unhandled_count": len(s["u"]),
+                    "disabled_count": len(s["d"]),
+                    "failed_count": len(s["f"]),
+                },
             )
 
-    async def _dispatch(self, task: HandleTask, handler: type, delay: float) -> TaskResult | TaskFailure | None:
+    async def _dispatch(
+        self, task: HandleTask, handler: type[BaseHandler], delay: float
+    ) -> TaskResult | TaskFailure | None:
         """
         Dispatch a task after a random delay to avoid rate limiting.
 
@@ -134,7 +198,12 @@ class TaskHandle:
 
         """
         if delay > 0:
-            LOG.debug(f"Delaying dispatch of task '{task.name}' by {delay:.1f} seconds.")
+            LOG.debug(
+                "Delaying dispatch of task '%s' by %.1f seconds.",
+                task.name,
+                delay,
+                extra={"task_id": task.id, "task_name": task.name, "delay_s": round(delay, 1)},
+            )
             await asyncio.sleep(delay)
         return await self.dispatch(task, handler=handler)
 
@@ -143,15 +212,29 @@ class TaskHandle:
             return
 
         if exc := fut.exception():
-            LOG.error(f"Exception while handling task '{task.name}': {exc}")
+            LOG.exception(
+                "Task handler raised after dispatch.",
+                extra={"task_id": task.id, "task_name": task.name, "exception_type": type(exc).__name__},
+                exc_info=(type(exc), exc, exc.__traceback__),
+            )
 
-    async def _find_handler(self, task: HandleTask) -> type | None:
+    async def _find_handler(self, task: HandleTask) -> type[BaseHandler] | None:
         for cls in self._handlers:
             try:
                 if await Services.get_instance().handle_async(handler=cls.can_handle, task=task):
                     return cls
             except Exception as e:
-                LOG.exception(e)
+                LOG.exception(
+                    "Handler '%s' capability check failed for task '%s'.",
+                    cls.__name__,
+                    task.name,
+                    extra={
+                        "task_id": task.id,
+                        "task_name": task.name,
+                        "handler_name": cls.__name__,
+                        "exception_type": type(e).__name__,
+                    },
+                )
                 continue
 
         return None
@@ -159,9 +242,10 @@ class TaskHandle:
     async def dispatch(
         self,
         task: HandleTask,
-        handler: type | None = None,
-        **kwargs,  # noqa: ARG002
+        handler: type[BaseHandler] | None = None,
+        **kwargs,
     ) -> TaskResult | TaskFailure | None:
+        _ = kwargs
         """
         Dispatch a task to the appropriate handler.
 
@@ -185,20 +269,59 @@ class TaskHandle:
             extraction: TaskResult | TaskFailure = await services.handle_async(
                 handler=handler.extract, task=task, config=self._config
             )
-        except NotImplementedError:
-            LOG.error(f"Handler '{handler.__name__}' does not implement extract().")
+        except NotImplementedError as exc:
+            LOG.exception(
+                "Task handler '%s' does not implement extraction for task '%s'.",
+                handler.__name__,
+                task.name,
+                extra={
+                    "task_id": task.id,
+                    "task_name": task.name,
+                    "handler_name": handler.__name__,
+                    "exception_type": type(exc).__name__,
+                },
+            )
             return TaskFailure(message="Handler does not support extraction.")
         except Exception as exc:
-            LOG.exception(exc)
+            LOG.exception(
+                "Handler '%s' extraction failed for task '%s'.",
+                handler.__name__,
+                task.name,
+                extra={
+                    "task_id": task.id,
+                    "task_name": task.name,
+                    "handler_name": handler.__name__,
+                    "exception_type": type(exc).__name__,
+                },
+            )
             raise
 
         if isinstance(extraction, TaskFailure):
-            LOG.error(f"Handler '{handler.__name__}' failed to extract items: {extraction.message}")
+            msg: str = extraction.message
+            if extraction.error and extraction.error != extraction.message:
+                msg = f"{msg} {extraction.error}"
+
+            LOG.error(
+                "Handler '%s' failed to extract items for task '%s' because %s.",
+                handler.__name__,
+                task.name,
+                msg,
+                extra={"task_id": task.id, "task_name": task.name, "handler_name": handler.__name__, "error": msg},
+            )
             return extraction
 
         if not isinstance(extraction, TaskResult):
             LOG.error(
-                f"Handler '{handler.__name__}' returned unexpected result type '{type(extraction).__name__}'.",
+                "Handler '%s' returned unexpected result type '%s' for task '%s'.",
+                handler.__name__,
+                type(extraction).__name__,
+                task.name,
+                extra={
+                    "task_id": task.id,
+                    "task_name": task.name,
+                    "handler_name": handler.__name__,
+                    "result_type": type(extraction).__name__,
+                },
             )
             return TaskFailure(
                 message="Handler returned invalid result type.", metadata={"type": type(extraction).__name__}
@@ -226,7 +349,18 @@ class TaskHandle:
 
         for item in raw_items:
             if not isinstance(item, TaskItem):
-                LOG.warning("Handler '{handler.__name__}' produced non-TaskItem entry: {item!r}")
+                LOG.warning(
+                    "Handler '%s' returned unexpected item type '%s' for task '%s'.",
+                    handler.__name__,
+                    type(item).__name__,
+                    task.name,
+                    extra={
+                        "task_id": task.id,
+                        "task_name": task.name,
+                        "handler_name": handler.__name__,
+                        "item_type": type(item).__name__,
+                    },
+                )
                 continue
 
             url: str = item.url
@@ -235,7 +369,13 @@ class TaskHandle:
 
             archive_id: str | None = item.archive_id
             if not archive_id:
-                LOG.warning(f"'{task.name}': Item with URL '{url}' is missing an archive ID. Skipping.")
+                LOG.warning(
+                    "Handler '%s' skipped '%s' for task '%s' because it has no archive ID.",
+                    handler.__name__,
+                    url,
+                    task.name,
+                    extra={"task_id": task.id, "task_name": task.name, "handler_name": handler.__name__, "url": url},
+                )
                 continue
 
             if archive_id in queued:
@@ -264,12 +404,31 @@ class TaskHandle:
         if not filtered:
             if raw_items:
                 LOG.debug(
-                    f"Handler '{handler.__name__}' produced '{len(raw_items)}' for '{task.name}' items, none queued after filtering."
+                    "Handler '%s' found %s item(s) for task '%s', but none were queued.",
+                    handler.__name__,
+                    len(raw_items),
+                    task.name,
+                    extra={
+                        "task_id": task.id,
+                        "task_name": task.name,
+                        "handler_name": handler.__name__,
+                        "raw_count": len(raw_items),
+                    },
                 )
             return TaskResult(items=[], metadata=metadata)
 
         LOG.info(
-            f"Handler '{handler.__name__}' Found '{len(filtered)}' new items for '{task.name}' (raw={len(raw_items)})."
+            "Handler '%s' found %s new item(s) for task '%s'.",
+            handler.__name__,
+            len(filtered),
+            task.name,
+            extra={
+                "task_id": task.id,
+                "task_name": task.name,
+                "handler_name": handler.__name__,
+                "item_count": len(filtered),
+                "raw_count": len(raw_items),
+            },
         )
 
         base_item = Item.format(
@@ -344,7 +503,12 @@ class TaskHandle:
             try:
                 matched = await services.handle_async(handler=handler_cls.can_handle, task=task)
             except Exception as exc:  # pragma: no cover - defensive
-                LOG.exception(exc)
+                LOG.exception(
+                    "Handler '%s' inspection capability check failed for '%s'.",
+                    handler_cls.__name__,
+                    url,
+                    extra={"handler_name": handler_cls.__name__, "url": url, "exception_type": type(exc).__name__},
+                )
                 message = str(exc)
                 return TaskFailure(
                     message=message,
@@ -382,7 +546,12 @@ class TaskHandle:
                 metadata={**base_metadata, "supported": False},
             )
         except Exception as exc:
-            LOG.exception(exc)
+            LOG.exception(
+                "Handler '%s' manual inspection failed for '%s'.",
+                handler_cls.__name__,
+                url,
+                extra={"handler_name": handler_cls.__name__, "url": url, "exception_type": type(exc).__name__},
+            )
             message = str(exc)
             return TaskFailure(
                 message=message,
@@ -403,7 +572,11 @@ class TaskHandle:
 
         if not isinstance(extraction, TaskResult):
             LOG.error(
-                f"Handler '{handler_cls.__name__}' returned unexpected result type '{type(extraction).__name__}' during inspection.",
+                "Handler '%s' returned unexpected result type '%s' while inspecting '%s'.",
+                handler_cls.__name__,
+                type(extraction).__name__,
+                url,
+                extra={"handler_name": handler_cls.__name__, "url": url, "result_type": type(extraction).__name__},
             )
             extraction = TaskResult()
 
@@ -413,11 +586,11 @@ class TaskHandle:
 
         return TaskResult(items=list(extraction.items), metadata=combined_metadata)
 
-    def _discover(self) -> list[type]:
+    def _discover(self) -> list[type[BaseHandler]]:
         """Discover all available task handlers."""
         import app.features.tasks.definitions.handlers as handlers_pkg
 
-        handlers: list[type] = []
+        handlers: list[type[BaseHandler]] = []
 
         for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__):
             if module_name.startswith("_"):

app/features/tasks/definitions/tests/test_generic_task_handler.py

@@ -8,6 +8,7 @@ from app.features.tasks.definitions.results import TaskFailure, TaskResult
 from app.features.tasks.definitions.schemas import (
     Definition,
     EngineConfig,
+    Parse,
     RequestConfig,
     ResponseConfig,
     TaskDefinition,
@@ -21,7 +22,7 @@ def reset_generic_handler(monkeypatch):
     monkeypatch.setattr(GenericTaskHandler, "_sources_mtime", {})
 
 
-def test_build_task_definition_parses_valid_payload():
+def test_build_def_payload():
     definition = TaskDefinition(
         id=1,
         name="example",
@@ -30,26 +31,24 @@ def test_build_task_definition_parses_valid_payload():
         created_at=datetime.now(),
         updated_at=datetime.now(),
         definition=Definition(
-            parse={
-                "link": {"type": "css", "expression": ".article a.link::attr(href)"},
-                "title": {"type": "css", "expression": ".article .title", "attribute": "text"},
-            },
+            parse=Parse.model_validate(
+                {
+                    "link": {"type": "css", "expression": ".article a.link::attr(href)"},
+                    "title": {"type": "css", "expression": ".article .title", "attribute": "text"},
+                }
+            ),
             engine=EngineConfig(),
             request=RequestConfig(),
             response=ResponseConfig(),
         ),
     )
 
-    assert definition is not None, "TaskDefinition should be created"
-    assert "example" == definition.name, "Name should match"
-    assert "https://example.com/articles/*" in definition.match_url, "Match URL should be in list"
-    assert "link" in definition.definition.parse, "Parse should contain link field"
-    assert ".article a.link::attr(href)" == definition.definition.parse["link"]["expression"], (
-        "Link expression should match"
-    )
+    assert definition.name == "example"
+    assert definition.match_url == ["https://example.com/articles/*"]
+    assert definition.definition.parse["link"]["expression"] == ".article a.link::attr(href)"
 
 
-def test_build_task_definition_handles_container():
+def test_build_def_container():
     definition = TaskDefinition(
         id=2,
         name="container",
@@ -58,28 +57,28 @@ def test_build_task_definition_handles_container():
         created_at=datetime.now(),
         updated_at=datetime.now(),
         definition=Definition(
-            parse={
-                "items": {
-                    "selector": ".cards .card",
-                    "fields": {
-                        "link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
-                        "title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
-                    },
+            parse=Parse.model_validate(
+                {
+                    "items": {
+                        "selector": ".cards .card",
+                        "fields": {
+                            "link": {"type": "css", "expression": ".card-header a", "attribute": "href"},
+                            "title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
+                        },
+                    }
                 }
-            },
+            ),
             engine=EngineConfig(),
             request=RequestConfig(),
             response=ResponseConfig(),
         ),
     )
 
-    assert definition is not None, "TaskDefinition should be created"
-    assert "items" in definition.definition.parse, "Parse should contain items container"
-    assert ".cards .card" == definition.definition.parse["items"]["selector"], "Items selector should match"
-    assert "link" in definition.definition.parse["items"]["fields"], "Items fields should contain link"
+    assert definition.definition.parse["items"]["selector"] == ".cards .card"
+    assert definition.definition.parse["items"]["fields"]["link"]["attribute"] == "href"
 
 
-def test_build_task_definition_handles_json():
+def test_build_def_json():
     definition = TaskDefinition(
         id=3,
         name="json-def",
@@ -88,32 +87,30 @@ def test_build_task_definition_handles_json():
         created_at=datetime.now(),
         updated_at=datetime.now(),
         definition=Definition(
-            parse={
-                "items": {
-                    "type": "jsonpath",
-                    "selector": "items",
-                    "fields": {
-                        "link": {"type": "jsonpath", "expression": "url"},
-                        "title": {"type": "jsonpath", "expression": "title"},
-                    },
+            parse=Parse.model_validate(
+                {
+                    "items": {
+                        "type": "jsonpath",
+                        "selector": "items",
+                        "fields": {
+                            "link": {"type": "jsonpath", "expression": "url"},
+                            "title": {"type": "jsonpath", "expression": "title"},
+                        },
+                    }
                 }
-            },
+            ),
             engine=EngineConfig(),
             request=RequestConfig(),
             response=ResponseConfig(type="json"),
         ),
     )
 
-    assert definition is not None, "TaskDefinition should be created"
-    assert "json" == definition.definition.response.type, "Response type should be json"
-    assert "items" in definition.definition.parse, "Parse should contain items container"
-    assert "jsonpath" == definition.definition.parse["items"]["type"], "Items type should be jsonpath"
-    assert "jsonpath" == definition.definition.parse["items"]["fields"]["link"]["type"], (
-        "Link field type should be jsonpath"
-    )
+    assert definition.definition.response.type == "json"
+    assert definition.definition.parse["items"]["type"] == "jsonpath"
+    assert definition.definition.parse["items"]["fields"]["link"]["type"] == "jsonpath"
 
 
-def test_parse_items_extracts_values():
+def test_parse_items_basic():
     definition = TaskDefinition(
         id=4,
         name="example",
@@ -122,11 +119,13 @@ def test_parse_items_extracts_values():
         created_at=datetime.now(),
         updated_at=datetime.now(),
         definition=Definition(
-            parse={
-                "link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
-                "title": {"type": "css", "expression": ".article .title", "attribute": "text"},
-                "id": {"type": "css", "expression": ".article", "attribute": "data-id"},
-            },
+            parse=Parse.model_validate(
+                {
+                    "link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
+                    "title": {"type": "css", "expression": ".article .title", "attribute": "text"},
+                    "id": {"type": "css", "expression": ".article", "attribute": "data-id"},
+                }
+            ),
             engine=EngineConfig(),
             request=RequestConfig(),
             response=ResponseConfig(),
@@ -146,14 +145,16 @@ def test_parse_items_extracts_values():
 
     items = GenericTaskHandler._parse_items(definition, html, "https://example.com/base/")
 
-    assert 2 == len(items), "Should extract 2 items"
-    assert "https://example.com/article-101" == items[0]["link"], "First item link should be absolute URL"
-    assert "First Title" == items[0]["title"], "First item title should match"
-    assert "101" == items[0]["id"], "First item id should match"
-    assert "https://example.com/article-102" == items[1]["link"], "Second item link should match"
+    assert len(items) == 2
+    assert items[0] == {
+        "link": "https://example.com/article-101",
+        "title": "First Title",
+        "id": "101",
+    }
+    assert items[1]["link"] == "https://example.com/article-102"
 
 
-def test_parse_items_handles_nested_card_layout():
+def test_parse_items_cards():
     definition = TaskDefinition(
         id=5,
         name="nested",
@@ -162,34 +163,36 @@ def test_parse_items_handles_nested_card_layout():
         created_at=datetime.now(),
         updated_at=datetime.now(),
         definition=Definition(
-            parse={
-                "items": {
-                    "type": "css",
-                    "selector": ".columns .card",
-                    "fields": {
-                        "link": {
-                            "type": "css",
-                            "expression": ".card-header a[href]",
-                            "attribute": "href",
+            parse=Parse.model_validate(
+                {
+                    "items": {
+                        "type": "css",
+                        "selector": ".columns .card",
+                        "fields": {
+                            "link": {
+                                "type": "css",
+                                "expression": ".card-header a[href]",
+                                "attribute": "href",
+                            },
+                            "title": {
+                                "type": "css",
+                                "expression": ".card-header a[href]",
+                                "attribute": "text",
+                            },
+                            "poet": {
+                                "type": "css",
+                                "expression": "footer .card-footer-item:first-child a",
+                                "attribute": "text",
+                            },
+                            "category": {
+                                "type": "css",
+                                "expression": "footer .card-footer-item:nth-child(2) a",
+                                "attribute": "text",
+                            },
                         },
-                        "title": {
-                            "type": "css",
-                            "expression": ".card-header a[href]",
-                            "attribute": "text",
-                        },
-                        "poet": {
-                            "type": "css",
-                            "expression": "footer .card-footer-item:first-child a",
-                            "attribute": "text",
-                        },
-                        "category": {
-                            "type": "css",
-                            "expression": "footer .card-footer-item:nth-child(2) a",
-                            "attribute": "text",
-                        },
-                    },
+                    }
                 }
-            },
+            ),
             engine=EngineConfig(),
             request=RequestConfig(),
             response=ResponseConfig(),
@@ -234,19 +237,21 @@ def test_parse_items_handles_nested_card_layout():
 
     items = GenericTaskHandler._parse_items(definition, html, "https://example.com")
 
-    assert 2 == len(items), "Should extract 2 items"
-    assert "https://example.com/poems/view/111" == items[0]["link"], "First item link should match"
-    assert "First Poem" == items[0]["title"], "First item title should match"
-    assert "Poet Alpha" == items[0]["poet"], "First item poet should match"
-    assert "Category One" == items[0]["category"], "First item category should match"
-
-    assert "https://example.com/poems/view/222" == items[1]["link"], "Second item link should match"
-    assert "Second Poem" == items[1]["title"], "Second item title should match"
-    assert "Poet Beta" == items[1]["poet"], "Second item poet should match"
-    assert "category" not in items[1], "Second item should not have category"
+    assert len(items) == 2
+    assert items[0] == {
+        "link": "https://example.com/poems/view/111",
+        "title": "First Poem",
+        "poet": "Poet Alpha",
+        "category": "Category One",
+    }
+    assert items[1] == {
+        "link": "https://example.com/poems/view/222",
+        "title": "Second Poem",
+        "poet": "Poet Beta",
+    }
 
 
-def test_parse_items_handles_json_container():
+def test_parse_items_json():
     definition = TaskDefinition(
         id=6,
         name="json",
@@ -255,17 +260,19 @@ def test_parse_items_handles_json_container():
         created_at=datetime.now(),
         updated_at=datetime.now(),
         definition=Definition(
-            parse={
-                "items": {
-                    "type": "jsonpath",
-                    "selector": "entries",
-                    "fields": {
-                        "link": {"type": "jsonpath", "expression": "url"},
-                        "title": {"type": "jsonpath", "expression": "title"},
-                        "id": {"type": "jsonpath", "expression": "id"},
-                    },
+            parse=Parse.model_validate(
+                {
+                    "items": {
+                        "type": "jsonpath",
+                        "selector": "entries",
+                        "fields": {
+                            "link": {"type": "jsonpath", "expression": "url"},
+                            "title": {"type": "jsonpath", "expression": "title"},
+                            "id": {"type": "jsonpath", "expression": "id"},
+                        },
+                    }
                 }
-            },
+            ),
             engine=EngineConfig(),
             request=RequestConfig(),
             response=ResponseConfig(type="json"),
@@ -287,14 +294,10 @@ def test_parse_items_handles_json_container():
         json_data=payload,
     )
 
-    assert 2 == len(items), "Should extract 2 items (third missing link)"
-    assert "https://example.com/video/1" == items[0]["link"], "First item link should be absolute"
-    assert "First" == items[0]["title"], "First item title should match"
-    assert "1" == items[0]["id"], "First item id should match"
-
-    assert "https://example.com/video/2" == items[1]["link"], "Second item link should match"
-    assert "Second" == items[1]["title"], "Second item title should match"
-    assert "2" == items[1]["id"], "Second item id should match"
+    assert items == [
+        {"link": "https://example.com/video/1", "title": "First", "id": "1"},
+        {"link": "https://example.com/video/2", "title": "Second", "id": "2"},
+    ]
 
 
 @pytest.mark.asyncio
@@ -307,16 +310,18 @@ async def test_generic_task_handler_inspect(monkeypatch):
         created_at=datetime.now(),
         updated_at=datetime.now(),
         definition=Definition(
-            parse={
-                "items": {
-                    "type": "jsonpath",
-                    "selector": "items",
-                    "fields": {
-                        "link": {"type": "jsonpath", "expression": "url"},
-                        "title": {"type": "jsonpath", "expression": "title"},
-                    },
+            parse=Parse.model_validate(
+                {
+                    "items": {
+                        "type": "jsonpath",
+                        "selector": "items",
+                        "fields": {
+                            "link": {"type": "jsonpath", "expression": "url"},
+                            "title": {"type": "jsonpath", "expression": "title"},
+                        },
+                    }
                 }
-            },
+            ),
             engine=EngineConfig(),
             request=RequestConfig(),
             response=ResponseConfig(type="json"),
@@ -345,14 +350,14 @@ async def test_generic_task_handler_inspect(monkeypatch):
         task = HandleTask(id=1, name="Inspect", url="https://example.com/api")
         result: TaskResult | TaskFailure = await GenericTaskHandler.extract(task)
 
-        assert isinstance(result, TaskResult), "Result should be TaskResult"
-        assert 1 == len(result.items), "Should have 1 item"
+        assert isinstance(result, TaskResult)
+        assert len(result.items) == 1
         item = result.items[0]
-        assert "https://example.com/video/1" == item.url, "Item URL should match"
-        assert "First" == item.title, "Item title should match"
+        assert item.url == "https://example.com/video/1"
+        assert item.title == "First"
 
 
-def test_parse_items_handles_json_top_level_list():
+def test_parse_items_json_list():
     definition = TaskDefinition(
         id=8,
         name="json-list",
@@ -361,16 +366,18 @@ def test_parse_items_handles_json_top_level_list():
         created_at=datetime.now(),
         updated_at=datetime.now(),
         definition=Definition(
-            parse={
-                "items": {
-                    "type": "jsonpath",
-                    "selector": "[]",
-                    "fields": {
-                        "link": {"type": "jsonpath", "expression": "url"},
-                        "title": {"type": "jsonpath", "expression": "title"},
-                    },
+            parse=Parse.model_validate(
+                {
+                    "items": {
+                        "type": "jsonpath",
+                        "selector": "[]",
+                        "fields": {
+                            "link": {"type": "jsonpath", "expression": "url"},
+                            "title": {"type": "jsonpath", "expression": "title"},
+                        },
+                    }
                 }
-            },
+            ),
             engine=EngineConfig(),
             request=RequestConfig(),
             response=ResponseConfig(type="json"),
@@ -389,8 +396,7 @@ def test_parse_items_handles_json_top_level_list():
         json_data=payload,
     )
 
-    assert 2 == len(items), "Should extract 2 items"
-    assert "https://example.com/video/1" == items[0]["link"], "First item link should match"
-    assert "First" == items[0]["title"], "First item title should match"
-    assert "https://example.com/video/2" == items[1]["link"], "Second item link should match"
-    assert "Second" == items[1]["title"], "Second item title should match"
+    assert items == [
+        {"link": "https://example.com/video/1", "title": "First"},
+        {"link": "https://example.com/video/2", "title": "Second"},
+    ]

app/features/tasks/definitions/tests/test_rss_handler.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 import pytest
 
 from app.features.tasks.definitions.handlers.rss import RssGenericHandler
@@ -21,6 +23,10 @@ class DummyOpts:
         return self._data
 
 
+def _opts(tmp_path: Path) -> DummyOpts:
+    return DummyOpts({"download_archive": str(tmp_path / "archive.txt")})
+
+
 class TestRssHandlerParsing:
     """Test URL parsing for RSS/Atom feeds using the tests() method."""
 
@@ -47,7 +53,7 @@ class TestRssHandlerExtraction:
     """Test RSS feed extraction and parsing."""
 
     @pytest.mark.asyncio
-    async def test_rss_atom_feed_extraction(self, monkeypatch):
+    async def test_rss_atom_feed_extraction(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
         """Test extraction from Atom feed."""
         atom_feed = """<?xml version="1.0" encoding="UTF-8"?>
 <feed xmlns="http://www.w3.org/2005/Atom">
@@ -69,7 +75,7 @@ class TestRssHandlerExtraction:
             return DummyResponse(atom_feed)
 
         monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
-        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"}))  # noqa: ARG005
+        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path))  # noqa: ARG005
 
         task = HandleTask(
             id=1,
@@ -89,7 +95,7 @@ class TestRssHandlerExtraction:
         assert result.metadata["entry_count"] == 2
 
     @pytest.mark.asyncio
-    async def test_rss_feed_extraction(self, monkeypatch):
+    async def test_rss_feed_extraction(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
         """Test extraction from RSS feed."""
         rss_feed = """<?xml version="1.0" encoding="UTF-8"?>
 <rss version="2.0">
@@ -113,7 +119,7 @@ class TestRssHandlerExtraction:
             return DummyResponse(rss_feed)
 
         monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
-        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"}))  # noqa: ARG005
+        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path))  # noqa: ARG005
 
         task = HandleTask(
             id=1,
@@ -133,7 +139,7 @@ class TestRssHandlerExtraction:
         assert result.metadata["entry_count"] == 2
 
     @pytest.mark.asyncio
-    async def test_can_handle(self, monkeypatch):
+    async def test_can_handle(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
         """Test can_handle method."""
         atom_feed = """<?xml version="1.0" encoding="UTF-8"?>
 <feed xmlns="http://www.w3.org/2005/Atom">
@@ -147,7 +153,7 @@ class TestRssHandlerExtraction:
             return DummyResponse(atom_feed)
 
         monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
-        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"}))  # noqa: ARG005
+        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path))  # noqa: ARG005
 
         task = HandleTask(
             id=1,
@@ -172,7 +178,7 @@ class TestRssHandlerEdgeCases:
     """Test edge cases in RSS handling."""
 
     @pytest.mark.asyncio
-    async def test_empty_feed(self, monkeypatch):
+    async def test_empty_feed(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
         """Test handling of empty feed."""
         empty_feed = """<?xml version="1.0" encoding="UTF-8"?>
 <rss version="2.0">
@@ -186,7 +192,7 @@ class TestRssHandlerEdgeCases:
             return DummyResponse(empty_feed)
 
         monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
-        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"}))  # noqa: ARG005
+        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path))  # noqa: ARG005
 
         task = HandleTask(
             id=1,
@@ -202,7 +208,7 @@ class TestRssHandlerEdgeCases:
         assert result.metadata["entry_count"] == 0
 
     @pytest.mark.asyncio
-    async def test_invalid_feed_url(self, monkeypatch):
+    async def test_invalid_feed_url(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
         """Test handling of invalid feed URL."""
         from app.features.tasks.definitions.results import TaskFailure
 
@@ -211,7 +217,7 @@ class TestRssHandlerEdgeCases:
             raise Exception(msg)
 
         monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
-        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"}))  # noqa: ARG005
+        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path))  # noqa: ARG005
 
         task = HandleTask(
             id=1,
@@ -225,7 +231,7 @@ class TestRssHandlerEdgeCases:
         assert isinstance(result, TaskFailure)
 
     @pytest.mark.asyncio
-    async def test_missing_urls_in_feed(self, monkeypatch):
+    async def test_missing_urls_in_feed(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
         """Test handling of entries missing URLs."""
         feed = """<?xml version="1.0" encoding="UTF-8"?>
 <rss version="2.0">
@@ -245,7 +251,7 @@ class TestRssHandlerEdgeCases:
             return DummyResponse(feed)
 
         monkeypatch.setattr(RssGenericHandler, "request", staticmethod(fake_request))
-        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"}))  # noqa: ARG005
+        monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: _opts(tmp_path))  # noqa: ARG005
 
         task = HandleTask(
             id=1,

app/features/tasks/definitions/tests/test_task_definitions.py

@@ -21,6 +21,7 @@ from app.features.tasks.definitions.router import (
 from app.library.encoder import Encoder
 from app.library.sqlite_store import SqliteStore
 from app.main import EventBus
+from app.tests.helpers import make_in_memory_db_path
 
 
 def _sample_definition(name: str = "example", *, priority: int = 0) -> dict:
@@ -49,17 +50,14 @@ async def repo() -> AsyncGenerator[TaskDefinitionsRepository, None]:
     TaskDefinitionsRepository._reset_singleton()
     SqliteStore._reset_singleton()
 
-    store = SqliteStore(db_path=":memory:")
+    store = SqliteStore(db_path=make_in_memory_db_path("task-definitions-repository"))
     await store.get_connection()
 
     repository = TaskDefinitionsRepository.get_instance()
 
     yield repository
 
-    if store._conn:
-        await store._conn.close()
-    if store._engine:
-        await store._engine.dispose()
+    await store.close()
 
     TaskDefinitionsRepository._reset_singleton()
     SqliteStore._reset_singleton()
@@ -71,7 +69,7 @@ class TestTaskDefinitionsRepository:
         await repo.create(_sample_definition("Alpha", priority=2))
         await repo.create(_sample_definition("Beta", priority=1))
 
-        items = await repo.list()
+        items = await repo.all()
         assert len(items) == 2, "Should return two task definitions"
         assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name"
 

app/features/tasks/definitions/tests/test_tver_handler.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 import pytest
 
 from app.features.tasks.definitions.handlers.tver import TverHandler
@@ -25,7 +27,7 @@ class DummyOpts:
 
 
 @pytest.mark.asyncio
-async def test_tver_handler_extract(monkeypatch):
+async def test_tver_handler_extract(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     """Test tver handler extraction of episodes from series."""
     session_response = {
         "result": {
@@ -118,8 +120,10 @@ async def test_tver_handler_extract(monkeypatch):
         msg = f"Unexpected URL: {url}"
         raise RuntimeError(msg)
 
+    archive_path = tmp_path / "archive.txt"
+
     monkeypatch.setattr(TverHandler, "request", staticmethod(fake_request))
-    monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": "/tmp/archive"}))
+    monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda _: DummyOpts({"download_archive": str(archive_path)}))
 
     task = HandleTask(id=1, name="Test Tver Series", url="https://tver.jp/series/sr8sb9pnhc", preset="default")
 
@@ -142,7 +146,7 @@ async def test_tver_handler_extract(monkeypatch):
 
 
 @pytest.mark.parametrize(("url", "should_match"), TverHandler.tests())
-def test_tver_handler_parse(url: str, should_match: bool):
+def test_parse(url: str, should_match: bool) -> None:
     """Test tver URL parsing."""
     result = TverHandler.parse(url)
     if should_match:
@@ -153,7 +157,7 @@ def test_tver_handler_parse(url: str, should_match: bool):
 
 
 @pytest.mark.asyncio
-async def test_tver_handler_can_handle():
+async def test_can_handle() -> None:
     """Test tver handler can_handle method."""
     task_valid = HandleTask(id=1, name="Test", url="https://tver.jp/series/sr8sb9pnhc", preset="default")
     task_invalid = HandleTask(id=2, name="Test", url="https://youtube.com/watch?v=123", preset="default")

app/features/tasks/definitions/tests/test_twitch_handler.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 import pytest
 
 from app.features.tasks.definitions.handlers.twitch import TwitchHandler
@@ -22,7 +24,7 @@ class DummyOpts:
 
 
 @pytest.mark.asyncio
-async def test_twitch_handler_inspect(monkeypatch):
+async def test_twitch_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     feed = """
     <rss><channel>
       <item>
@@ -39,8 +41,10 @@ async def test_twitch_handler_inspect(monkeypatch):
     async def fake_request(**kwargs):  # noqa: ARG001
         return DummyResponse(feed)
 
+    archive_path = tmp_path / "archive.txt"
+
     monkeypatch.setattr(TwitchHandler, "request", staticmethod(fake_request))
-    monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"}))  # noqa: ARG005
+    monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": str(archive_path)}))  # noqa: ARG005
 
     task = HandleTask(
         id=1,

app/features/tasks/definitions/tests/test_youtube_handler.py

@@ -1,3 +1,5 @@
+from pathlib import Path
+
 import pytest
 
 from app.features.tasks.definitions.handlers.youtube import YoutubeHandler
@@ -22,7 +24,7 @@ class DummyOpts:
 
 
 @pytest.mark.asyncio
-async def test_youtube_handler_inspect(monkeypatch):
+async def test_youtube_handler_inspect(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     feed = """
     <feed xmlns='http://www.w3.org/2005/Atom' xmlns:yt='http://www.youtube.com/xml/schemas/2015'>
       <entry>
@@ -41,8 +43,10 @@ async def test_youtube_handler_inspect(monkeypatch):
     async def fake_request(**kwargs):  # noqa: ARG001
         return DummyResponse(feed)
 
+    archive_path = tmp_path / "archive.txt"
+
     monkeypatch.setattr(YoutubeHandler, "request", staticmethod(fake_request))
-    monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": "/tmp/archive"}))  # noqa: ARG005
+    monkeypatch.setattr(HandleTask, "get_ytdlp_opts", lambda self: DummyOpts({"download_archive": str(archive_path)}))  # noqa: ARG005
 
     task = HandleTask(
         id=1,

app/features/tasks/definitions/utils.py

@@ -1,9 +1,17 @@
-from typing import Any
+from typing import Any, Literal, overload
 
 from app.features.tasks.definitions.models import TaskDefinitionModel
 from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary
 
 
+@overload
+def model_to_schema(model: TaskDefinitionModel, summary: Literal[False] = False) -> TaskDefinition: ...
+
+
+@overload
+def model_to_schema(model: TaskDefinitionModel, summary: Literal[True]) -> TaskDefinitionSummary: ...
+
+
 def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary:
     """
     Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema.

app/features/tasks/migration.py

@@ -1,18 +1,18 @@
 from __future__ import annotations
 
 import json
-import logging
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from app.features.core.migration import Migration as FeatureMigration
 from app.features.tasks.schemas import Task
 from app.library.config import Config
+from app.library.log import get_logger
 
 if TYPE_CHECKING:
     from app.features.tasks.repository import TasksRepository
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 class Migration(FeatureMigration):
@@ -36,7 +36,11 @@ class Migration(FeatureMigration):
         try:
             items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text())
         except Exception as exc:
-            LOG.exception("Failed to read %s: %s. Ignoring", self._source_file, exc)
+            LOG.exception(
+                "Failed to read tasks migration file '%s'. Ignoring.",
+                self._source_file,
+                extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
+            )
             await self._move_file(self._source_file)
             return
 
@@ -54,7 +58,11 @@ class Migration(FeatureMigration):
                 await self._repo.create(normalized)
                 inserted += 1
             except Exception as exc:
-                LOG.exception("Failed to insert task '%s': %s", normalized["name"], exc)
+                LOG.exception(
+                    "Failed to insert task '%s'.",
+                    normalized["name"],
+                    extra={"task_name": normalized["name"], "exception_type": type(exc).__name__},
+                )
 
         LOG.info("Migrated %s task(s) from %s.", inserted, self._source_file)
         await self._move_file(self._source_file)

app/features/tasks/repository.py

@@ -1,29 +1,48 @@
 from __future__ import annotations
 
-import logging
 from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import func, or_, select
 
 from app.features.core.deps import get_session
 from app.features.tasks.models import TaskModel
+from app.library.log import get_logger
 from app.library.Singleton import Singleton
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator
+    from collections.abc import Callable
+    from contextlib import AbstractAsyncContextManager
 
     from sqlalchemy.engine.result import Result
     from sqlalchemy.ext.asyncio import AsyncSession
     from sqlalchemy.sql.elements import ColumnElement
     from sqlalchemy.sql.selectable import Select
 
-LOG: logging.Logger = logging.getLogger(__name__)
+    SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
+
+LOG = get_logger()
+
+
+def _model_from_payload(payload: dict[str, Any]) -> TaskModel:
+    model = TaskModel()
+    for key, value in payload.items():
+        if not hasattr(model, key):
+            msg = f"'{key}' is an invalid keyword argument for TaskModel"
+            raise TypeError(msg)
+        setattr(model, key, value)
+    return model
+
+
+def _coerce_model(payload: TaskModel | dict[str, Any]) -> TaskModel:
+    if isinstance(payload, TaskModel):
+        return payload
+    return _model_from_payload(payload)
 
 
 class TasksRepository(metaclass=Singleton):
-    def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
+    def __init__(self, session: SessionFactory | None = None) -> None:
         self._migrated = False
-        self.session = session or get_session
+        self.session: SessionFactory = session or get_session
 
     async def run_migrations(self) -> None:
         if self._migrated:
@@ -38,7 +57,7 @@ class TasksRepository(metaclass=Singleton):
     def get_instance() -> TasksRepository:
         return TasksRepository()
 
-    async def list(self) -> list[TaskModel]:
+    async def all(self) -> list[TaskModel]:
         async with self.session() as session:
             result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc()))
             return list(result.scalars().all())
@@ -90,7 +109,7 @@ class TasksRepository(metaclass=Singleton):
         """Get all enabled tasks."""
         async with self.session() as session:
             result: Result[tuple[TaskModel]] = await session.execute(
-                select(TaskModel).where(TaskModel.enabled == True).order_by(TaskModel.name.asc())  # noqa: E712
+                select(TaskModel).where(TaskModel.enabled).order_by(TaskModel.name.asc())
             )
             return list(result.scalars().all())
 
@@ -102,11 +121,11 @@ class TasksRepository(metaclass=Singleton):
             )
             return list(result.scalars().all())
 
-    async def create(self, payload: TaskModel | dict) -> TaskModel:
+    async def create(self, payload: TaskModel | dict[str, Any]) -> TaskModel:
         async with self.session() as session:
-            model: TaskModel = TaskModel(**payload) if isinstance(payload, dict) else payload
+            model = _coerce_model(payload)
             if model.id is not None:
-                model.id = None
+                model.id = None  # ty: ignore
 
             if await self.get_by_name(name=model.name) is not None:
                 msg: str = f"Task with name '{model.name}' already exists."

app/features/tasks/router.py

@@ -1,4 +1,4 @@
-import logging
+import asyncio
 from typing import TYPE_CHECKING, Any
 
 from aiohttp import web
@@ -17,6 +17,7 @@ from app.library.ag_utils import ag
 from app.library.config import Config
 from app.library.encoder import Encoder
 from app.library.Events import EventBus, Events
+from app.library.log import get_logger
 from app.library.router import route
 from app.library.Utils import get_channel_images, get_file, validate_url
 
@@ -24,7 +25,7 @@ if TYPE_CHECKING:
     from pathlib import Path
 
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 TIMER_SLOTS_PER_HOUR: int = 12
 
@@ -93,7 +94,11 @@ def _offset_timer(timer: str, index: int) -> str:
 
         return f"{minute} {hour} {dom} {month} {dow}"
     except Exception as e:
-        LOG.warning(f"Failed to offset timer '{timer}': {e}")
+        LOG.warning(
+            "Failed to offset task timer.",
+            extra={"timer": timer, "index": index, "error": str(e), "exception_type": type(e).__name__},
+            exc_info=True,
+        )
         return timer
 
 
@@ -136,10 +141,15 @@ async def _get_info(url: str, preset: str) -> tuple[str | None, str | None]:
 
         return (name, converted_url)
     except TimeoutError:
-        LOG.debug(f"Timeout while inferring name from '{url}'")
+        LOG.debug("Timeout while inferring name from '%s'.", url, extra={"url": url, "preset": preset})
         return (None, None)
     except Exception as e:
-        LOG.debug(f"Failed to infer name from '{url}': {e}")
+        LOG.debug(
+            "Failed to infer a task name from '%s' because %s.",
+            url,
+            e,
+            extra={"url": url, "preset": preset, "error": str(e)},
+        )
         return (None, None)
 
 
@@ -151,6 +161,22 @@ def _serialize(model: Any) -> dict:
     return _model(model).model_dump()
 
 
+async def _require_timer_or_handler(task: Task, handler: TaskHandle) -> None:
+    if task.timer:
+        return
+
+    if task.handler_enabled is False:
+        msg = "Task requires a timer when the handler is disabled."
+        raise ValueError(msg)
+
+    result = await handler.inspect(url=task.url, preset=task.preset, static_only=True)
+    if isinstance(result, TaskResult) and result.metadata.get("matched") is True:
+        return
+
+    msg = "Task requires a timer when no supported handler matches the URL."
+    raise ValueError(msg)
+
+
 @route("GET", "api/tasks/", "tasks_list")
 async def tasks_list(request: Request, repo: TasksRepository, encoder: Encoder) -> Response:
     page, per_page = normalize_pagination(request)
@@ -166,7 +192,13 @@ async def tasks_list(request: Request, repo: TasksRepository, encoder: Encoder)
 
 
 @route("POST", "api/tasks/", "tasks_add")
-async def tasks_add(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response:
+async def tasks_add(
+    request: Request,
+    repo: TasksRepository,
+    encoder: Encoder,
+    notify: EventBus,
+    handler: TaskHandle,
+) -> Response:
     data = await request.json()
 
     if isinstance(data, dict):
@@ -205,17 +237,18 @@ async def tasks_add(request: Request, repo: TasksRepository, encoder: Encoder, n
             status=web.HTTPConflict.status_code,
         )
 
+    pending_tasks: list[dict[str, Any]] = []
     created_tasks: list[dict[str, Any]] = []
     base_settings = first_task.model_dump(exclude={"id", "name", "url", "created_at", "updated_at"})
 
     for idx, item in enumerate(data):
         if not isinstance(item, dict):
-            LOG.warning(f"Skipping item {idx}: not a dict")
+            LOG.warning("Skipping item %s: not a dict.", idx, extra={"index": idx})
             continue
 
         url = str(item.get("url", "")).strip()
         if not url:
-            LOG.debug(f"Skipping item {idx}: empty URL")
+            LOG.debug("Skipping item %s: empty URL.", idx, extra={"index": idx})
             continue
 
         inferred_name, converted_url = await _get_info(url, item.get("preset", first_task.preset))
@@ -243,11 +276,23 @@ async def tasks_add(request: Request, repo: TasksRepository, encoder: Encoder, n
 
         try:
             validated = Task.model_validate(task_dict)
-            task_data = validated.model_dump()
         except ValidationError as exc:
-            LOG.warning(f"Skipping task {idx}: validation failed - {exc}")
-            continue
+            return web.json_response(
+                data={"error": "Failed to validate task.", "detail": format_validation_errors(exc)},
+                status=web.HTTPBadRequest.status_code,
+            )
 
+        try:
+            await _require_timer_or_handler(validated, handler)
+        except ValueError as exc:
+            return web.json_response(
+                data={"error": str(exc)},
+                status=web.HTTPBadRequest.status_code,
+            )
+
+        pending_tasks.append(validated.model_dump())
+
+    for idx, task_data in enumerate(pending_tasks):
         try:
             created = await repo.create(task_data)
             saved = _serialize(created)
@@ -257,7 +302,11 @@ async def tasks_add(request: Request, repo: TasksRepository, encoder: Encoder, n
                 data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.CREATE, data=saved),
             )
         except ValueError as exc:
-            LOG.warning(f"Failed to create task {idx}: {exc}")
+            LOG.warning(
+                "Failed to create task from request item %s.",
+                idx,
+                extra={"index": idx, "error": str(exc), "exception_type": type(exc).__name__},
+            )
             continue
 
     if len(created_tasks) == 0:
@@ -304,7 +353,13 @@ async def tasks_delete(request: Request, repo: TasksRepository, encoder: Encoder
 
 
 @route("PATCH", r"api/tasks/{id:\d+}", "tasks_patch")
-async def tasks_patch(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response:
+async def tasks_patch(
+    request: Request,
+    repo: TasksRepository,
+    encoder: Encoder,
+    notify: EventBus,
+    handler: TaskHandle,
+) -> Response:
     if not (identifier := request.match_info.get("id")):
         return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
 
@@ -334,6 +389,16 @@ async def tasks_patch(request: Request, repo: TasksRepository, encoder: Encoder,
             status=web.HTTPConflict.status_code,
         )
 
+    merged = _model(model).model_copy(update=validated.model_dump(exclude_unset=True))
+
+    try:
+        await _require_timer_or_handler(merged, handler)
+    except ValueError as exc:
+        return web.json_response(
+            data={"error": str(exc)},
+            status=web.HTTPBadRequest.status_code,
+        )
+
     updated = _serialize(await repo.update(model.id, validated.model_dump(exclude_unset=True)))
     notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.UPDATE, data=updated))
     return web.json_response(
@@ -344,7 +409,13 @@ async def tasks_patch(request: Request, repo: TasksRepository, encoder: Encoder,
 
 
 @route("PUT", r"api/tasks/{id:\d+}", "tasks_update")
-async def tasks_update(request: Request, repo: TasksRepository, encoder: Encoder, notify: EventBus) -> Response:
+async def tasks_update(
+    request: Request,
+    repo: TasksRepository,
+    encoder: Encoder,
+    notify: EventBus,
+    handler: TaskHandle,
+) -> Response:
     if not (identifier := request.match_info.get("id")):
         return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
 
@@ -374,6 +445,14 @@ async def tasks_update(request: Request, repo: TasksRepository, encoder: Encoder
             status=web.HTTPConflict.status_code,
         )
 
+    try:
+        await _require_timer_or_handler(validated, handler)
+    except ValueError as exc:
+        return web.json_response(
+            data={"error": str(exc)},
+            status=web.HTTPBadRequest.status_code,
+        )
+
     updated = _serialize(await repo.update(model.id, validated.model_dump(exclude_unset=True)))
     notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.TASKS, action=CEAction.UPDATE, data=updated))
 
@@ -404,7 +483,7 @@ async def task_handler_inspect(request: Request, handler: TaskHandle, encoder: E
     static_only: bool = data.get("static_only", False) if isinstance(data, dict) else False
     if not static_only:
         try:
-            validate_url(url, allow_internal=config.allow_internal_urls)
+            await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
         except ValueError as e:
             return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code)
 
@@ -416,7 +495,16 @@ async def task_handler_inspect(request: Request, handler: TaskHandle, encoder: E
             url=url, preset=preset, handler_name=handler_name, static_only=static_only
         )
     except Exception as e:
-        LOG.exception(e)
+        LOG.exception(
+            "Failed to inspect task handler for '%s'.",
+            url,
+            extra={
+                "handler_name": handler_name,
+                "url": url,
+                "static_only": static_only,
+                "exception_type": type(e).__name__,
+            },
+        )
         return web.json_response(
             {"error": "Failed to inspect handler.", "message": str(e)},
             status=web.HTTPInternalServerError.status_code,
@@ -565,7 +653,12 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
                     if not save_path.exists():
                         save_path.mkdir(parents=True, exist_ok=True)
             except Exception as e:
-                LOG.warning(f"Failed to resolve final path from outtmpl. '{e!s}'")
+                LOG.warning(
+                    "Failed to resolve the metadata output path for task '%s'.",
+                    task.name,
+                    extra={"task_id": task.id, "task_name": task.name, "error": str(e)},
+                    exc_info=True,
+                )
 
         info = {
             "id": ag(metadata, ["id", "channel_id"]),
@@ -583,7 +676,12 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
                 data={"error": "Failed to get title from metadata."}, status=web.HTTPBadRequest.status_code
             )
 
-        LOG.info(f"Generating metadata for task '{task.name}' in '{save_path!s}'")
+        LOG.info(
+            "Generating metadata for task '%s' in '%s'.",
+            task.name,
+            save_path,
+            extra={"task_id": task.id, "task_name": task.name, "save_path": str(save_path)},
+        )
 
         from yt_dlp.utils import sanitize_filename
 
@@ -607,8 +705,9 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
             xml_content += f'  <uniqueid type="{NFOMakerPP._escape_text(info.get("id_type"))}" default="true">{NFOMakerPP._escape_text(info.get("id"))}</uniqueid>\n'
         if info.get("uploader"):
             xml_content += f"  <studio>{NFOMakerPP._escape_text(info.get('uploader'))}</studio>\n"
-        if info.get("tags", []):
-            for tag in info.get("tags", []):
+        tags = info.get("tags", [])
+        if isinstance(tags, list):
+            for tag in tags:
                 xml_content += f"  <tag>{NFOMakerPP._escape_text(tag)}</tag>\n"
         if info.get("year"):
             xml_content += f"  <year>{info.get('year')}</year>\n"
@@ -640,14 +739,25 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
                 url: str | None = None
                 try:
                     url = thumbnails.get(key)
-                    LOG.info(f"Fetching thumbnail '{key}' from '{url}'")
+                    LOG.info(
+                        "Fetching '%s' thumbnail for task '%s'.",
+                        key,
+                        task.name,
+                        extra={"task_id": task.id, "task_name": task.name, "thumbnail": key, "url": url},
+                    )
                     if not url:
                         continue
 
                     try:
-                        validate_url(url, allow_internal=config.allow_internal_urls)
-                    except ValueError:
-                        LOG.warning(f"Invalid thumbnail url '{url}'")
+                        await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
+                    except ValueError as exc:
+                        LOG.warning(
+                            "Task '%s' has an invalid '%s' thumbnail URL because %s.",
+                            task.name,
+                            key,
+                            exc,
+                            extra={"task_id": task.id, "task_name": task.name, "thumbnail": key, "url": url},
+                        )
                         continue
 
                     resp = await client.request(
@@ -661,11 +771,27 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
                     img_file.write_bytes(resp.content)
                     thumbnails[key] = str(img_file.relative_to(config.download_path))
                 except Exception as e:
-                    url_log = url or "unknown"
-                    LOG.warning(f"Failed to fetch thumbnail '{key}' from '{url_log}'. '{e!s}'")
+                    LOG.warning(
+                        "Failed to fetch '%s' thumbnail for task '%s'.",
+                        key,
+                        task.name,
+                        extra={
+                            "task_id": task.id,
+                            "task_name": task.name,
+                            "thumbnail": key,
+                            "url": url,
+                            "error": str(e),
+                        },
+                        exc_info=True,
+                    )
                     continue
         except Exception as e:
-            LOG.warning(f"Failed to fetch thumbnails. '{e!s}'")
+            LOG.warning(
+                "Failed to fetch metadata thumbnails for task '%s'.",
+                task.name,
+                extra={"task_id": task.id, "task_name": task.name, "error": str(e)},
+                exc_info=True,
+            )
 
         return web.json_response(data=info, status=web.HTTPOk.status_code, dumps=encoder.encode)
     except ValueError as e:

app/features/tasks/service.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
-import logging
 from typing import TYPE_CHECKING
 
 from app.features.core.schemas import CEAction, CEFeature, ConfigEvent
 from app.features.tasks.models import TaskModel
 from app.features.tasks.utils import cron_time
 from app.library.Events import Event, EventBus, Events
+from app.library.log import get_logger
 from app.library.Scheduler import Scheduler
 from app.library.Services import Services
 from app.library.Singleton import Singleton
@@ -14,7 +14,7 @@ from app.library.Singleton import Singleton
 if TYPE_CHECKING:
     from aiohttp import web
 
-LOG: logging.Logger = logging.getLogger("tasks.service")
+LOG = get_logger()
 
 
 class Tasks(metaclass=Singleton):
@@ -51,7 +51,7 @@ class Tasks(metaclass=Singleton):
         pass
 
     async def _load_tasks(self) -> None:
-        tasks = await self._repo.list()
+        tasks = await self._repo.all()
 
         for task in tasks:
             if not task.timer or not task.enabled:
@@ -59,10 +59,23 @@ class Tasks(metaclass=Singleton):
 
             try:
                 self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=f"task-cronjob-{task.id}")
-                LOG.info(f"Task '{task.id}: {task.name}' queued to be executed '{cron_time(task.timer)}'.")
+                LOG.info(
+                    "Queued task '%s' to run at '%s'.",
+                    task.name,
+                    cron_time(task.timer),
+                    extra={"task_id": task.id, "task_name": task.name, "timer": task.timer},
+                )
             except Exception as e:
-                LOG.exception(e)
-                LOG.error(f"Failed to queue task '{task.name}'. '{e!s}'.")
+                LOG.exception(
+                    "Failed to queue task '%s'.",
+                    task.name,
+                    extra={
+                        "task_id": task.id,
+                        "task_name": task.name,
+                        "timer": task.timer,
+                        "exception_type": type(e).__name__,
+                    },
+                )
 
     async def _init_handlers_service(self, scheduler) -> None:
         """Initialize the handlers service after migrations."""
@@ -95,7 +108,12 @@ class Tasks(metaclass=Singleton):
 
             if task.timer and task.enabled:
                 self._scheduler.add(timer=task.timer, func=self._runner, args=(task,), id=task_id)
-                LOG.info(f"Task '{task.id}: {task.name}' queued to be executed '{cron_time(task.timer)}'.")
+                LOG.info(
+                    "Queued task '%s' to run at '%s'.",
+                    task.name,
+                    cron_time(task.timer),
+                    extra={"task_id": task.id, "task_name": task.name, "timer": task.timer},
+                )
 
     async def _runner(self, task: TaskModel) -> None:
         """
@@ -113,17 +131,29 @@ class Tasks(metaclass=Singleton):
         from app.library.ItemDTO import Item
 
         timeNow: str = datetime.now(UTC).isoformat()
+        task_id = task.id
+        task_name = task.name
         try:
-            if not (task := await self._repo.get(task.id)):
-                LOG.info(f"Task '{task.name}' no longer exists.")
+            current_task = await self._repo.get(task_id)
+            if not current_task:
+                LOG.info("Task '%s' no longer exists.", task_name, extra={"task_id": task_id, "task_name": task_name})
                 return
+            task = current_task
 
             if not task.enabled:
-                LOG.debug(f"Task '{task.name}' is disabled. Skipping execution.")
+                LOG.debug(
+                    "Task '%s' is disabled. Skipping execution.",
+                    task.name,
+                    extra={"task_id": task.id, "task_name": task.name},
+                )
                 return
 
             if not task.url:
-                LOG.error(f"Failed to dispatch '{task.name}'. No URL found.")
+                LOG.error(
+                    "Failed to dispatch task '%s' because it has no URL.",
+                    task.name,
+                    extra={"task_id": task.id, "task_name": task.name},
+                )
                 return
 
             started: float = time.time()
@@ -156,7 +186,19 @@ class Tasks(metaclass=Singleton):
 
             timeNow = datetime.now(UTC).isoformat()
             ended: float = time.time()
-            LOG.info(f"Task '{task.name}' completed at '{timeNow}' took '{ended - started:.2f}' seconds.")
+            LOG.info(
+                "Task '%s' completed in %.2f seconds.",
+                task.name,
+                ended - started,
+                extra={
+                    "task_id": task.id,
+                    "task_name": task.name,
+                    "url": task.url,
+                    "preset": preset,
+                    "elapsed_s": round(ended - started, 2),
+                    "status": status.get("status") if isinstance(status, dict) else None,
+                },
+            )
 
             notify.emit(
                 Events.TASK_DISPATCHED,
@@ -171,7 +213,17 @@ class Tasks(metaclass=Singleton):
                 message=f"Task '{task.name}' completed in '{ended - started:.2f}'.",
             )
         except Exception as e:
-            LOG.error(f"Failed to execute '{task.name}' at '{timeNow}'. '{e!s}'.")
+            LOG.exception(
+                "Failed to execute scheduled task '%s'.",
+                task.name,
+                extra={
+                    "task_id": task.id,
+                    "task_name": task.name,
+                    "url": task.url,
+                    "time": timeNow,
+                    "exception_type": type(e).__name__,
+                },
+            )
             EventBus.get_instance().emit(
                 Events.LOG_ERROR,
                 data={"preset": task.preset},

app/features/tasks/tests/test_tasks_repository.py

@@ -5,28 +5,25 @@ from __future__ import annotations
 import pytest
 import pytest_asyncio
 
-from app.features.tasks.models import TaskModel
 from app.features.tasks.repository import TasksRepository
 from app.library.sqlite_store import SqliteStore
+from app.tests.helpers import make_in_memory_db_path
 
 
 @pytest_asyncio.fixture
-async def repo(tmp_path):
+async def repo():
     """Provide a fresh repository instance with initialized database for each test."""
     TasksRepository._reset_singleton()
     SqliteStore._reset_singleton()
 
-    store = SqliteStore(db_path=str(":memory:"))
+    store = SqliteStore(db_path=make_in_memory_db_path("tasks-repository"))
     await store.get_connection()
 
     repository = TasksRepository.get_instance()
 
     yield repository
 
-    if store._conn:
-        await store._conn.close()
-    if store._engine:
-        await store._engine.dispose()
+    await store.close()
 
     TasksRepository._reset_singleton()
     SqliteStore._reset_singleton()
@@ -35,25 +32,6 @@ async def repo(tmp_path):
 class TestTasksRepository:
     """Test suite for TasksRepository database operations."""
 
-    @pytest.mark.asyncio
-    async def test_repository_singleton(self, repo):
-        """Verify repository follows singleton pattern."""
-        instance1 = TasksRepository.get_instance()
-        instance2 = TasksRepository.get_instance()
-        assert instance1 is instance2, "Should return same singleton instance"
-
-    @pytest.mark.asyncio
-    async def test_list_empty(self, repo):
-        """List returns empty when no tasks exist."""
-        tasks = await repo.list()
-        assert tasks == [], "Should return empty list when no tasks"
-
-    @pytest.mark.asyncio
-    async def test_count_empty(self, repo):
-        """Count returns 0 when no tasks exist."""
-        count = await repo.count()
-        assert count == 0, "Should return 0 when no tasks exist"
-
     @pytest.mark.asyncio
     async def test_create_task(self, repo):
         """Create task with valid data."""
@@ -282,7 +260,7 @@ class TestTasksRepository:
         await repo.create({"name": "Alice", "url": "https://example.com"})
         await repo.create({"name": "Bob", "url": "https://example.com"})
 
-        items = await repo.list()
+        items = await repo.all()
 
         assert items[0].name == "Alice", "Should be alphabetically first"
         assert items[1].name == "Bob", "Should be alphabetically second"

app/features/tasks/tests/test_tasks_router.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+import pytest_asyncio
+from aiohttp import web
+from aiohttp.test_utils import make_mocked_request
+
+from app.features.tasks import router
+from app.features.tasks.definitions.results import TaskFailure, TaskResult
+from app.features.tasks.repository import TasksRepository
+from app.library.encoder import Encoder
+from app.library.sqlite_store import SqliteStore
+from app.tests.helpers import make_in_memory_db_path
+
+
+@pytest_asyncio.fixture
+async def repo():
+    TasksRepository._reset_singleton()
+    SqliteStore._reset_singleton()
+
+    store = SqliteStore(db_path=make_in_memory_db_path("tasks-router"))
+    await store.get_connection()
+
+    repository = TasksRepository.get_instance()
+
+    yield repository
+
+    await store.close()
+    TasksRepository._reset_singleton()
+    SqliteStore._reset_singleton()
+
+
+@pytest.fixture(autouse=True)
+def patch_get_info(monkeypatch: pytest.MonkeyPatch) -> None:
+    async def _fake_get_info(_url: str, _preset: str) -> tuple[None, None]:
+        return (None, None)
+
+    monkeypatch.setattr(router, "_get_info", _fake_get_info)
+
+
+def _json_request(method: str, path: str, payload: object, *, match_info: dict[str, str] | None = None) -> web.Request:
+    mock_request: Any = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace())
+
+    async def _json() -> object:
+        return payload
+
+    mock_request.json = _json
+    return mock_request
+
+
+class _Notify:
+    def emit(self, *_args, **_kwargs) -> None:
+        return None
+
+
+class _Handler:
+    def __init__(self, matched: bool | dict[str, bool]) -> None:
+        self._matched = matched
+
+    async def inspect(self, *, url: str, preset: str | None = None, static_only: bool = False, **_kwargs):
+        del preset, static_only
+
+        if isinstance(self._matched, dict):
+            matched = self._matched.get(url, False)
+        else:
+            matched = self._matched
+
+        if matched:
+            return TaskResult(metadata={"matched": True, "handler": "TestHandler"})
+
+        return TaskFailure(message="No handler", metadata={"matched": False, "handler": None})
+
+
+@pytest.mark.asyncio
+async def test_add_requires_timer_without_handler(repo) -> None:
+    request = _json_request(
+        "POST",
+        "/api/tasks/",
+        {"name": "No Timer", "url": "https://example.com/channel"},
+    )
+
+    response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=False))
+
+    assert response.status == web.HTTPBadRequest.status_code
+    assert b"requires a timer" in response.body
+    assert await repo.all() == []
+
+
+@pytest.mark.asyncio
+async def test_add_all_or_nothing(repo) -> None:
+    request = _json_request(
+        "POST",
+        "/api/tasks/",
+        [
+            {"name": "First", "url": "https://example.com/first"},
+            {"url": "https://example.com/second"},
+        ],
+    )
+
+    response = await router.tasks_add(
+        request,
+        repo,
+        Encoder(),
+        _Notify(),
+        _Handler({"https://example.com/first": True, "https://example.com/second": False}),
+    )
+
+    assert response.status == web.HTTPBadRequest.status_code
+    assert b"requires a timer" in response.body
+    assert await repo.all() == []
+
+
+@pytest.mark.asyncio
+async def test_add_allows_handler_only(repo) -> None:
+    request = _json_request(
+        "POST",
+        "/api/tasks/",
+        {"name": "Handler Only", "url": "https://example.com/feed"},
+    )
+
+    response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=True))
+
+    assert response.status == web.HTTPOk.status_code
+    items = await repo.all()
+    assert len(items) == 1
+    assert items[0].name == "Handler Only"
+
+
+@pytest.mark.asyncio
+async def test_update_requires_timer_without_handler(repo) -> None:
+    item = await repo.create({"name": "Needs Timer", "url": "https://example.com/a", "timer": "0 0 * * *"})
+
+    request = _json_request(
+        "PUT",
+        f"/api/tasks/{item.id}",
+        {"name": item.name, "url": item.url, "timer": "", "preset": "", "folder": "", "template": "", "cli": ""},
+        match_info={"id": str(item.id)},
+    )
+
+    response = await router.tasks_update(request, repo, Encoder(), _Notify(), _Handler(matched=False))
+
+    assert response.status == web.HTTPBadRequest.status_code
+    refreshed = await repo.get(item.id)
+    assert refreshed is not None
+    assert refreshed.timer == "0 0 * * *"
+
+
+@pytest.mark.asyncio
+async def test_patch_requires_timer_without_handler(repo) -> None:
+    item = await repo.create({"name": "Patch Timer", "url": "https://example.com/b", "timer": "0 0 * * *"})
+
+    request = _json_request(
+        "PATCH",
+        f"/api/tasks/{item.id}",
+        {"timer": ""},
+        match_info={"id": str(item.id)},
+    )
+
+    response = await router.tasks_patch(request, repo, Encoder(), _Notify(), _Handler(matched=False))
+
+    assert response.status == web.HTTPBadRequest.status_code
+    refreshed = await repo.get(item.id)
+    assert refreshed is not None
+    assert refreshed.timer == "0 0 * * *"
+
+
+@pytest.mark.asyncio
+async def test_patch_requires_timer_when_handler_disabled(repo) -> None:
+    item = await repo.create({"name": "Disabled Handler", "url": "https://example.com/c", "timer": "0 0 * * *"})
+
+    request = _json_request(
+        "PATCH",
+        f"/api/tasks/{item.id}",
+        {"timer": "", "handler_enabled": False},
+        match_info={"id": str(item.id)},
+    )
+
+    response = await router.tasks_patch(request, repo, Encoder(), _Notify(), _Handler(matched=True))
+
+    assert response.status == web.HTTPBadRequest.status_code
+    assert b"handler is disabled" in response.body

app/features/tasks/utils.py

@@ -1,6 +1,6 @@
-import logging
+from app.library.log import get_logger
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 def cron_time(timer: str) -> str:
@@ -12,5 +12,9 @@ def cron_time(timer: str) -> str:
         cs = CronSim(timer, datetime.now(UTC))
         return cs.explain()
     except Exception as exc:
-        LOG.exception(exc)
+        LOG.exception(
+            "Failed to explain task timer '%s'.",
+            timer,
+            extra={"timer": timer, "exception_type": type(exc).__name__},
+        )
         return timer

app/features/ytdlp/archiver.py

@@ -1,12 +1,12 @@
-import logging
 import os
 import threading
 import time
 from pathlib import Path
 
+from app.library.log import get_logger
 from app.library.Singleton import ThreadSafe
 
-LOG: logging.Logger = logging.getLogger("Archiver")
+LOG = get_logger()
 
 
 class _Entry:
@@ -169,12 +169,21 @@ class Archiver(metaclass=ThreadSafe):
                             continue
                         ids.add(s)
             except OSError as e:
-                LOG.error(f"Failed to read archive file '{key}': {e!s}")
+                LOG.exception(
+                    "Failed to read archive file '%s'.",
+                    key,
+                    extra={"archive_file": key, "operation": "read", "exception_type": type(e).__name__},
+                )
                 ids = set()
 
             try:
                 elapsed_ms: float = (time.perf_counter() - start) * 1000.0
-                LOG.debug(f"_ensure_loaded took {elapsed_ms:.2f}ms (loaded={len(ids)})")
+                LOG.debug(
+                    "_ensure_loaded took %.2fms (loaded=%s)",
+                    elapsed_ms,
+                    len(ids),
+                    extra={"archive_file": key, "elapsed_ms": round(elapsed_ms, 2), "loaded_count": len(ids)},
+                )
             except Exception:
                 pass
 
@@ -283,7 +292,17 @@ class Archiver(metaclass=ThreadSafe):
                         f.write("".join(f"{x}\n" for x in new_ids))
 
             except OSError as e:
-                LOG.error(f"Failed to write to archive file '{key}': {e!s}")
+                LOG.exception(
+                    "Failed to write %s item(s) to archive file '%s'.",
+                    len(new_ids),
+                    key,
+                    extra={
+                        "archive_file": key,
+                        "operation": "add",
+                        "item_count": len(new_ids),
+                        "exception_type": type(e).__name__,
+                    },
+                )
                 return False
 
             entry.ids.update(new_ids)
@@ -335,7 +354,17 @@ class Archiver(metaclass=ThreadSafe):
                             continue
                         kept_lines.append(line)
             except OSError as e:
-                LOG.error(f"Failed reading archive for delete '{key}': {e!s}")
+                LOG.exception(
+                    "Failed to read archive file '%s' before deleting %s item(s).",
+                    key,
+                    len(remove_ids),
+                    extra={
+                        "archive_file": key,
+                        "operation": "delete_read",
+                        "item_count": len(remove_ids),
+                        "exception_type": type(e).__name__,
+                    },
+                )
                 return False
 
             if not changed:
@@ -351,7 +380,17 @@ class Archiver(metaclass=ThreadSafe):
                     with path.open("w", encoding="utf-8") as f:
                         f.writelines(kept_lines)
             except OSError as e:
-                LOG.error(f"Failed writing archive after delete '{key}': {e!s}")
+                LOG.exception(
+                    "Failed to write archive file '%s' after deleting %s item(s).",
+                    key,
+                    len(remove_ids),
+                    extra={
+                        "archive_file": key,
+                        "operation": "delete_write",
+                        "item_count": len(remove_ids),
+                        "exception_type": type(e).__name__,
+                    },
+                )
                 return False
 
             if entry.loaded:

app/features/ytdlp/extractor.py

@@ -1,7 +1,9 @@
 import asyncio
 import functools
 import logging
+import multiprocessing
 import pickle
+import sys
 from concurrent.futures import ProcessPoolExecutor
 from pathlib import Path
 from typing import Any
@@ -10,10 +12,51 @@ from aiohttp import web
 
 from app.features.ytdlp.utils import _DATA, LogWrapper, get_archive_id
 from app.features.ytdlp.ytdlp import YTDLP
+from app.library.log import get_logger
 from app.library.Services import Services
 from app.library.Singleton import Singleton
 
-LOG: logging.Logger = logging.getLogger("downloads.extractor")
+LOG = get_logger()
+
+LIVE_REEXTRACT_STATUSES: set[str] = {"is_live", "post_live"}
+REEXTRACT_INFO_KEY = "_ytptube_reextract"
+
+
+def _ytdlp_logger(target: logging.Logger):
+    return _YTDLPLogger(target)
+
+
+class _YTDLPLogger:
+    def __init__(self, target: logging.Logger) -> None:
+        self.target = target
+
+    def __call__(self, level: int, msg: str, *args: Any, **kwargs: Any) -> None:
+        kwargs.setdefault("stacklevel", 4)
+        if level <= logging.DEBUG and isinstance(msg, str) and msg.startswith("[debug] "):
+            self.target.debug(msg.removeprefix("[debug] "), *args, **kwargs)
+            return
+
+        if level <= logging.DEBUG:
+            self.target.info(msg, *args, **kwargs)
+            return
+
+        self.target.log(level, msg, *args, **kwargs)
+
+
+class _LogCapture:
+    def __init__(self, logs: list[str]) -> None:
+        self.logs = logs
+
+    def __call__(self, _: int, msg: str, *args: Any, **__: Any) -> None:
+        self.logs.append(msg % args if args else msg)
+
+
+def _get_process_pool_kwargs() -> dict[str, Any]:
+    """Use a fork-based pool for frozen Linux builds."""
+    if sys.platform == "linux" and getattr(sys, "frozen", False):
+        return {"mp_context": multiprocessing.get_context("fork")}
+
+    return {}
 
 
 class ExtractorConfig:
@@ -39,6 +82,17 @@ class ExtractorConfig:
         self.wait_threshold = wait_threshold
 
 
+def _sleep_timeout(config: dict[str, Any], timeout: float, budget_sleep: bool) -> float:
+    if not budget_sleep:
+        return timeout
+
+    sleep_requests = config.get("sleep_interval_requests")
+    if not isinstance(sleep_requests, int | float) or sleep_requests <= 0:
+        return timeout
+
+    return timeout + min(float(sleep_requests) * 20, 300.0)
+
+
 class ExtractorPool(metaclass=Singleton):
     """
     Manages process pool and semaphore for video information extraction.
@@ -66,7 +120,7 @@ class ExtractorPool(metaclass=Singleton):
     def attach(self, app: web.Application) -> None:
         """Attach the extractor pool to the application (no-op for now)."""
         app.on_shutdown.append(self.shutdown)
-        Services.get_instance().add(__class__.__name__, self)
+        Services.get_instance().add(type(self).__name__, self)
 
     def _ensure_initialized(self, config: ExtractorConfig) -> None:
         """
@@ -83,7 +137,7 @@ class ExtractorPool(metaclass=Singleton):
             self._semaphore = asyncio.Semaphore(config.concurrency)
 
         if self._pool is None:
-            self._pool = ProcessPoolExecutor(max_workers=config.concurrency)
+            self._pool = ProcessPoolExecutor(max_workers=config.concurrency, **_get_process_pool_kwargs())
             LOG.info("Initialized extractor process pool with %s workers", config.concurrency)
 
     def get_pool(self, config: ExtractorConfig) -> ProcessPoolExecutor:
@@ -127,7 +181,10 @@ class ExtractorPool(metaclass=Singleton):
                 self._pool.shutdown(wait=False, cancel_futures=False)
                 LOG.debug("Extractor process pool shutdown complete")
             except Exception as exc:
-                LOG.error("Error shutting down extractor process pool: %s", exc)
+                LOG.exception(
+                    "Failed to shut down the extractor process pool.",
+                    extra={"exception_type": type(exc).__name__},
+                )
             else:
                 self._pool = None
 
@@ -197,6 +254,23 @@ def _sanitize_config(config: dict[str, Any]) -> dict[str, Any]:
     return sanitized
 
 
+def needs_reextract(info: dict[str, Any]) -> bool:
+    return bool(info.get("is_live") or info.get("live_status") in LIVE_REEXTRACT_STATUSES)
+
+
+def _process_safe_info(info: dict[str, Any]) -> dict[str, Any]:
+    if needs_reextract(info):
+        info = dict(info)
+        info[REEXTRACT_INFO_KEY] = True
+        for key in ("formats", "requested_formats", "requested_downloads", "fragments"):
+            info.pop(key, None)
+
+    if _is_picklable(info):
+        return info
+
+    return YTDLP.sanitize_info(info, remove_private_keys=False)
+
+
 def extract_info_sync(
     config: dict[str, Any],
     url: str,
@@ -205,8 +279,9 @@ def extract_info_sync(
     follow_redirect: bool = False,
     sanitize_info: bool = False,
     capture_logs: int | None = None,
+    process_safe: bool = False,
     **kwargs,
-) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]:
+) -> tuple[dict[str, Any] | None, list[str]]:
     """
     Extract video information from a URL.
 
@@ -218,10 +293,11 @@ def extract_info_sync(
         follow_redirect: Follow URL redirects
         sanitize_info: Sanitize the extracted information
         capture_logs: If provided (e.g., logging.WARNING), capture logs at this level.
+        process_safe: Strip non-pickleable data for safe inter-process communication.
         **kwargs: Additional arguments
 
     Returns:
-        tuple[dict | None, list[dict]]: Extracted information and captured logs.
+        tuple[dict | None, list[str]]: Extracted information and captured logs.
 
     """
     params: dict[str, Any] = {**config, **_DATA.YTDLP_PARAMS, "simulate": True}
@@ -230,67 +306,81 @@ def extract_info_sync(
         params["verbose"] = True
         params.pop("quiet", None)
 
-    log_wrapper = LogWrapper()
+    suppress: tuple[str, ...] = ()
+    patterns = kwargs.get("suppress_logs")
+    if isinstance(patterns, list | tuple):
+        suppress = tuple(value for value in patterns if isinstance(value, str) and value)
+
+    log_wrapper = LogWrapper(suppress=suppress)
     id_dict: dict[str, str | None] = get_archive_id(url=url)
     archive_id: str | None = f".{id_dict['id']}" if id_dict.get("id") else None
-
-    log_wrapper.add_target(
-        target=logging.getLogger(f"yt-dlp{archive_id or '.extract_info'}"),
-        level=logging.DEBUG if debug else logging.WARNING,
-    )
-
-    captured_logs: list[str] = kwargs.get("captured_logs", [])
-    if capture_logs is not None:
-        log_wrapper.add_target(
-            target=lambda _, msg: captured_logs.append(msg),
-            level=capture_logs,
-            name="log-capture",
-        )
-
-    if log_wrapper.has_targets():
-        if "logger" in params:
-            log_wrapper.add_target(target=params["logger"], level=logging.DEBUG)
-
-        params["logger"] = log_wrapper
-
-    if kwargs.get("no_log", False):
-        params["logger"] = LogWrapper()
-        params["quiet"] = True
-        params["no_warnings"] = True
-
-    if no_archive and "download_archive" in params:
-        del params["download_archive"]
-
-    data: dict[str, Any] | None = YTDLP(params=params).extract_info(url, download=False)
-
-    if data and follow_redirect and "_type" in data and "url" == data["_type"]:
-        return extract_info_sync(
-            config,
-            data["url"],
-            debug=debug,
-            no_archive=no_archive,
-            follow_redirect=follow_redirect,
-            sanitize_info=sanitize_info,
-            capture_logs=capture_logs,
-            captured_logs=captured_logs,
-            **kwargs,
-        )
-
-    if not data:
-        return (data, captured_logs)
+    logger_name: str = f"yt-dlp{archive_id or '.extract_info'}"
 
     try:
-        from app.features.ytdlp.mini_filter import match_str
+        log_wrapper.add_target(
+            target=_ytdlp_logger(logging.getLogger(logger_name)),
+            level=logging.DEBUG,
+            name=logger_name,
+        )
 
-        data["is_premiere"] = match_str("media_type=video & duration & is_live", data)
-        if not data["is_premiere"]:
-            data["is_premiere"] = "video" == data.get("media_type") and "is_upcoming" == data.get("live_status")
-    except ImportError:
-        pass
+        captured_logs: list[str] = kwargs.get("captured_logs", [])
+        if capture_logs is not None:
+            log_wrapper.add_target(
+                target=_LogCapture(captured_logs),
+                level=capture_logs,
+                name="log-capture",
+            )
 
-    result = YTDLP.sanitize_info(data, remove_private_keys=True) if sanitize_info else data
+        if log_wrapper.has_targets():
+            if "logger" in params:
+                log_wrapper.add_target(target=params["logger"], level=logging.DEBUG)
 
-    return (result, captured_logs)
+            params["logger"] = log_wrapper
+
+        if kwargs.get("no_log", False):
+            params["logger"] = LogWrapper()
+            params["quiet"] = True
+            params["no_warnings"] = True
+
+        if no_archive and "download_archive" in params:
+            del params["download_archive"]
+
+        data: dict[str, Any] | None = YTDLP(params=params).extract_info(url, download=False)
+
+        if data and follow_redirect and "_type" in data and "url" == data["_type"]:
+            return extract_info_sync(
+                config,
+                data["url"],
+                debug=debug,
+                no_archive=no_archive,
+                follow_redirect=follow_redirect,
+                sanitize_info=sanitize_info,
+                capture_logs=capture_logs,
+                process_safe=process_safe,
+                captured_logs=captured_logs,
+                **kwargs,
+            )
+
+        if not data:
+            return (data, captured_logs)
+
+        try:
+            from app.features.ytdlp.mini_filter import match_str
+
+            data["is_premiere"] = match_str("media_type=video & duration & is_live", data)
+            if not data["is_premiere"]:
+                data["is_premiere"] = "video" == data.get("media_type") and "is_upcoming" == data.get("live_status")
+        except ImportError:
+            pass
+
+        result = YTDLP.sanitize_info(data, remove_private_keys=True) if sanitize_info else data
+
+        if process_safe and isinstance(result, dict):
+            result = _process_safe_info(result)
+
+        return (result, captured_logs)
+    finally:
+        logging.Logger.manager.loggerDict.pop(logger_name, None)
 
 
 async def fetch_info(
@@ -302,8 +392,9 @@ async def fetch_info(
     sanitize_info: bool = False,
     capture_logs: int | None = None,
     extractor_config: ExtractorConfig | None = None,
+    budget_sleep: bool = False,
     **kwargs,
-) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]:
+) -> tuple[dict[str, Any] | None, list[str]]:
     """
     Extract video information from a URL.
 
@@ -319,10 +410,11 @@ async def fetch_info(
         sanitize_info: Sanitize the extracted information
         capture_logs: If provided (e.g., logging.WARNING), capture logs
         extractor_config: Configuration for the extractor
+        budget_sleep: Whether to add extra timeout budget for request-sleep-heavy extraction
         **kwargs: Additional arguments
 
     Returns:
-        tuple[dict | None, list[dict]]: Extracted information and captured logs.
+        tuple[dict | None, list[str]]: Extracted information and captured logs.
 
     """
     if extractor_config is None:
@@ -342,6 +434,9 @@ async def fetch_info(
     loop = asyncio.get_running_loop()
 
     safe_config = _sanitize_config(config)
+    safe_kwargs = _sanitize_picklable(kwargs)
+    safe_kwargs.pop("process_safe", None)
+    timeout = _sleep_timeout(safe_config, extractor_config.timeout, budget_sleep)
 
     try:
         try:
@@ -359,14 +454,34 @@ async def fetch_info(
                         follow_redirect=follow_redirect,
                         sanitize_info=sanitize_info,
                         capture_logs=capture_logs,
-                        **kwargs,
+                        process_safe=True,
+                        **safe_kwargs,
                     ),
                 ),
-                timeout=extractor_config.timeout,
+                timeout=timeout,
             )
 
+        except TimeoutError:
+            raise
+
         except Exception as exc:
-            LOG.warning("extract_info process pool failed, falling back to thread pool url=%s error=%s", url, exc)
+            LOG.warning(
+                "yt-dlp extraction for '%s' fell back to the thread pool after the process pool failed.",
+                url,
+                extra={
+                    "url": url,
+                    "timeout": timeout,
+                    "concurrency": extractor_config.concurrency,
+                    "pool": "process",
+                    "fallback": "thread",
+                    "follow_redirect": follow_redirect,
+                    "no_archive": no_archive,
+                    "sanitize_info": sanitize_info,
+                    "capture_logs_level": capture_logs,
+                    "exception_type": type(exc).__name__,
+                },
+                exc_info=True,
+            )
             return await asyncio.wait_for(
                 fut=loop.run_in_executor(
                     None,
@@ -382,7 +497,7 @@ async def fetch_info(
                         **kwargs,
                     ),
                 ),
-                timeout=extractor_config.timeout,
+                timeout=timeout,
             )
     finally:
         semaphore.release()

app/features/ytdlp/mini_filter.py

@@ -1,9 +1,14 @@
+from __future__ import annotations
+
 import operator
 import re
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
-TOKEN = tuple[str, str]
-AST_NODE = tuple[str, ...]
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+type TOKEN = tuple[str, str]
+type AST_NODE = tuple[Any, ...]
 
 
 def match_str(expr: str, dct: dict) -> bool:
@@ -42,7 +47,7 @@ class MiniFilter:
     ASTNode = tuple[str, ...]
 
     # Supported string operators
-    STRING_OPERATORS: dict[str, callable] = {
+    STRING_OPERATORS: dict[str, Callable[[Any, Any], bool]] = {
         "*=": operator.contains,
         "^=": lambda attr, value: isinstance(attr, str) and attr.startswith(value),
         "$=": lambda attr, value: isinstance(attr, str) and attr.endswith(value),
@@ -50,7 +55,7 @@ class MiniFilter:
     }
 
     # Comparison operators (numeric + string)
-    COMPARISON_OPERATORS: dict[str, callable] = {
+    COMPARISON_OPERATORS: dict[str, Callable[[Any, Any], bool]] = {
         **STRING_OPERATORS,
         "<=": operator.le,
         "<": operator.lt,
@@ -60,7 +65,7 @@ class MiniFilter:
     }
 
     # Unary operators (for presence/absence checks)
-    UNARY_OPERATORS: dict[str, callable] = {
+    UNARY_OPERATORS: dict[str, Callable[[Any], bool]] = {
         "": lambda v: (v is True) if isinstance(v, bool) else (v is not None),
         "!": lambda v: (v is False) if isinstance(v, bool) else (v is None),
     }
@@ -75,9 +80,12 @@ class MiniFilter:
         self.tokens: list[TOKEN] = self._tokenize(expr)
         self.pos: int = 0
         self.ast: AST_NODE = self._parse_or()
+        if self.pos != len(self.tokens):
+            msg = f"Unexpected token {self.tokens[self.pos][1]!r}"
+            raise SyntaxError(msg)
 
     @staticmethod
-    def run(expr: str, dct: dict | bool = False) -> bool:
+    def run(expr: str, dct: dict[str, Any] | bool = False) -> bool:
         """
         Convenience method to evaluate an expression directly.
 
@@ -87,14 +95,15 @@ class MiniFilter:
         """
         return MiniFilter(expr).evaluate(dct)
 
-    def evaluate(self, dct: dict | bool = False) -> bool:
+    def evaluate(self, dct: dict[str, Any] | bool = False) -> bool:
         """
         Evaluate the parsed expression against a dictionary.
 
         :param dct: Dictionary of attributes (video metadata, etc.)
         :return:    True/False result of expression
         """
-        return self._eval(self.ast, dct)
+        data: dict[str, Any] = dct if isinstance(dct, dict) else {}
+        return self._eval(self.ast, data)
 
     def export(self) -> list[str]:
         """
@@ -115,34 +124,124 @@ class MiniFilter:
     def _tokenize(self, expr: str) -> list[TOKEN]:
         """
         Split expression into tokens (AND, OR, LPAREN, RPAREN, ATOM).
-        Ensures that OR (||) and AND (&) are recognized before ATOM.
+        Parentheses, OR, and AND are only treated as syntax outside quoted values.
         """
-        # First, let's normalize spaces around operators to make parsing easier
-        # Replace spaced operators with non-spaced ones
-        normalized_expr: str = expr
-        for op in ["<=", ">=", "<", ">", "*=", "^=", "$=", "~=", "="]:
-            # Replace "key op value" with "key op value" (removing extra spaces)
-            pattern: str = rf"([a-z_]+)\s*(!?)\s*({re.escape(op)})\s*"
-            replacement = r"\1\2\3"
-            normalized_expr = re.sub(pattern, replacement, normalized_expr)
-
-        token_spec: list[set] = [
-            (r"\|\|", "OR"),  # must come before ATOM
-            (r"&", "AND"),
-            (r"\(", "LPAREN"),
-            (r"\)", "RPAREN"),
-            (r"[^\s&|()]+", "ATOM"),  # don't allow whitespace in ATOM
-            (r"\s+", None),  # skip whitespace
-        ]
-        regex: str = "|".join(f"(?P<{name}>{pat})" for pat, name in token_spec if name)
-        master: re.Pattern[str] = re.compile(regex)
         tokens: list[TOKEN] = []
-        for m in master.finditer(normalized_expr):
-            kind: str | None = m.lastgroup
-            if kind:
-                tokens.append((kind, m.group()))
+        atom: list[str] = []
+        quote: str | None = None
+        escaped: bool = False
+        idx: int = 0
+
+        def flush_atom() -> None:
+            if not atom:
+                return
+
+            normalized_atom: str = self._normalize_atom("".join(atom))
+            if normalized_atom:
+                tokens.append(("ATOM", normalized_atom))
+            atom.clear()
+
+        while idx < len(expr):
+            char: str = expr[idx]
+
+            if quote:
+                atom.append(char)
+                if escaped:
+                    escaped = False
+                elif "\\" == char:
+                    escaped = True
+                elif quote == char:
+                    quote = None
+                idx += 1
+                continue
+
+            if escaped:
+                atom.append(char)
+                escaped = False
+                idx += 1
+                continue
+
+            if "\\" == char:
+                atom.append(char)
+                escaped = True
+                idx += 1
+                continue
+
+            if char in {'"', "'"}:
+                atom.append(char)
+                quote = char
+                idx += 1
+                continue
+
+            if expr.startswith("||", idx):
+                flush_atom()
+                tokens.append(("OR", "||"))
+                idx += 2
+                continue
+
+            if "&" == char:
+                flush_atom()
+                tokens.append(("AND", "&"))
+                idx += 1
+                continue
+
+            if "(" == char:
+                flush_atom()
+                tokens.append(("LPAREN", "("))
+                idx += 1
+                continue
+
+            if ")" == char:
+                flush_atom()
+                tokens.append(("RPAREN", ")"))
+                idx += 1
+                continue
+
+            if char.isspace() and not atom:
+                idx += 1
+                continue
+
+            atom.append(char)
+            idx += 1
+
+        if quote:
+            msg = "Unterminated quoted string in filter expression"
+            raise SyntaxError(msg)
+
+        flush_atom()
         return tokens
 
+    def _normalize_atom(self, atom: str) -> str:
+        normalized_atom: str = atom.strip()
+
+        for op in ["<=", ">=", "<", ">", "*=", "^=", "$=", "~=", "="]:
+            pattern: str = rf"([a-z_]+)\s*(!?)\s*({re.escape(op)})\s*"
+            normalized_atom = re.sub(pattern, r"\1\2\3", normalized_atom, count=1)
+
+        quote: str | None = None
+        escaped: bool = False
+        for idx, char in enumerate(normalized_atom):
+            if quote:
+                if escaped:
+                    escaped = False
+                elif "\\" == char:
+                    escaped = True
+                elif quote == char:
+                    quote = None
+                continue
+
+            if char in {'"', "'"}:
+                quote = char
+                continue
+
+            if char.isspace():
+                unexpected: str = normalized_atom[idx:].strip()
+                if unexpected:
+                    msg = f"Unexpected token {unexpected!r}"
+                    raise SyntaxError(msg)
+
+        return normalized_atom
+
     def _parse_or(self) -> AST_NODE:
         """
         Parse OR-expression: and_expr ('||' and_expr)*
@@ -197,7 +296,7 @@ class MiniFilter:
 
         raise SyntaxError("Expected " + kind)
 
-    def _eval(self, node: AST_NODE, dct: dict) -> bool:
+    def _eval(self, node: AST_NODE, dct: dict[str, Any]) -> bool:
         """
         Recursively evaluate AST node against dict.
         """
@@ -234,7 +333,9 @@ class MiniFilter:
 
         raise ValueError("Invalid AST node " + node_type)
 
-    def _match_one(self, filter_part: str, dct: dict) -> bool:
+    def _match_one(self, filter_part: str, dct: dict[str, Any]) -> bool:
+        filter_part = filter_part.replace(r"\&", "&")
+
         operator_rex: re.Pattern[str] = re.compile(
             r"""(?x)
             (?P<key>[a-z_]+)

app/features/ytdlp/outtmpl.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import re
+import secrets
+import string
+from typing import TYPE_CHECKING, Any
+
+from yt_dlp.utils._utils import STR_FORMAT_RE_TMPL, STR_FORMAT_TYPES
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+OUTTMPL_RX: re.Pattern[str] = re.compile(STR_FORMAT_RE_TMPL.format("[^)]*", f"[{STR_FORMAT_TYPES}ljhqBUDS]"))
+CALL_RX: re.Pattern[str] = re.compile(r"^(?P<name>ytp_[A-Za-z0-9_]+)(?P<args>(?::[A-Za-z0-9_]+)*)(?P<rest>.*)$")
+OPERATORS: tuple[str, ...] = (",", "&", "|", ">", "+", "-", "*")
+
+
+def random_text(args: tuple[str, ...], _info_dict: dict[str, Any], _state: dict[str, Any]) -> str:
+    if not args:
+        msg = "ytp_random requires a length argument. Use %(ytp_random:<length>)s."
+        raise ValueError(msg)
+
+    if len(args) > 2:
+        msg = "ytp_random accepts at most 2 arguments: length and optional mode."
+        raise ValueError(msg)
+
+    try:
+        length = int(args[0])
+    except ValueError as exc:
+        msg: str = f"ytp_random length must be an integer, got {args[0]!r}."
+        raise ValueError(msg) from exc
+
+    if length < 1:
+        msg = "ytp_random length must be greater than 0."
+        raise ValueError(msg)
+
+    mode = args[1].lower() if len(args) > 1 else "mixed"
+    if mode in ("mixed", "m"):
+        alphabet = string.ascii_letters + string.digits
+    elif mode in ("d", "digit", "digits", "int", "ints", "number", "numbers"):
+        alphabet = string.digits
+    elif mode in ("s", "str", "string", "strings", "alpha", "letter", "letters"):
+        alphabet = string.ascii_letters
+    else:
+        msg = f"ytp_random mode must be one of mixed, d, or s. Got {args[1]!r}."
+        raise ValueError(msg)
+
+    return "".join(secrets.choice(alphabet) for _ in range(length))
+
+
+CALLS: dict[str, Callable[[tuple[str, ...], dict[str, Any], dict[str, Any]], Any]] = {
+    "ytp_random": random_text,
+}
+
+
+def split_call(key: str) -> tuple[str, tuple[str, ...], str] | None:
+    if not key.startswith("ytp_"):
+        return None
+
+    if not (match := CALL_RX.match(key)):
+        msg = f"Invalid YTPTube output template callable {key!r}."
+        raise ValueError(msg)
+
+    name: str | Any = match.group("name")
+    rest: str | Any = match.group("rest") or ""
+    if rest and rest[0] not in OPERATORS:
+        msg = f"Invalid YTPTube output template callable {key!r}."
+        raise ValueError(msg)
+
+    if name not in CALLS:
+        msg = f"Unsupported YTPTube output template callable {name!r}."
+        raise ValueError(msg)
+
+    raw_args: str | Any = match.group("args")
+    args: tuple[str | Any, ...] = tuple(part for part in raw_args.split(":") if part)
+    return name, args, rest
+
+
+def rewrite_outtmpl(
+    outtmpl: str,
+    info_dict: dict[str, Any],
+    cache: dict[str, Any] | None = None,
+) -> tuple[str, dict[str, Any]]:
+    if "%(ytp_" not in outtmpl:
+        return outtmpl, info_dict
+
+    state: dict[str, Any] = {}
+    values: dict[str, Any] = {} if cache is None else cache
+    fields: dict[str, str] = {}
+    enriched = dict(info_dict)
+
+    def replace(match: re.Match[str]) -> str:
+        if not (key := match.group("key")):
+            return match.group(0)
+
+        parsed = split_call(key)
+        if not parsed:
+            return match.group(0)
+
+        name, args, rest = parsed
+        call_key: str = ":".join((name, *args)) if args else name
+
+        if call_key not in values:
+            values[call_key] = CALLS[name](args, enriched, state)
+
+        if call_key not in fields:
+            fields[call_key] = f"__ytptube_outtmpl_{len(fields)}"
+
+        synthetic_key: str = fields[call_key]
+        enriched[synthetic_key] = values[call_key]
+        return f"{match.group('prefix')}%({synthetic_key}{rest}){match.group('format')}"
+
+    return OUTTMPL_RX.sub(replace, outtmpl), enriched

app/features/ytdlp/patches.py

@@ -0,0 +1,109 @@
+import subprocess
+import sys
+from typing import Any
+
+from app.library.log import get_logger
+
+LOG = get_logger()
+
+
+def patch_metadataparser() -> None:
+    """
+    Patches yt_dlp MetadataParserPP action to handle subprocess pickling issues.
+    """
+    try:
+        from yt_dlp.postprocessor.metadataparser import MetadataParserPP
+        from yt_dlp.utils import Namespace
+    except Exception as exc:
+        LOG.warning(
+            "Unable to import yt-dlp metadata parser for patching: %s",
+            exc,
+            extra={"patch": "metadata_parser", "exception_type": type(exc).__name__},
+            exc_info=True,
+        )
+        return
+
+    if getattr(MetadataParserPP.Actions, "_ytptube_patched", False):
+        return
+
+    class _ActionNS(Namespace):
+        _ACTIONS_STR: list[str] = []
+
+        @staticmethod
+        def _get_name(func) -> str | None:
+            if not callable(func):
+                return None
+
+            target = getattr(func, "__func__", func)
+            module_name = getattr(target, "__module__", None)
+            qual_name = getattr(target, "__qualname__", getattr(target, "__name__", None))
+
+            return f"{module_name}.{qual_name}" if module_name and qual_name else None
+
+        def __contains__(self, candidate: object) -> bool:
+            if candidate in self.__dict__.values():
+                return True
+
+            if func_name := _ActionNS._get_name(candidate):
+                if len(_ActionNS._ACTIONS_STR) < 1:
+                    _ActionNS._ACTIONS_STR.extend(
+                        [value for value in (_ActionNS._get_name(value) for value in self.__dict__.values()) if value]
+                    )
+
+                return func_name in _ActionNS._ACTIONS_STR
+
+            return False
+
+    actions_dict: dict[str, Any] = dict(MetadataParserPP.Actions.items_)
+    MetadataParserPP.Actions = _ActionNS(**actions_dict)
+    MetadataParserPP.Actions._ytptube_patched = True
+    LOG.debug("MetadataParserPP action namespace patch applied successfully.")
+
+
+def patch_windows_popen_wait() -> None:
+    if sys.platform != "win32":
+        return
+
+    try:
+        from yt_dlp.utils import Popen
+    except Exception as exc:
+        LOG.warning(
+            "Unable to import yt-dlp Popen for patching: %s",
+            exc,
+            extra={"patch": "windows_popen_wait", "exception_type": type(exc).__name__},
+            exc_info=True,
+        )
+        return
+
+    if getattr(Popen, "_ytptube_wait_patched", False):
+        return
+
+    original_wait = Popen.wait
+
+    # Windows subprocess waits can swallow the synthetic interrupt we use to
+    # stop live downloads, especially while yt-dlp is blocked on ffmpeg.
+    def interruptible_wait(self, timeout=None):
+        if timeout is not None:
+            return original_wait(self, timeout=timeout)
+
+        while True:
+            try:
+                return original_wait(self, timeout=0.1)
+            except subprocess.TimeoutExpired:
+                continue
+
+    Popen.wait = interruptible_wait
+    Popen._ytptube_wait_patched = True
+    LOG.debug("yt_dlp Popen.wait Windows patch applied successfully.")
+
+
+def apply_ytdlp_patches() -> None:
+    try:
+        patch_metadataparser()
+    except Exception as exc:
+        LOG.debug("Metadata parser patch failed to apply: %s", exc)
+
+    try:
+        patch_windows_popen_wait()
+    except Exception as exc:
+        LOG.debug("Windows Popen wait patch failed to apply: %s", exc)

app/features/ytdlp/router.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import logging
 import time
@@ -17,10 +18,11 @@ from app.library.cache import Cache
 from app.library.config import Config
 from app.library.encoder import Encoder
 from app.library.ItemDTO import Item
+from app.library.log import get_logger
 from app.library.router import route
 from app.library.Utils import validate_url
 
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 def _get_preset_archive(preset: str) -> str | None:
@@ -36,7 +38,12 @@ def _get_preset_archive(preset: str) -> str | None:
     try:
         opts: dict = YTDLPOpts.get_instance().preset(preset).get_all()
     except Exception as e:
-        LOG.error(f"Failed to build yt-dlp opts for preset '{preset}'. {e!s}")
+        LOG.exception(
+            "Failed to build yt-dlp options for preset '%s': %s.",
+            preset,
+            e,
+            extra={"preset": preset, "exception_type": type(e).__name__},
+        )
         return None
 
     if not (archive_file := opts.get("download_archive")):
@@ -124,7 +131,19 @@ async def archiver_get(request: Request) -> Response:
             data={"file": archive_file, "items": data, "count": len(data)}, status=web.HTTPOk.status_code
         )
     except Exception as e:
-        LOG.exception(e)
+        LOG.exception(
+            "Failed to read archive file '%s' for preset '%s': %s.",
+            archive_file,
+            preset,
+            e,
+            extra={
+                "route": "api/archiver/",
+                "action": "read_archive",
+                "preset": preset,
+                "archive_file": archive_file,
+                "ids": ids,
+            },
+        )
         return web.json_response(
             data={"error": f"Failed to read archive file for preset '{preset}'.", "message": str(e)},
             status=web.HTTPInternalServerError.status_code,
@@ -170,7 +189,21 @@ async def archiver_add(request: Request) -> Response:
             status=web.HTTPOk.status_code if status else web.HTTPNotModified.status_code,
         )
     except Exception as e:
-        LOG.exception(e)
+        LOG.exception(
+            "Failed to add %s item(s) to archive file '%s' for preset '%s': %s.",
+            len(items),
+            archive_file,
+            preset,
+            e,
+            extra={
+                "route": "api/archiver/",
+                "action": "add_archive_items",
+                "preset": preset,
+                "archive_file": archive_file,
+                "item_count": len(items),
+                "skip_check": skip_check,
+            },
+        )
         return web.json_response(
             data={"error": f"Failed to add items to archive for preset '{preset}'.", "message": str(e)},
             status=web.HTTPInternalServerError.status_code,
@@ -214,7 +247,20 @@ async def archiver_delete(request: Request) -> Response:
             status=web.HTTPOk.status_code if status else web.HTTPNotModified.status_code,
         )
     except Exception as e:
-        LOG.exception(e)
+        LOG.exception(
+            "Failed to delete %s item(s) from archive file '%s' for preset '%s': %s.",
+            len(items),
+            archive_file,
+            preset,
+            e,
+            extra={
+                "route": "api/archiver/",
+                "action": "delete_archive_items",
+                "preset": preset,
+                "archive_file": archive_file,
+                "item_count": len(items),
+            },
+        )
         return web.json_response(
             data={"error": f"Failed to delete items from archive for preset '{preset}'.", "message": str(e)},
             status=web.HTTPInternalServerError.status_code,
@@ -240,7 +286,7 @@ async def convert(request: Request) -> Response:
         return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code)
 
     try:
-        response = {"opts": {}, "output_template": None, "download_path": None}
+        response: dict[str, Any] = {"opts": {}, "output_template": None, "download_path": None}
 
         data = arg_converter(args, dumps=True)
 
@@ -301,7 +347,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
         )
 
     try:
-        validate_url(url, allow_internal=config.allow_internal_urls)
+        await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
     except ValueError as e:
         return web.json_response(
             data={"status": False, "message": str(e), "error": str(e)},
@@ -338,6 +384,8 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
 
         if cache.has(key) and not request.query.get("force", False):
             data: Any | None = cache.get(key)
+            if data is None:
+                data = {}
             data["_cached"] = {
                 "status": "hit",
                 "preset": preset,
@@ -347,7 +395,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
                 "ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(),
                 "expires": data.get("_cached", {}).get("expires", time.time() + 300),
             }
-            return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
+            return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
 
         ytdlp_opts: dict = opts.get_all()
 
@@ -404,10 +452,21 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
 
         cache.set(key=key, value=data, ttl=300)
 
-        return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
+        return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
     except Exception as e:
-        LOG.exception(e)
-        LOG.error(f"Error encountered while getting video info for '{url}'. '{e!s}'.")
+        LOG.exception(
+            "Failed to get video info for '%s': %s.",
+            url,
+            e,
+            extra={
+                "route": "api/yt-dlp/url/info/",
+                "action": "get_video_info",
+                "url": url,
+                "preset": preset,
+                "cache_key": key if "key" in locals() else None,
+                "has_cli_args": bool(cli_args),
+            },
+        )
         return web.json_response(
             data={
                 "error": "failed to get video info.",
@@ -429,7 +488,7 @@ async def get_options() -> Response:
     """
     from app.features.ytdlp.ytdlp import ytdlp_options
 
-    return web.json_response(body=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
+    return web.json_response(text=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
 
 
 @route("POST", "api/yt-dlp/archive_id/", "get_archive_ids")
@@ -451,9 +510,13 @@ async def get_archive_ids(request: Request, config: Config) -> Response:
     response = []
 
     for i, url in enumerate(data):
-        dct = {"index": i, "url": url}
+        dct: dict[str, Any] = {"index": i, "url": url}
+        if not isinstance(url, str):
+            dct.update({"id": None, "ie_key": None, "archive_id": None, "error": "URL must be a string."})
+            response.append(dct)
+            continue
         try:
-            validate_url(url, allow_internal=config.allow_internal_urls)
+            await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
             dct.update(get_archive_id(url))
         except ValueError as e:
             dct.update({"id": None, "ie_key": None, "archive_id": None, "error": str(e)})
@@ -495,7 +558,19 @@ async def make_command(request: Request, config: Config, encoder: Encoder) -> Re
     try:
         command, info = YTDLPCli(item=it, config=config).build()
     except Exception as e:
-        LOG.exception(e)
+        LOG.exception(
+            "Failed to build yt-dlp command for '%s': %s.",
+            it.url,
+            e,
+            extra={
+                "route": "api/yt-dlp/command/",
+                "action": "build_command",
+                "url": it.url,
+                "preset": it.preset,
+                "has_cookies": bool(it.cookies),
+                "exception_type": type(e).__name__,
+            },
+        )
         return web.json_response(
             data={"error": "Failed to build CLI command"},
             status=web.HTTPBadRequest.status_code,

app/features/ytdlp/tests/test_ytdlp_extractor.py

@@ -1,6 +1,60 @@
+import logging
+import pickle
 from unittest.mock import MagicMock, patch
 
-from app.features.ytdlp.extractor import extract_info_sync
+from app.features.ytdlp.extractor import (
+    ExtractorConfig,
+    ExtractorPool,
+    REEXTRACT_INFO_KEY,
+    _LogCapture,
+    _get_process_pool_kwargs,
+    _process_safe_info,
+    _ytdlp_logger,
+    extract_info_sync,
+)
+from app.features.ytdlp.utils import LogWrapper
+
+
+class TestProcessPoolConfiguration:
+    def setup_method(self):
+        ExtractorPool._reset_singleton()
+
+    def test_uses_fork_context_for_frozen_linux(self, monkeypatch):
+        monkeypatch.setattr("app.features.ytdlp.extractor.sys.platform", "linux")
+        monkeypatch.setattr("app.features.ytdlp.extractor.sys.frozen", True, raising=False)
+
+        context = object()
+        get_context = MagicMock(return_value=context)
+        monkeypatch.setattr("app.features.ytdlp.extractor.multiprocessing.get_context", get_context)
+
+        kwargs = _get_process_pool_kwargs()
+
+        assert kwargs == {"mp_context": context}
+        get_context.assert_called_once_with("fork")
+
+    def test_default_context_linux(self, monkeypatch):
+        monkeypatch.setattr("app.features.ytdlp.extractor.sys.platform", "linux")
+        monkeypatch.delattr("app.features.ytdlp.extractor.sys.frozen", raising=False)
+
+        get_context = MagicMock()
+        monkeypatch.setattr("app.features.ytdlp.extractor.multiprocessing.get_context", get_context)
+
+        assert _get_process_pool_kwargs() == {}
+        get_context.assert_not_called()
+
+    def test_init_pool_kwargs(self, monkeypatch):
+        context = object()
+        monkeypatch.setattr("app.features.ytdlp.extractor._get_process_pool_kwargs", lambda: {"mp_context": context})
+
+        executor = MagicMock()
+        executor_cls = MagicMock(return_value=executor)
+        monkeypatch.setattr("app.features.ytdlp.extractor.ProcessPoolExecutor", executor_cls)
+
+        pool = ExtractorPool.get_instance()
+        pool._ensure_initialized(ExtractorConfig(concurrency=3))
+
+        executor_cls.assert_called_once_with(max_workers=3, mp_context=context)
+        assert pool.get_pool(ExtractorConfig(concurrency=3)) is executor
 
 
 class TestExtractInfo:
@@ -34,3 +88,145 @@ class TestExtractInfo:
         (result, logs) = extract_info_sync(config, url, debug=True)
         assert isinstance(result, dict), "Result should be a dictionary"
         assert isinstance(logs, list), "Logs should be a list"
+
+    @patch("app.features.ytdlp.extractor.YTDLP")
+    def test_extract_info_mirrors_debug_to_console(self, mock_ytdlp_class):
+        seen: list[tuple[int, str]] = []
+
+        def fake_extract_info(url, download=False):  # noqa: ARG001
+            logger = mock_ytdlp_class.call_args.kwargs["params"]["logger"]
+            logger.debug("[generic_browser] Using remote browser for https://example.com/video")
+            logger.debug("[debug] [generic_browser] Loading page https://example.com/video")
+            logger.warning("[generic_browser] Browser fallback warning")
+            return {"title": "Test Video", "id": "test123"}
+
+        mock_ytdlp = MagicMock()
+        mock_ytdlp.extract_info.side_effect = fake_extract_info
+        mock_ytdlp_class.return_value = mock_ytdlp
+
+        logger = logging.getLogger("yt-dlp.extract_info")
+        with patch.object(
+            logger, "info", side_effect=lambda msg, *a, **k: seen.append((logging.INFO, msg % a if a else msg))
+        ):
+            with patch.object(
+                logger,
+                "debug",
+                side_effect=lambda msg, *a, **k: seen.append((logging.DEBUG, msg % a if a else msg)),
+            ):
+                with patch.object(
+                    logger,
+                    "log",
+                    side_effect=lambda level, msg, *a, **k: seen.append((level, msg % a if a else msg)),
+                ):
+                    (result, logs) = extract_info_sync(
+                        {}, "https://example.com/video", debug=True, capture_logs=logging.WARNING
+                    )
+
+        assert result is not None
+        assert result["id"] == "test123"
+        assert logs == ["[generic_browser] Browser fallback warning"]
+        assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
+        assert (logging.DEBUG, "[generic_browser] Loading page https://example.com/video") in seen
+        assert (logging.WARNING, "[generic_browser] Browser fallback warning") in seen
+
+    @patch("app.features.ytdlp.extractor.YTDLP")
+    def test_extract_info_mirrors_screen_logs_without_debug(self, mock_ytdlp_class):
+        seen: list[tuple[int, str]] = []
+
+        def fake_extract_info(url, download=False):  # noqa: ARG001
+            logger = mock_ytdlp_class.call_args.kwargs["params"]["logger"]
+            logger.debug("[generic_browser] Using remote browser for https://example.com/video")
+            logger.warning("[generic_browser] Browser fallback warning")
+            return {"title": "Test Video", "id": "test123"}
+
+        mock_ytdlp = MagicMock()
+        mock_ytdlp.extract_info.side_effect = fake_extract_info
+        mock_ytdlp_class.return_value = mock_ytdlp
+
+        logger = logging.getLogger("yt-dlp.extract_info")
+        with patch.object(
+            logger, "info", side_effect=lambda msg, *a, **k: seen.append((logging.INFO, msg % a if a else msg))
+        ):
+            with patch.object(
+                logger,
+                "log",
+                side_effect=lambda level, msg, *a, **k: seen.append((level, msg % a if a else msg)),
+            ):
+                (result, logs) = extract_info_sync(
+                    {}, "https://example.com/video", debug=False, capture_logs=logging.WARNING
+                )
+
+        assert result is not None
+        assert result["id"] == "test123"
+        assert logs == ["[generic_browser] Browser fallback warning"]
+        assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
+        assert (logging.WARNING, "[generic_browser] Browser fallback warning") in seen
+
+    def test_process_safe_live(self) -> None:
+        data = {
+            "id": "live-id",
+            "is_live": True,
+            "formats": [{"format_id": "dash", "fragments": ({"url": "https://example.test/sq/1"} for _ in range(1))}],
+            "requested_formats": [{"format_id": "dash"}],
+        }
+
+        result = _process_safe_info(data)
+
+        assert result[REEXTRACT_INFO_KEY] is True
+        assert "formats" not in result
+        assert "requested_formats" not in result
+        pickle.dumps(result)
+
+    def test_process_safe_post_live(self) -> None:
+        data = {
+            "id": "post-live-id",
+            "live_status": "post_live",
+            "formats": [{"format_id": "dash", "fragments": ({"url": "https://example.test/sq/1"} for _ in range(1))}],
+        }
+
+        result = _process_safe_info(data)
+
+        assert result[REEXTRACT_INFO_KEY] is True
+        assert "formats" not in result
+        pickle.dumps(result)
+
+    def test_process_safe_lazy(self) -> None:
+        from yt_dlp.utils import LazyList
+
+        data = {"id": "video-id", "formats": LazyList({"format_id": str(i)} for i in range(2))}
+
+        result = _process_safe_info(data)
+
+        assert result["formats"] == [{"format_id": "0"}, {"format_id": "1"}]
+        pickle.dumps(result)
+
+
+class TestYtdlpLogger:
+    def test_debug_prefix_uses_debug(self) -> None:
+        logger = MagicMock()
+
+        _ytdlp_logger(logger)(logging.DEBUG, "[debug] hello")
+
+        logger.debug.assert_called_once_with("hello", stacklevel=4)
+
+    def test_screen_style_debug_uses_info(self) -> None:
+        logger = MagicMock()
+
+        _ytdlp_logger(logger)(logging.DEBUG, "screen line")
+
+        logger.info.assert_called_once_with("screen line", stacklevel=4)
+
+    def test_targets_are_picklable(self) -> None:
+        logs: list[str] = []
+        wrapper = LogWrapper()
+        wrapper.add_target(_ytdlp_logger(logging.getLogger("yt-dlp.test")), level=logging.DEBUG, name="yt-dlp.test")
+        wrapper.add_target(_LogCapture(logs), level=logging.WARNING, name="log-capture")
+
+        pickle.dumps(wrapper)
+
+    def test_capture_formats_args(self) -> None:
+        logs: list[str] = []
+
+        _LogCapture(logs)(logging.WARNING, "hello %s", "world")
+
+        assert logs == ["hello world"]

app/features/ytdlp/tests/test_ytdlp_mini_filter.py

@@ -1,6 +1,8 @@
 import unittest
 from typing import Any
 
+import pytest
+
 from app.features.ytdlp.mini_filter import MiniFilter
 
 
@@ -193,7 +195,7 @@ class TestMiniFilter(unittest.TestCase):
         self._test("filesize>=1GiB", {"filesize": 2**30}, expected_result=True, test_name="filesize_1gib_positive")
         self._test("filesize>=1GiB", {"filesize": 1000000}, expected_result=False, test_name="filesize_1gib_negative")
 
-    def test_complex_duration_units_with_or_and_grouping(self):
+    def test_duration_or_grouping(self):
         """Test complex expressions with duration units, OR operations, and grouping. Skip yt-dlp due to known parsing bugs."""
         # Test grouping with duration units
         expr = "(filesize>1MB & duration<10m) || uploader='BBC'"
@@ -274,6 +276,65 @@ class TestMiniFilter(unittest.TestCase):
         self._test("uploader^='CNN'", d, expected_result=False, test_name="string_startswith_negative")
         self._test("uploader$='BBC'", d, expected_result=False, test_name="string_endswith_negative")
 
+    def test_regex_inline_flags_tokenization(self):
+        expr = "artist ~= '(?i)^[0-9A-L]'"
+
+        parser = MiniFilter(expr)
+
+        assert parser.tokens == [("ATOM", "artist~='(?i)^[0-9A-L]'")]
+        assert parser.ast == ("ATOM", "artist~='(?i)^[0-9A-L]'")
+        assert parser.export() == ["artist~='(?i)^[0-9A-L]'"]
+
+        self._test(expr, {"artist": "Adele"}, expected_result=True, test_name="regex_inline_flags_match")
+        self._test(expr, {"artist": "muse"}, expected_result=False, test_name="regex_inline_flags_no_match")
+
+    def test_quoted_values_with_spaces(self):
+        expr = "uploader = 'BBC News'"
+
+        parser = MiniFilter(expr)
+
+        assert parser.tokens == [("ATOM", "uploader='BBC News'")]
+        assert parser.export() == ["uploader='BBC News'"]
+
+        self._test(expr, {"uploader": "BBC News"}, expected_result=True, test_name="quoted_spaces_match")
+        self._test(expr, {"uploader": "BBC"}, expected_result=False, test_name="quoted_spaces_no_match")
+
+    def test_quoted_values_with_parentheses(self):
+        expr = "title='a(b)c'"
+
+        parser = MiniFilter(expr)
+
+        assert parser.tokens == [("ATOM", "title='a(b)c'")]
+        assert parser.export() == ["title='a(b)c'"]
+
+        self._test(expr, {"title": "a(b)c"}, expected_result=True, test_name="quoted_parentheses_match")
+        self._test(expr, {"title": "abc"}, expected_result=False, test_name="quoted_parentheses_no_match")
+
+    def test_quoted_regex_ampersand(self):
+        expr = r"description~='(?i)\bcats \& dogs\b'"
+
+        parser = MiniFilter(expr)
+
+        assert parser.tokens == [("ATOM", r"description~='(?i)\bcats \& dogs\b'")]
+        assert parser.export() == [r"description~='(?i)\bcats \& dogs\b'"]
+
+        self._test(
+            expr,
+            {"description": "Cats & Dogs are great"},
+            expected_result=True,
+            test_name="quoted_regex_escaped_ampersand_match",
+        )
+        self._test(
+            expr,
+            {"description": "Cats and dogs are great"},
+            expected_result=False,
+            test_name="quoted_regex_escaped_ampersand_no_match",
+        )
+
+    def test_parser_rejects_trailing_tokens(self):
+        with pytest.raises(SyntaxError, match="Unexpected token"):
+            MiniFilter("uploader='BBC' stray")
+
     def test_spaces_around_operators(self):
         """Test that spaces around operators are handled correctly."""
         d: dict[str, str] = {"channel_id": "UC-7oMv6E4Uz2tF51w5Sj49w", "uploader": "BBC"}
@@ -467,7 +528,7 @@ class TestMiniFilter(unittest.TestCase):
     def test_export_and_yt_dlp_compat(self):
         from yt_dlp.utils import match_str
 
-        d: dict[str, str] = {"filesize": 2000000, "duration": 200, "uploader": "BBC"}
+        d: dict[str, Any] = {"filesize": 2000000, "duration": 200, "uploader": "BBC"}
         # Use numeric values to avoid yt-dlp unit parsing inconsistencies
         expr = "(filesize>1000000 & duration<600) || uploader='BBC'"
 

app/features/ytdlp/tests/test_ytdlp_module.py

@@ -1,11 +1,23 @@
+import importlib
+from pathlib import Path
+from typing import Any
 from unittest.mock import MagicMock, Mock, patch
 
+import pytest
+from yt_dlp.globals import extractors as ytdlp_extractors
+
+from app.features.ytdlp.outtmpl import rewrite_outtmpl
+from app.features.ytdlp.patches import patch_windows_popen_wait
 from app.features.ytdlp.utils import _DATA
 from app.features.ytdlp.ytdlp import YTDLP, _ArchiveProxy, ytdlp_options
 
 
+def _archive_path(tmp_path: Path) -> str:
+    return str(tmp_path / "archive.txt")
+
+
 class TestArchiveProxy:
-    def test_bool_and_falsey_cases(self) -> None:
+    def test_bool_and_falsey_cases(self, tmp_path: Path) -> None:
         # No file path means proxy is falsey and operations return False
         p = _ArchiveProxy(file=None)
         assert bool(p) is False
@@ -13,22 +25,24 @@ class TestArchiveProxy:
         assert p.add("id") is False
 
         # Empty item also returns False
-        p2 = _ArchiveProxy(file="/tmp/archive.txt")
+        p2 = _ArchiveProxy(file=_archive_path(tmp_path))
         assert bool(p2) is True
         assert ("" in p2) is False
         assert p2.add("") is False
 
     @patch("app.features.ytdlp.archiver.Archiver.get_instance")
-    def test_contains_and_add_delegate_to_archiver(self, mock_get_instance) -> None:
+    def test_delegates_to_archiver(self, mock_get_instance, tmp_path: Path) -> None:
         arch = MagicMock()
         mock_get_instance.return_value = arch
 
-        p = _ArchiveProxy(file="/tmp/archive.txt")
+        file = _archive_path(tmp_path)
+
+        p = _ArchiveProxy(file=file)
 
         # contains -> read(file, [item]) and check membership
         arch.read.return_value = ["abc"]
         assert ("abc" in p) is True
-        arch.read.assert_called_with("/tmp/archive.txt", ["abc"])
+        arch.read.assert_called_with(file, ["abc"])
 
         arch.read.return_value = []
         assert ("xyz" in p) is False
@@ -36,14 +50,14 @@ class TestArchiveProxy:
         # add -> add(file, [item]) returns boolean
         arch.add.return_value = True
         assert p.add("abc") is True
-        arch.add.assert_called_with("/tmp/archive.txt", ["abc"])
+        arch.add.assert_called_with(file, ["abc"])
 
         arch.add.return_value = False
         assert p.add("xyz") is False
 
 
 class TestYtDlpOptions:
-    def test_options_structure_and_no_suppresshelp(self) -> None:
+    def test_options_shape(self) -> None:
         opts = ytdlp_options()
 
         assert isinstance(opts, list)
@@ -90,11 +104,12 @@ class TestYTDLP:
             return ytdlp
 
     @patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
-    def test_init_patches_download_archive_param(self, mock_super_init) -> None:
+    def test_init_archive_param(self, mock_super_init, tmp_path: Path) -> None:
         """Test that __init__ removes download_archive before calling super, then restores it."""
         mock_super_init.return_value = None
 
-        params = {"download_archive": "/tmp/archive.txt", "quiet": True}
+        file = _archive_path(tmp_path)
+        params = {"download_archive": file, "quiet": True}
 
         ytdlp = YTDLP(params=params, auto_init=True)
         # Set params as it would be after super().__init__
@@ -108,16 +123,16 @@ class TestYTDLP:
         assert call_kwargs["auto_init"] is True
 
         # Our __init__ code manually sets these after super()
-        ytdlp.params["download_archive"] = "/tmp/archive.txt"
+        ytdlp.params["download_archive"] = file
 
-        assert ytdlp.params["download_archive"] == "/tmp/archive.txt", "Verify download_archive was restored to params"
+        assert ytdlp.params["download_archive"] == file, "Verify download_archive was restored to params"
 
         # Verify archive proxy was set up
         assert isinstance(ytdlp.archive, _ArchiveProxy)
-        assert ytdlp.archive._file == "/tmp/archive.txt"
+        assert ytdlp.archive._file == file
 
     @patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
-    def test_init_handles_no_download_archive(self, mock_super_init) -> None:
+    def test_init_no_archive(self, mock_super_init) -> None:
         """Test __init__ works correctly when download_archive is not in params."""
         mock_super_init.return_value = None
 
@@ -135,7 +150,7 @@ class TestYTDLP:
         assert not ytdlp.archive
 
     @patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
-    def test_init_handles_none_params(self, mock_super_init) -> None:
+    def test_init_none_params(self, mock_super_init) -> None:
         """Test __init__ handles None params gracefully."""
         mock_super_init.return_value = None
 
@@ -145,12 +160,68 @@ class TestYTDLP:
         assert isinstance(ytdlp.archive, _ArchiveProxy)
         assert not ytdlp.archive
 
+    @patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__")
+    def test_init_patches_wait(self, mock_super_init) -> None:
+        mock_super_init.return_value = None
+
+        class FakePopen:
+            def wait(self, timeout=None):
+                return timeout
+
+        with patch("app.features.ytdlp.patches.sys.platform", "win32"):
+            with patch("yt_dlp.utils.Popen", FakePopen):
+                YTDLP(params={})
+
+        assert getattr(FakePopen, "_ytptube_wait_patched", False) is True
+
+    def test_wait_patch_polls(self) -> None:
+        calls: list[float | None] = []
+
+        class FakePopen:
+            _ytptube_wait_patched = False
+
+            def wait(self, timeout=None):
+                calls.append(timeout)
+                if len(calls) < 3:
+                    raise TimeoutError
+                return 0
+
+        with patch("app.features.ytdlp.patches.sys.platform", "win32"):
+            with (
+                patch("yt_dlp.utils.Popen", FakePopen),
+                patch("app.features.ytdlp.patches.subprocess.TimeoutExpired", TimeoutError),
+            ):
+                patch_windows_popen_wait()
+                result = FakePopen().wait()
+
+        assert result == 0
+        assert calls == [0.1, 0.1, 0.1]
+
+    def test_wait_patch_timeout(self) -> None:
+        calls: list[float | None] = []
+
+        class FakePopen:
+            _ytptube_wait_patched = False
+
+            def wait(self, timeout=None):
+                calls.append(timeout)
+                return 0
+
+        with patch("app.features.ytdlp.patches.sys.platform", "win32"):
+            with patch("yt_dlp.utils.Popen", FakePopen):
+                patch_windows_popen_wait()
+                result = FakePopen().wait(timeout=5)
+
+        assert result == 0
+        assert calls == [5]
+
     @patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL._delete_downloaded_files")
-    def test_delete_downloaded_files_skips_when_interrupted(self, mock_super_delete) -> None:
+    def test_delete_interrupted(self, mock_super_delete) -> None:
         """Test _delete_downloaded_files skips cleanup when _interrupted is True."""
         with patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__", return_value=None):
             ytdlp = YTDLP(params={})
-            ytdlp.to_screen = Mock()
+            mock_obj: Any = ytdlp
+            mock_obj.to_screen = Mock()
 
             # Set interrupted flag
             ytdlp._interrupted = True
@@ -160,12 +231,12 @@ class TestYTDLP:
             # Should not call super method
             mock_super_delete.assert_not_called()
             # Should show message
-            ytdlp.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
+            mock_obj.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
             # Should return None
             assert result is None
 
     @patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL._delete_downloaded_files")
-    def test_delete_downloaded_files_calls_super_when_not_interrupted(self, mock_super_delete) -> None:
+    def test_delete_calls_super(self, mock_super_delete) -> None:
         """Test _delete_downloaded_files calls super when not interrupted."""
         mock_super_delete.return_value = "cleanup_result"
 
@@ -180,7 +251,7 @@ class TestYTDLP:
             # Should return super's result
             assert result == "cleanup_result"
 
-    def test_record_download_archive_does_nothing_without_download_archive_param(self) -> None:
+    def test_record_archive_missing(self) -> None:
         """Test record_download_archive returns early when download_archive is not set."""
         ytdlp = self._create_ytdlp(params={})
         ytdlp.archive = Mock()
@@ -190,9 +261,9 @@ class TestYTDLP:
         # Should not interact with archive
         ytdlp.archive.add.assert_not_called()
 
-    def test_record_download_archive_adds_archive_id(self) -> None:
+    def test_record_archive_adds_id(self, tmp_path: Path) -> None:
         """Test record_download_archive adds the archive ID."""
-        ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
+        ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
         ytdlp.write_debug = Mock()
         ytdlp.archive = Mock()
         ytdlp._make_archive_id = Mock(return_value="youtube test123")
@@ -208,9 +279,9 @@ class TestYTDLP:
         ytdlp.archive.add.assert_called_once_with("youtube test123")
         ytdlp.write_debug.assert_called_with("Adding to archive: youtube test123")
 
-    def test_record_download_archive_handles_old_archive_ids(self) -> None:
+    def test_record_archive_old_ids(self, tmp_path: Path) -> None:
         """Test record_download_archive adds _old_archive_ids when present."""
-        ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
+        ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
         ytdlp.write_debug = Mock()
         ytdlp.archive = Mock()
         ytdlp._make_archive_id = Mock(return_value="youtube new123")
@@ -235,9 +306,9 @@ class TestYTDLP:
         # Should not add duplicate (youtube new123 appears only once)
         assert calls.count("youtube new123") == 1
 
-    def test_record_download_archive_skips_empty_old_archive_ids(self) -> None:
+    def test_record_archive_empty_old_ids(self, tmp_path: Path) -> None:
         """Test record_download_archive handles empty or invalid _old_archive_ids."""
-        ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
+        ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
         ytdlp.write_debug = Mock()
         ytdlp.archive = Mock()
         ytdlp._make_archive_id = Mock(return_value="youtube test123")
@@ -261,9 +332,9 @@ class TestYTDLP:
         ytdlp.record_download_archive(info_dict)
         assert ytdlp.archive.add.call_count == 1  # Only main ID
 
-    def test_record_download_archive_returns_early_on_empty_archive_id(self) -> None:
+    def test_record_archive_empty_id(self, tmp_path: Path) -> None:
         """Test record_download_archive returns early when _make_archive_id returns empty."""
-        ytdlp = self._create_ytdlp(params={"download_archive": "/tmp/archive.txt"})
+        ytdlp = self._create_ytdlp(params={"download_archive": _archive_path(tmp_path)})
         ytdlp.archive = Mock()
         ytdlp._make_archive_id = Mock(return_value=None)
 
@@ -271,3 +342,124 @@ class TestYTDLP:
 
         # Should not add anything
         ytdlp.archive.add.assert_not_called()
+
+    def test_outtmpl_callable(self) -> None:
+        ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
+
+        result = ytdlp.evaluate_outtmpl("%(ytp_random:8)s", {"title": "x"})
+
+        assert len(result) == 8
+        assert result.isalnum()
+
+    def test_exec_init(self) -> None:
+        YTDLP(
+            params={
+                "compat_opts": set(),
+                "postprocessors": [{"key": "Exec", "exec_cmd": "echo %(title)q"}],
+            }
+        )
+
+    def test_outtmpl_reuses_value(self) -> None:
+        ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
+
+        result = ytdlp.evaluate_outtmpl("%(ytp_random:8)s/%(ytp_random:8)s", {"title": "x"})
+        first, second = result.split("/")
+
+        assert first == second
+
+    def test_outtmpl_new_value(self) -> None:
+        ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
+
+        first = ytdlp.evaluate_outtmpl("%(ytp_random:8)s", {"title": "x"})
+        second = ytdlp.evaluate_outtmpl("%(ytp_random:8)s", {"title": "y"})
+
+        assert first != second
+
+    def test_prepare_filename_sidecars(self) -> None:
+        ytdlp = YTDLP(
+            params={
+                "outtmpl": {
+                    "default": "%(ytp_random:6)s.%(ext)s",
+                    "thumbnail": "%(ytp_random:6)s.%(ext)s",
+                    "subtitle": "%(ytp_random:6)s.%(ext)s",
+                    "infojson": "%(ytp_random:6)s.%(ext)s",
+                }
+            }
+        )
+
+        info = {"id": "abc123", "title": "Example", "ext": "mp4"}
+
+        default_name = ytdlp.prepare_filename(info)
+        thumbnail_name = ytdlp.prepare_filename(info, "thumbnail")
+        subtitle_name = ytdlp.prepare_filename(info, "subtitle")
+        infojson_name = ytdlp.prepare_filename(info, "infojson")
+
+        default_base = default_name.rsplit(".", 1)[0]
+        thumbnail_base = thumbnail_name.rsplit(".", 1)[0]
+        subtitle_base = subtitle_name.rsplit(".", 1)[0]
+        infojson_base = infojson_name.removesuffix(".info.json")
+
+        assert default_base == thumbnail_base
+        assert default_base == subtitle_base
+        assert default_base == infojson_base
+
+    def test_prepare_filename_resets(self) -> None:
+        ytdlp = YTDLP(params={"outtmpl": {"default": "%(ytp_random:8)s.%(ext)s"}})
+
+        first = ytdlp.prepare_filename({"id": "one", "title": "One", "ext": "mp4"})
+        second = ytdlp.prepare_filename({"id": "two", "title": "Two", "ext": "mp4"})
+
+        assert first != second
+
+    @pytest.mark.parametrize(
+        ("template", "expected"),
+        [
+            ("%(ytp_random:8|fallback)s", 8),
+            ("%(ytp_random:8&{} - |)s", 11),
+            ("%(ytp_random:8)S", 8),
+        ],
+    )
+    def test_outtmpl_suffix(self, template: str, expected: int) -> None:
+        ytdlp = YTDLP(
+            params={
+                "outtmpl": {"default": "%(title)s"},
+                "restrictfilenames": True,
+            },
+        )
+
+        result = ytdlp.evaluate_outtmpl(template, {"title": "x"})
+
+        assert len(result) == expected
+
+    def test_outtmpl_modes(self) -> None:
+        ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
+
+        digits = ytdlp.evaluate_outtmpl("%(ytp_random:6:d)s", {"title": "x"})
+        letters = ytdlp.evaluate_outtmpl("%(ytp_random:6:s)s", {"title": "x"})
+
+        assert digits.isdigit()
+        assert letters.isalpha()
+
+    def test_outtmpl_unknown_callable(self) -> None:
+        ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
+
+        with pytest.raises(ValueError, match="Unsupported YTPTube output template callable"):
+            ytdlp.prepare_outtmpl("%(ytp_unknown:8)s", {"title": "x"})
+
+    def test_outtmpl_invalid_length(self) -> None:
+        ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
+
+        with pytest.raises(ValueError, match="ytp_random length must be an integer"):
+            ytdlp.prepare_outtmpl("%(ytp_random:nope)s", {"title": "x"})
+
+
+class TestOuttmpl:
+    def test_rewrite_outtmpl_cache(self) -> None:
+        cache: dict[str, object] = {}
+        template = "%(ytp_random:8)s/%(ytp_random:8)s.%(ext)s"
+
+        rewritten, info = rewrite_outtmpl(template, {"ext": "mp4"}, cache=cache)
+
+        assert rewritten == "%(__ytptube_outtmpl_0)s/%(__ytptube_outtmpl_0)s.%(ext)s"
+        assert len(cache) == 1
+        assert info["__ytptube_outtmpl_0"] == next(iter(cache.values()))

app/features/ytdlp/tests/test_ytdlp_opts.py

@@ -1,5 +1,6 @@
 """Tests for YTDLPOpts class."""
 
+from pathlib import Path
 from unittest.mock import Mock, patch
 
 import pytest
@@ -25,7 +26,7 @@ class TestYTDLPOpts:
             assert opts._item_cli == []
             assert opts._preset_cli == ""
 
-    def test_get_instance_returns_reset_instance(self):
+    def test_get_instance(self):
         """Test that get_instance returns a reset YTDLPOpts instance."""
         with patch("app.features.ytdlp.ytdlp_opts.Config"):
             opts = YTDLPOpts.get_instance()
@@ -51,7 +52,7 @@ class TestYTDLPOpts:
             assert "--format best" in opts._item_cli
             mock_converter.assert_called_once_with(args="--format best", level=False)
 
-    def test_add_cli_with_invalid_args_raises_error(self):
+    def test_add_cli_invalid(self):
         """Test that invalid CLI arguments raise ValueError."""
         with (
             patch("app.features.ytdlp.ytdlp_opts.Config"),
@@ -63,7 +64,7 @@ class TestYTDLPOpts:
             with pytest.raises(ValueError, match="Invalid command options for yt-dlp were given"):
                 opts.add_cli("--invalid-arg", from_user=True)
 
-    def test_add_cli_with_empty_args_returns_self(self):
+    def test_add_cli_empty(self):
         """Test that empty or invalid args return self without processing."""
         with patch("app.features.ytdlp.ytdlp_opts.Config"):
             opts = YTDLPOpts()
@@ -95,7 +96,7 @@ class TestYTDLPOpts:
             assert opts._item_opts["format"] == "best"
             assert opts._item_opts["quality"] == "720p"
 
-    def test_add_with_user_config_filters_bad_options(self):
+    def test_add_filters_bad_options(self):
         """Test that user config filters out dangerous options."""
         with patch("app.features.ytdlp.ytdlp_opts.Config"):
             opts = YTDLPOpts()
@@ -211,7 +212,7 @@ class TestYTDLPOpts:
             assert opts._preset_opts["cookiefile"] == expected_path
             mock_preset.get_cookies_file.assert_called_once_with(config=mock_config_instance)
 
-    def test_preset_with_invalid_cli_raises_error(self):
+    def test_preset_invalid_cli(self):
         """Test that preset with invalid CLI raises ValueError."""
         with (
             patch("app.features.ytdlp.ytdlp_opts.Config"),
@@ -289,7 +290,7 @@ class TestYTDLPOpts:
             expected_cli = "--quality 720p\n--format best"
             mock_converter.assert_called_once_with(args=expected_cli, level=True)
 
-    def test_get_all_handles_format_special_cases(self):
+    def test_get_all_format_cases(self):
         """Test get_all handles special format values correctly."""
         with (
             patch("app.features.ytdlp.ytdlp_opts.Config") as mock_config,
@@ -325,7 +326,7 @@ class TestYTDLPOpts:
             result = opts.get_all(keep=True)
             assert result["format"] == "best"
 
-    def test_get_all_with_invalid_cli_raises_error(self):
+    def test_get_all_invalid_cli(self):
         """Test get_all raises error for invalid CLI arguments."""
         with (
             patch("app.features.ytdlp.ytdlp_opts.Config") as mock_config,
@@ -441,7 +442,7 @@ class TestYTDLPOpts:
             opts = YTDLPOpts()
             opts.get_all(keep=True)
 
-            mock_log.debug.assert_called_once_with(f"Final yt-dlp options: '{test_data!s}'.")
+            mock_log.debug.assert_called_once_with("Prepared final yt-dlp options.", extra={"ytdlp_options": test_data})
 
     def test_cookie_loading_error_handling(self):
         """Test error handling when cookie loading fails."""
@@ -474,11 +475,12 @@ class TestYTDLPOpts:
             opts = YTDLPOpts()
             opts.preset("cookie_preset")
 
-            # Should log error but not raise
-            mock_log.error.assert_called_once()
-            error_args = mock_log.error.call_args[0][0]
-            assert "Failed to load" in error_args
-            assert "Cookie Preset" in error_args
+            # Should log exception but not raise
+            mock_log.exception.assert_called_once()
+            error_fields = mock_log.exception.call_args.kwargs["extra"]
+            assert error_fields["preset"] == "Cookie Preset"
+            assert error_fields["has_cookies"] is True
+            assert error_fields["exception_type"] == "ValueError"
 
             assert "cookiefile" not in opts._preset_opts, "cookiefile should not be set"
 
@@ -578,7 +580,7 @@ class TestARGSMerger:
         assert "--output" in merger.args
         assert "--socket-timeout" in merger.args
 
-    def test_add_filters_complex_commented_extractor_args(self):
+    def test_add_complex_comments(self):
         """Test filtering of complex real-world commented extractor-args."""
         from app.features.ytdlp.ytdlp_opts import ARGSMerger
 
@@ -631,7 +633,7 @@ class TestARGSMerger:
         assert "--format" in merger.args
         assert "bestvideo[height<=1080]+bestaudio/best" in merger.args
 
-    def test_add_empty_string_returns_self(self):
+    def test_add_empty(self):
         """Test that adding empty string returns self without modifying args."""
         from app.features.ytdlp.ytdlp_opts import ARGSMerger
 
@@ -641,7 +643,7 @@ class TestARGSMerger:
         assert result is merger
         assert merger.args == []
 
-    def test_add_short_string_returns_self(self):
+    def test_add_short(self):
         """Test that adding short string (len < 2) returns self without modifying args."""
         from app.features.ytdlp.ytdlp_opts import ARGSMerger
 
@@ -651,7 +653,7 @@ class TestARGSMerger:
         assert result is merger
         assert merger.args == []
 
-    def test_add_non_string_returns_self(self):
+    def test_add_non_string(self):
         """Test that adding non-string returns self without modifying args."""
         from app.features.ytdlp.ytdlp_opts import ARGSMerger
 
@@ -769,12 +771,12 @@ class TestYTDLPCli:
         assert cli.item is item
         assert cli._config is mock_config_instance
 
-    def test_constructor_with_invalid_type_raises_error(self):
+    def test_constructor_invalid_item(self):
         """Test YTDLPCli constructor raises error with non-Item type."""
         from app.features.ytdlp.ytdlp_opts import YTDLPCli
 
         with pytest.raises(ValueError, match="Expected Item instance"):
-            YTDLPCli(item="not an item")  # type: ignore
+            YTDLPCli(item="not an item")
 
     @patch("app.features.presets.service.Presets")
     @patch("app.features.ytdlp.ytdlp_opts.Config")
@@ -912,7 +914,7 @@ class TestYTDLPCli:
     @patch("app.features.presets.service.Presets")
     @patch("app.features.ytdlp.ytdlp_opts.create_cookies_file")
     @patch("app.features.ytdlp.ytdlp_opts.Config")
-    def test_build_with_cookies_from_user(self, mock_config, mock_create_cookies, mock_presets):
+    def test_build_with_cookies_from_user(self, mock_config, mock_create_cookies, mock_presets, tmp_path: Path):
         """Test build with cookies from user."""
         from app.library.ItemDTO import Item
         from app.features.ytdlp.ytdlp_opts import YTDLPCli
@@ -920,11 +922,13 @@ class TestYTDLPCli:
         mock_config_instance = Mock()
         mock_config_instance.download_path = "/downloads"
         mock_config_instance.output_template = "%(title)s.%(ext)s"
+        mock_config_instance.temp_path = str(tmp_path)
         mock_config_instance.get_replacers.return_value = {}
         mock_config.get_instance.return_value = mock_config_instance
 
         mock_cookie_path = Mock()
-        mock_cookie_path.__str__ = Mock(return_value="/tmp/cookies.txt")
+        cookie_path = tmp_path / "cookies.txt"
+        mock_cookie_path.__str__ = Mock(return_value=str(cookie_path))
         mock_create_cookies.return_value = mock_cookie_path
 
         mock_presets.get_instance.return_value.get.return_value = None
@@ -936,8 +940,8 @@ class TestYTDLPCli:
 
         mock_create_cookies.assert_called_once_with("session_id=abc123")
         assert "--cookies" in command
-        assert "/tmp/cookies.txt" in command
-        assert info["merged"]["cookie_file"] == "/tmp/cookies.txt"
+        assert str(cookie_path) in command
+        assert info["merged"]["cookie_file"] == str(cookie_path)
 
     @patch("app.features.presets.service.Presets")
     @patch("app.features.ytdlp.ytdlp_opts.Config")
@@ -950,6 +954,7 @@ class TestYTDLPCli:
         mock_config_instance = Mock()
         mock_config_instance.download_path = "/downloads"
         mock_config_instance.output_template = "%(title)s.%(ext)s"
+        mock_config_instance.temp_path = "/tmp"
         mock_config_instance.get_replacers.return_value = {}
         mock_config.get_instance.return_value = mock_config_instance
 

app/features/ytdlp/tests/test_ytdlp_utils.py

@@ -1,7 +1,6 @@
 import logging
 from pathlib import Path
 import re
-import tempfile
 from typing import Any
 from unittest.mock import MagicMock, patch
 
@@ -20,6 +19,7 @@ from app.features.ytdlp.utils import (
     archive_add,
     archive_read,
 )
+from app.tests.helpers import make_test_temp_dir
 
 
 class CaptureHandler(logging.Handler):
@@ -46,16 +46,17 @@ class TestLogWrapper:
     def test_add_target_type_validation(self) -> None:
         lw = LogWrapper()
         with pytest.raises(TypeError, match=r"Target must be a logging\.Logger instance or a callable"):
-            lw.add_target(123)  # type: ignore[arg-type]
+            bad: Any = 123
+            lw.add_target(bad)
 
-    def test_add_target_name_inference_and_custom(self) -> None:
+    def test_add_target_names(self) -> None:
         lw = LogWrapper()
         logger, _ = make_logger("one")
 
         # Name inferred from logger
         lw.add_target(logger)
         assert lw.targets[-1].name == "one"
-        assert lw.has_targets() is True
+        assert lw.has_targets()
 
         # Name inferred from callable
         def sink(level: int, msg: str, *args: Any, **kwargs: Any) -> None:  # noqa: ARG001
@@ -90,6 +91,7 @@ class TestLogWrapper:
         assert len(cap.records) == 1
         assert cap.records[0].levelno == logging.INFO
         assert cap.records[0].getMessage() == "hello X"
+        assert cap.records[0].funcName == "test_level_filtering_and_dispatch"
         assert len(calls) == 0
 
         # WARNING hits both
@@ -127,9 +129,9 @@ class TestExtractYtdlpLogs:
     def test_extract_ytdlp_logs_with_filters(self):
         """Test log extraction with filters."""
         logs = ["INFO: Downloading", "ERROR: Failed", "WARNING: Deprecated"]
-        filters = [re.compile(r"ERROR")]
+        filters: list[str | re.Pattern[str]] = [re.compile(r"ERROR")]
         result = extract_ytdlp_logs(logs, filters)
-        assert len(result) >= 0  # Should filter based on patterns
+        assert result == ["ERROR: Failed"]
 
     def test_extract_ytdlp_logs_empty(self):
         """Test with empty logs."""
@@ -146,8 +148,8 @@ class TestYtdlpReject:
         yt_params = {}
 
         passed, message = ytdlp_reject(entry, yt_params)
-        assert isinstance(passed, bool)
-        assert isinstance(message, str)
+        assert passed is True
+        assert message == ""
 
     def test_ytdlp_reject_with_filters(self):
         """Test rejection with filters."""
@@ -158,8 +160,9 @@ class TestYtdlpReject:
         yt_params["daterange"].__contains__ = MagicMock(return_value=False)
 
         passed, message = ytdlp_reject(entry, yt_params)
-        assert isinstance(passed, bool)
-        assert isinstance(message, str)
+        assert passed is False
+        assert "20230101" in message
+        assert "not in range" in message
 
 
 class TestParseOuttmpl:
@@ -256,7 +259,7 @@ class TestParseOuttmpl:
 
         assert result == "Test Channel/Best Videos/005 - Amazing Content [abc123xyz].mp4"
 
-    def test_parse_outtmpl_with_special_characters(self):
+    def test_parse_outtmpl_special_chars(self):
         """Test template parsing handles special characters in values."""
 
         template = "%(title)s.%(ext)s"
@@ -284,7 +287,7 @@ class TestParseOuttmpl:
 
         assert result == "My Playlist/Video Title.webm"
 
-    def test_parse_outtmpl_with_restrict_filename(self):
+    def test_parse_outtmpl_restrict(self):
         """Test template parsing with restrict_filename parameter."""
 
         template = "%(uploader)s/%(title)s.%(ext)s"
@@ -302,19 +305,22 @@ class TestParseOuttmpl:
 
 
 class TestGetThumbnail:
-    def test_returns_none_for_empty_list(self):
+    def test_empty_list(self):
         """Test that None is returned for an empty thumbnail list."""
 
         assert get_thumbnail([]) is None
 
-    def test_returns_none_for_non_list(self):
+    def test_non_list(self):
         """Test that None is returned for non-list input."""
 
-        assert get_thumbnail(None) is None
-        assert get_thumbnail("not a list") is None
-        assert get_thumbnail({"not": "list"}) is None
+        bad_none: Any = None
+        bad_str: Any = "not a list"
+        bad_dict: Any = {"not": "list"}
+        assert get_thumbnail(bad_none) is None
+        assert get_thumbnail(bad_str) is None
+        assert get_thumbnail(bad_dict) is None
 
-    def test_returns_highest_preference_thumbnail(self):
+    def test_thumbnail_preference(self):
         """Test that the thumbnail with highest preference is returned."""
 
         thumbnails = [
@@ -326,7 +332,7 @@ class TestGetThumbnail:
         result = get_thumbnail(thumbnails)
         assert result == {"url": "high.jpg", "preference": 10, "width": 200, "height": 200}
 
-    def test_returns_highest_width_when_preference_equal(self):
+    def test_thumbnail_width(self):
         """Test that the thumbnail with highest width is returned when preference is equal."""
 
         thumbnails = [
@@ -338,7 +344,7 @@ class TestGetThumbnail:
         result = get_thumbnail(thumbnails)
         assert result == {"url": "large.jpg", "preference": 1, "width": 200, "height": 200}
 
-    def test_handles_missing_attributes(self):
+    def test_missing_attrs(self):
         """Test that thumbnails with missing attributes are handled correctly."""
 
         thumbnails = [
@@ -347,9 +353,10 @@ class TestGetThumbnail:
         ]
 
         result = get_thumbnail(thumbnails)
+        assert result is not None
         assert result["url"] == "with_pref.jpg"
 
-    def test_returns_first_when_all_equal(self):
+    def test_all_equal(self):
         """Test that any thumbnail is returned when all attributes are equal."""
 
         thumbnails = [
@@ -357,20 +364,21 @@ class TestGetThumbnail:
             {"url": "second.jpg"},
         ]
 
-        result = get_thumbnail(thumbnails)
-        assert result is not None
-        assert result["url"] in ["first.jpg", "second.jpg"]
+        assert get_thumbnail(thumbnails) == {"url": "second.jpg"}
 
 
 class TestGetExtras:
-    def test_returns_empty_dict_for_none(self):
+    def test_none(self):
         """Test that empty dict is returned for None input."""
-        assert get_extras(None) == {}
+        bad: Any = None
+        assert get_extras(bad) == {}
 
-    def test_returns_empty_dict_for_non_dict(self):
+    def test_non_dict(self):
         """Test that empty dict is returned for non-dict input."""
-        assert get_extras("not a dict") == {}
-        assert get_extras([]) == {}
+        bad_str: Any = "not a dict"
+        bad_list: Any = []
+        assert get_extras(bad_str) == {}
+        assert get_extras(bad_list) == {}
 
     def test_extracts_video_information(self):
         """Test extracting information from a video entry."""
@@ -409,7 +417,7 @@ class TestGetExtras:
         assert result["playlist_uploader"] == "Playlist Owner"
         assert result["playlist_uploader_id"] == "owner123"
 
-    def test_handles_release_timestamp(self):
+    def test_release_timestamp(self):
         """Test handling of release_timestamp for upcoming content."""
 
         entry = {
@@ -421,7 +429,7 @@ class TestGetExtras:
         assert "release_in" in result
         assert result["release_in"] == "Fri, 13 Feb 2009 23:31:30 GMT"
 
-    def test_handles_upcoming_live_stream(self):
+    def test_upcoming_live(self):
         """Test handling of upcoming live stream."""
 
         entry = {
@@ -434,7 +442,7 @@ class TestGetExtras:
         assert result["is_live"] == 1234567890
         assert "release_in" in result
 
-    def test_handles_premiere_flag(self):
+    def test_premiere_flag(self):
         """Test handling of is_premiere flag."""
 
         entry = {
@@ -481,7 +489,7 @@ class TestGetStaticYtdlp:
 
         _DATA.YTDLP_INFO_CLS = None
 
-    def test_get_static_ytdlp_returns_instance(self):
+    def test_instance(self):
         """Test that get_static_ytdlp returns a YTDLP instance."""
         from app.features.ytdlp.ytdlp import YTDLP
 
@@ -491,7 +499,7 @@ class TestGetStaticYtdlp:
         assert instance is not None
         assert isinstance(instance, YTDLP)
 
-    def test_get_static_ytdlp_returns_same_instance(self):
+    def test_get_static_ytdlp_same(self):
         """Test that get_static_ytdlp returns the same cached instance."""
 
         instance1 = get_ytdlp()
@@ -508,7 +516,7 @@ class TestGetStaticYtdlp:
         assert instance1 is not instance2
         assert instance2 is not None
 
-    def test_get_static_ytdlp_has_correct_params(self):
+    def test_get_static_ytdlp_params(self):
         """Test that get_static_ytdlp initializes with correct parameters."""
 
         instance = get_ytdlp()
@@ -549,7 +557,7 @@ class TestArchiveFunctions:
 
     def setup_method(self):
         """Set up test archive file."""
-        self.temp_dir = tempfile.mkdtemp()
+        self.temp_dir = str(make_test_temp_dir("ytdlp-archive"))
         self.archive_file = Path(self.temp_dir) / "archive.txt"
 
     def teardown_method(self):
@@ -560,22 +568,22 @@ class TestArchiveFunctions:
 
     def test_archive_add_and_read(self):
         """Test adding and reading archive entries."""
-        ids = ["id1", "id2", "id3"]
+        ids = ["youtube id1", "youtube id2", "youtube id3"]
 
-        # Add entries - just test it returns a boolean
         result = archive_add(self.archive_file, ids)
-        assert isinstance(result, bool)
+        assert result is True
 
-        # Read entries - just test it returns a list
         read_ids = archive_read(self.archive_file)
-        assert isinstance(read_ids, list)
+        assert set(read_ids) == set(ids)
 
     def test_archive_delete(self):
         """Test deleting archive entries."""
-        # Delete some entries - just test it returns a boolean
-        delete_ids = ["id2"]
+        archive_add(self.archive_file, ["youtube id1", "youtube id2", "youtube id3"])
+
+        delete_ids = ["youtube id2"]
         result = archive_delete(self.archive_file, delete_ids)
-        assert isinstance(result, bool)
+        assert result is True
+        assert set(archive_read(self.archive_file)) == {"youtube id1", "youtube id3"}
 
     def test_archive_read_nonexistent(self):
         """Test reading from non-existent archive."""

app/features/ytdlp/utils.py

@@ -9,10 +9,12 @@ from email.utils import formatdate
 from pathlib import Path
 from typing import Any
 
+from app.features.ytdlp.patches import apply_ytdlp_patches
 from app.features.ytdlp.ytdlp import YTDLP
+from app.library.log import get_logger
 from app.library.Utils import merge_dict, timed_lru_cache
 
-LOG: logging.Logger = logging.getLogger("ytdlp.utils")
+LOG = get_logger()
 
 
 class _DATA:
@@ -37,9 +39,7 @@ class _DATA:
         {
             "quiet": "-q, --quiet",
             "no_warnings": "--no-warnings",
-            "skip_download": "--skip-download",
             "forceprint": "-O, --print",
-            "simulate": "--simulate",
             "noprogress": "--no-progress",
             "wait_for_video": "--wait-for-video",
             "progress_delta": " --progress-delta",
@@ -87,8 +87,9 @@ class LogTarget:
 
 
 class LogWrapper:
-    def __init__(self):
+    def __init__(self, suppress: list[str] | tuple[str, ...] | None = None):
         self.targets: list[LogTarget] = []
+        self.suppress: tuple[str, ...] = tuple(value for value in (suppress or ()) if isinstance(value, str) and value)
 
     def add_target(self, target: logging.Logger | Callable, level: int = logging.DEBUG, name: str | None = None):
         """
@@ -100,12 +101,12 @@ class LogWrapper:
             name (str|None): The name of the logging target. Defaults to None.
 
         """
-        if not isinstance(target, logging.Logger | Callable):
+        if not isinstance(target, logging.Logger) and not callable(target):
             msg = "Target must be a logging.Logger instance or a callable."
             raise TypeError(msg)
 
         if name is None:
-            name = target.name if isinstance(target, logging.Logger) else target.__name__
+            name = target.name if isinstance(target, logging.Logger) else getattr(target, "__name__", "callable")
 
         self.targets.append(
             LogTarget(
@@ -119,14 +120,22 @@ class LogWrapper:
     def has_targets(self):
         return len(self.targets) > 0
 
+    def _skip(self, msg: Any) -> bool:
+        return isinstance(msg, str) and any(value in msg for value in self.suppress)
+
     def _log(self, level, msg, *args, **kwargs):
+        if self._skip(msg):
+            return
+
         for target in self.targets:
             if level < target.level:
                 continue
 
-            if target.logger:
-                target.target.log(level, msg, *args, **kwargs)
-            else:
+            if isinstance(target.target, logging.Logger):
+                log_kwargs: dict[str, Any] = {**kwargs}
+                log_kwargs.setdefault("stacklevel", 3)
+                target.target.log(level, msg, *args, **log_kwargs)
+            elif callable(target.target):
                 target.target(level, msg, *args, **kwargs)
 
     def debug(self, msg, *args, **kwargs):
@@ -145,52 +154,6 @@ class LogWrapper:
         self._log(logging.CRITICAL, msg, *args, **kwargs)
 
 
-def patch_metadataparser() -> None:
-    """
-    Patches yt_dlp MetadataParserPP action to handle subprocess pickling issues.
-    """
-    try:
-        from yt_dlp.postprocessor.metadataparser import MetadataParserPP
-        from yt_dlp.utils import Namespace
-    except Exception as exc:
-        LOG.warning(f"Unable to import yt_dlp metadata parser for patching: {exc!s}")
-        return
-
-    if getattr(MetadataParserPP.Actions, "_ytptube_patched", False):
-        return
-
-    class _ActionNS(Namespace):
-        _ACTIONS_STR: list[str] = []
-
-        @staticmethod
-        def _get_name(func) -> str | None:
-            if not callable(func):
-                return None
-
-            target = getattr(func, "__func__", func)
-            module_name = getattr(target, "__module__", None)
-            qual_name = getattr(target, "__qualname__", getattr(target, "__name__", None))
-
-            return f"{module_name}.{qual_name}" if module_name and qual_name else None
-
-        def __contains__(self, candidate: object) -> bool:
-            if candidate in self.__dict__.values():
-                return True
-
-            if func_name := _ActionNS._get_name(candidate):
-                if len(_ActionNS._ACTIONS_STR) < 1:
-                    _ActionNS._ACTIONS_STR.extend([_ActionNS._get_name(value) for value in self.__dict__.values()])
-
-                return func_name in _ActionNS._ACTIONS_STR
-
-            return False
-
-    actions_dict: dict[str, Any] = dict(MetadataParserPP.Actions.items_)
-    MetadataParserPP.Actions = _ActionNS(**actions_dict)
-    MetadataParserPP.Actions._ytptube_patched = True
-    LOG.debug("MetadataParserPP action namespace patch applied successfully.")
-
-
 def arg_converter(
     args: str,
     level: int | bool | None = None,
@@ -216,18 +179,19 @@ def arg_converter(
 
     create_parser = yt_dlp.options.create_parser
 
-    def _default_opts(args: str):
+    def _default_opts(args: str | list[str]):
         patched_parser = create_parser()
+
+        def patched_create_parser():
+            return patched_parser
+
         try:
-            yt_dlp.options.create_parser = lambda: patched_parser
+            yt_dlp.options.__dict__["create_parser"] = patched_create_parser
             return yt_dlp.parse_options(args)
         finally:
-            yt_dlp.options.create_parser = create_parser
+            yt_dlp.options.__dict__["create_parser"] = create_parser
 
-    try:
-        patch_metadataparser()
-    except Exception as exc:
-        LOG.debug("Metadata parser patch failed to apply: %s", exc)
+    apply_ytdlp_patches()
 
     default_opts = _default_opts([]).ydl_opts
 
@@ -278,7 +242,7 @@ def arg_converter(
     return diff
 
 
-def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) -> list[str]:
+def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern[str]] | None = None) -> list[str]:
     """
     Extract yt-dlp log lines matching built-in filters plus any extras.
 
@@ -372,7 +336,7 @@ def get_ytdlp(params: dict | None = None) -> YTDLP:
     return _DATA.YTDLP_INFO_CLS
 
 
-def get_thumbnail(thumbnails: list) -> str | None:
+def get_thumbnail(thumbnails: list) -> dict | None:
     """
     Extract thumbnail URL from a yt-dlp entry.
 
@@ -380,7 +344,7 @@ def get_thumbnail(thumbnails: list) -> str | None:
         thumbnails (list): The list of thumbnail dictionaries from yt-dlp entry.
 
     Returns:
-        str | None: The thumbnail URL if available, otherwise None.
+        dict | None: The best thumbnail dict if available, otherwise None.
 
     """
     if not thumbnails or not isinstance(thumbnails, list):
@@ -421,7 +385,7 @@ def get_extras(entry: dict, kind: str = "video") -> dict:
                 extras[f"playlist_{property}"] = val
 
     if thumbnail := get_thumbnail(entry.get("thumbnails", [])):
-        extras["thumbnail"] = thumbnail.get("url")
+        extras["thumbnail"] = thumbnail.get("url") if isinstance(thumbnail, dict) else thumbnail
     elif thumbnail := entry.get("thumbnail"):
         extras["thumbnail"] = thumbnail
 
@@ -479,7 +443,7 @@ def get_archive_id(url: str) -> dict[str, str | None]:
         }
 
     """
-    idDict: dict[str, None] = {
+    idDict: dict[str, str | None] = {
         "id": None,
         "ie_key": None,
         "archive_id": None,
@@ -503,8 +467,12 @@ def get_archive_id(url: str) -> dict[str, str | None]:
             idDict["archive_id"] = make_archive_id(_ie, temp_id)
             break
         except Exception as e:
-            LOG.exception(e)
-            LOG.error(f"Error getting archive ID: {e}")
+            LOG.exception(
+                "Failed to get archive ID for '%s' with extractor '%s'.",
+                url,
+                key,
+                extra={"url": url, "extractor": key, "exception_type": type(e).__name__},
+            )
 
     return idDict
 

app/features/ytdlp/ytdlp.py

@@ -3,9 +3,13 @@ import sys
 from typing import Any
 
 import yt_dlp
+from yt_dlp.globals import extractors as ytdlp_extractors
 from yt_dlp.utils import make_archive_id
 
+from app.features.ytdlp.outtmpl import rewrite_outtmpl
+from app.features.ytdlp.patches import apply_ytdlp_patches
 from app.library.cf_solver_handler import set_cf_handler
+from app.library.log import get_logger
 
 
 class _ArchiveProxy:
@@ -52,18 +56,34 @@ class YTDLP(yt_dlp.YoutubeDL):
     _registered = False
 
     def __init__(self, params=None, auto_init=True):
+        apply_ytdlp_patches()
+
+        if not YTDLP._registered:
+            try:
+                from app.yt_dlp_plugins.extractor import generic_browser
+                from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP
+                from yt_dlp.postprocessor import postprocessors
+
+                postprocessors.value.update({"NFOMakerPP": NFOMakerPP})
+                YTDLP._registered = True
+            except Exception:
+                get_logger().exception("Failed to register yt-dlp plugins")
+
         # Avoid yt-dlp preloading the archive file by stripping the param first
         orig_file = None
-        patched_params = None
+        patched_params: dict[str, Any] | None = None
         if params is not None:
             try:
                 orig_file: str | None = params.get("download_archive")
-                patched_params: dict = dict(params)
+                patched_params = dict(params)
                 if "download_archive" in patched_params:
                     patched_params.pop("download_archive", None)
             except Exception:
                 patched_params = params
 
+        self._ytptube_outtmpl_info: dict[str, Any] | None = None
+        self._ytptube_outtmpl_cache: dict[str, Any] = {}
+
         super().__init__(params=patched_params, auto_init=auto_init)
 
         # Restore param and replace upstream archive set with our proxy
@@ -74,15 +94,6 @@ class YTDLP(yt_dlp.YoutubeDL):
                 pass
 
         self.archive = _ArchiveProxy(orig_file)
-        if not YTDLP._registered:
-            try:
-                from app.yt_dlp_plugins.postprocessor.nfo_maker import NFOMakerPP
-                from yt_dlp.postprocessor import postprocessors
-
-                postprocessors.value.update({"NFOMakerPP": NFOMakerPP})
-                YTDLP._registered = True
-            except Exception:
-                pass
 
     def _delete_downloaded_files(self, *args, **kwargs) -> None:
         if self._interrupted:
@@ -91,6 +102,25 @@ class YTDLP(yt_dlp.YoutubeDL):
 
         return super()._delete_downloaded_files(*args, **kwargs)
 
+    def _reset_outtmpl_cache(self) -> None:
+        self._ytptube_outtmpl_info = None
+        self._ytptube_outtmpl_cache = {}
+
+    def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False, *, _exec=False):
+        if self._ytptube_outtmpl_info is not info_dict:
+            self._ytptube_outtmpl_info = info_dict
+            self._ytptube_outtmpl_cache = {}
+
+        outtmpl, enriched = rewrite_outtmpl(outtmpl, info_dict, cache=self._ytptube_outtmpl_cache)
+
+        return super().prepare_outtmpl(outtmpl, enriched, sanitize=sanitize, _exec=_exec)
+
+    def process_info(self, info_dict):
+        try:
+            return super().process_info(info_dict)
+        finally:
+            self._reset_outtmpl_cache()
+
     def record_download_archive(self, info_dict) -> None:
         if not self.params.get("download_archive"):
             return

app/features/ytdlp/ytdlp_opts.py

@@ -1,4 +1,3 @@
-import logging
 import shlex
 from pathlib import Path
 from typing import Any
@@ -6,9 +5,10 @@ from typing import Any
 from app.features.presets.schemas import Preset
 from app.features.ytdlp.utils import arg_converter
 from app.library.config import Config
+from app.library.log import get_logger
 from app.library.Utils import calc_download_path, create_cookies_file, merge_dict
 
-LOG: logging.Logger = logging.getLogger("ytdlp.ytdlp_opts")
+LOG = get_logger()
 
 
 class ARGSMerger:
@@ -50,12 +50,12 @@ class ARGSMerger:
         """
         return str(self)
 
-    def as_dict(self) -> dict:
+    def as_dict(self) -> list[str]:
         """
-        Get all the options as a dict.
+        Get all the options as a shell argument list.
 
         Returns:
-            dict: The options as a dict
+            list[str]: The options as shell arguments.
 
         """
         return shlex.split(shlex.join(self.args))
@@ -335,7 +335,12 @@ class YTDLPOpts:
                 if file and file.exists():
                     self._preset_opts["cookiefile"] = str(file)
             except ValueError as e:
-                LOG.error(f"Failed to load '{preset.name}' cookies. {e!s}")
+                LOG.exception(
+                    "Failed to load cookies for preset '%s': %s.",
+                    preset.name,
+                    e,
+                    extra={"preset": preset.name, "has_cookies": True, "exception_type": type(e).__name__},
+                )
 
         if preset.template:
             self._preset_opts["outtmpl"] = {"default": preset.template, "chapter": self._config.output_template_chapter}
@@ -392,6 +397,30 @@ class YTDLPOpts:
 
         data: dict = merge_dict(user_cli, merge_dict(self._item_opts, merge_dict(self._preset_opts, default_opts)))
 
+        if self._config.disable_exec:
+            stripped: list[str] = []
+            if data.pop("netrc_cmd", None):
+                stripped.append("netrc_cmd")
+
+            if "postprocessors" in data:
+                exec_pps = [
+                    pp for pp in data["postprocessors"] if isinstance(pp, dict) and pp.get("key", "").startswith("Exec")
+                ]
+                if exec_pps:
+                    stripped.extend(pp.get("key") for pp in exec_pps)
+                    data["postprocessors"] = [
+                        pp
+                        for pp in data["postprocessors"]
+                        if not (isinstance(pp, dict) and pp.get("key", "").startswith("Exec"))
+                    ]
+
+            if stripped:
+                LOG.warning(
+                    "Stripped %d dangerous options from yt-dlp options.",
+                    len(stripped),
+                    extra={"stripped": stripped, "reason": "YTP_DISABLE_EXEC is enabled"},
+                )
+
         if not keep:
             self.reset()
 
@@ -403,7 +432,7 @@ class YTDLPOpts:
                 data["format"] = data["format"][1:]
 
         if self._config.debug:
-            LOG.debug(f"Final yt-dlp options: '{data!s}'.")
+            LOG.debug("Prepared final yt-dlp options.", extra={"ytdlp_options": data})
 
         return data
 

app/library/BackgroundWorker.py

@@ -1,15 +1,16 @@
 import asyncio
 import inspect
-import logging
 import threading
 from queue import Empty, Queue
 
 from aiohttp import web
 
+from app.library.log import get_logger
+
 from .Services import Services
 from .Singleton import Singleton
 
-LOG: logging.Logger = logging.getLogger("BackgroundWorker")
+LOG = get_logger()
 
 
 class CloseThread:
@@ -28,7 +29,7 @@ class BackgroundWorker(metaclass=Singleton):
         "The queue to hold the tasks."
         self.running = True
         "Whether the background worker is running or not."
-        self.thread: threading.Thread = None
+        self.thread: threading.Thread | None = None
         "The thread that runs the background worker."
 
     @staticmethod
@@ -39,19 +40,23 @@ class BackgroundWorker(metaclass=Singleton):
         Services.get_instance().add("background_worker", self)
         app.on_shutdown.append(self.on_shutdown)
 
-        LOG.debug("Starting background worker...")
+        LOG.debug("Started background worker thread.")
         self.thread = threading.Thread(target=self._run, daemon=True)
         self.thread.start()
 
     async def on_shutdown(self, _: web.Application):
         self.running = False
         try:
-            LOG.debug("Shutting down background worker...")
+            LOG.debug("Stopping background worker thread.")
             self.queue.put((CloseThread, (), {}))
-            self.thread.join(timeout=5)
-            LOG.debug("Background worker has been shut down.")
+            if self.thread:
+                self.thread.join(timeout=5)
+            LOG.debug("Background worker thread has been shut down.")
         except Exception as e:
-            LOG.error(f"Failed to shut down background worker: {e}")
+            LOG.exception(
+                "Failed to shut down background worker thread.",
+                extra={"exception_type": type(e).__name__},
+            )
 
     def _run(self):
         asyncio.set_event_loop(asyncio.new_event_loop())
@@ -79,8 +84,16 @@ class BackgroundWorker(metaclass=Singleton):
                     if inspect.iscoroutine(result):
                         loop.call_soon_threadsafe(loop.create_task, result)
                 except Exception as e:
-                    LOG.exception(e)
-                    LOG.error(f"Failed to run '{fn.__name__}'. {e!s}")
+                    function = getattr(fn, "__name__", fn.__class__.__name__)
+                    LOG.exception(
+                        "Failed to run background worker function '%s'.",
+                        function,
+                        extra={
+                            "function": function,
+                            "thread_name": threading.current_thread().name,
+                            "exception_type": type(e).__name__,
+                        },
+                    )
             except Empty:
                 continue
 
@@ -88,9 +101,12 @@ class BackgroundWorker(metaclass=Singleton):
             loop.call_soon_threadsafe(loop.stop)
             loop_thread.join(timeout=5)
             loop.close()
-            LOG.debug("Event loop has been stopped and closed.")
+            LOG.debug("Stopped background worker event loop.")
         except Exception as e:
-            LOG.error(f"Failed to stop the event loop: {e!s}")
+            LOG.exception(
+                "Failed to stop background worker event loop.",
+                extra={"thread_name": loop_thread.name, "exception_type": type(e).__name__},
+            )
 
     def submit(self, fn, *args, **kwargs):
         self.queue.put((fn, args, kwargs))

app/library/DataStore.py

@@ -1,15 +1,16 @@
 import copy
-import logging
 from collections import OrderedDict
 from collections.abc import Iterable
 from enum import Enum
 
+from app.library.log import get_logger
+
 from .downloads import Download
 from .ItemDTO import ItemDTO
 from .operations import matches_condition
 from .sqlite_store import SqliteStore
 
-LOG = logging.getLogger("datastore")
+LOG = get_logger()
 
 
 class StoreType(str, Enum):
@@ -120,9 +121,57 @@ class DataStore:
 
         return None
 
+    async def get_many_by_ids(self, ids: Iterable[str]) -> list[tuple[str, Download]]:
+        ids_list = list(ids)
+        if not ids_list:
+            return []
+
+        items: list[tuple[str, Download]] = []
+        missing_ids: list[str] = []
+
+        for item_id in ids_list:
+            cached = self._dict.get(item_id)
+            if cached:
+                items.append((item_id, cached))
+                continue
+            missing_ids.append(item_id)
+
+        if StoreType.HISTORY == self._type and missing_ids:
+            loaded = await self._connection.get_many_by_ids(str(self._type), missing_ids)
+            for item_id, item in loaded:
+                self._dict[item_id] = Download(info=item)
+
+        items.extend((item_id, download) for item_id in ids_list if (download := self._dict.get(item_id)))
+
+        seen: set[str] = set()
+        ordered: list[tuple[str, Download]] = []
+        for item_id, download in items:
+            if item_id in seen:
+                continue
+            seen.add(item_id)
+            ordered.append((item_id, download))
+        return ordered
+
+    async def get_many_by_status(self, status_filter: str) -> list[tuple[str, Download]]:
+        if StoreType.HISTORY != self._type:
+            return []
+
+        items = await self._connection.get_many_by_status(str(self._type), status_filter)
+        downloads: list[tuple[str, Download]] = []
+        for item_id, item in items:
+            self._dict[item_id] = Download(info=item)
+            downloads.append((item_id, self._dict[item_id]))
+        return downloads
+
     def items(self):
         return self._dict.items()
 
+    def __contains__(self, key: str) -> bool:
+        return key in self._dict
+
+    def __len__(self) -> int:
+        return len(self._dict)
+
     async def put(self, value: Download, no_notify: bool = False) -> Download:
         _ = no_notify
         self._dict[value.info._id] = value
@@ -175,11 +224,41 @@ class DataStore:
         return [(item_id, Download(info=item)) for item_id, item in items], total_items, current_page, total_pages
 
     async def bulk_delete(self, ids: Iterable[str]) -> int:
-        deleted = await self._connection.bulk_delete(str(self._type), ids)
-        for _id in ids:
+        ids_list = list(ids)
+        deleted = await self._connection.bulk_delete(str(self._type), ids_list)
+        for _id in ids_list:
             self._dict.pop(_id, None)
         return deleted
 
+    async def bulk_delete_by_status(self, status_filter: str) -> int:
+        deleted = await self._connection.bulk_delete_by_status(str(self._type), status_filter)
+        if deleted > 0:
+            self._drop_cached_by_status(status_filter)
+        return deleted
+
+    def _drop_cached_by_status(self, status_filter: str) -> None:
+        raw_statuses = [entry.strip() for entry in status_filter.split(",") if entry.strip()]
+        if not raw_statuses:
+            return
+
+        if all(entry.startswith("!") for entry in raw_statuses):
+            excluded = {entry[1:].strip() for entry in raw_statuses if entry[1:].strip()}
+            if not excluded:
+                return
+
+            for item_id, download in list(self._dict.items()):
+                if download.info and download.info.status not in excluded:
+                    self._dict.pop(item_id, None)
+            return
+
+        included = {entry for entry in raw_statuses if not entry.startswith("!")}
+        if not included:
+            return
+
+        for item_id, download in list(self._dict.items()):
+            if download.info and download.info.status in included:
+                self._dict.pop(item_id, None)
+
     async def test(self) -> bool:
         await self._connection.count(str(self._type))
         return True

app/library/Events.py

@@ -1,16 +1,16 @@
 import asyncio
 import datetime
-import logging
 from collections.abc import Callable
 from dataclasses import dataclass, field
 from typing import Any
 
 from app.features.core.utils import gen_random
+from app.library.log import get_logger
 
 from .BackgroundWorker import BackgroundWorker
 from .Singleton import Singleton
 
-LOG: logging.Logger = logging.getLogger("events")
+LOG = get_logger()
 
 
 class Events:
@@ -34,9 +34,11 @@ class Events:
 
     ITEM_ADDED: str = "item_added"
     ITEM_UPDATED: str = "item_updated"
+    ITEM_PROGRESS: str = "item_progress"
     ITEM_COMPLETED: str = "item_completed"
     ITEM_CANCELLED: str = "item_cancelled"
     ITEM_DELETED: str = "item_deleted"
+    ITEM_BULK_DELETED: str = "item_bulk_deleted"
     ITEM_PAUSED: str = "item_paused"
     ITEM_RESUMED: str = "item_resumed"
     ITEM_MOVED: str = "item_moved"
@@ -85,8 +87,10 @@ class Events:
             Events.LOG_SUCCESS,
             Events.ITEM_ADDED,
             Events.ITEM_UPDATED,
+            Events.ITEM_PROGRESS,
             Events.ITEM_CANCELLED,
             Events.ITEM_DELETED,
+            Events.ITEM_BULK_DELETED,
             Events.ITEM_MOVED,
             Events.ITEM_STATUS,
             Events.PAUSED,
@@ -101,7 +105,7 @@ class Events:
             list: The list of debug events.
 
         """
-        return [Events.ITEM_UPDATED]
+        return [Events.ITEM_UPDATED, Events.ITEM_PROGRESS]
 
 
 @dataclass(kw_only=True)
@@ -240,7 +244,12 @@ class EventBus(metaclass=Singleton):
                 event = Events.frontend()
             else:
                 if event not in all_events:
-                    LOG.error(f"'{name}' attempted to listen on '{event}' which does not exist.")
+                    LOG.error(
+                        "Listener '%s' tried to subscribe to unknown event '%s'.",
+                        name,
+                        event,
+                        extra={"listener": name, "event": event},
+                    )
                     return self
 
                 event = [event]
@@ -250,7 +259,12 @@ class EventBus(metaclass=Singleton):
 
         for e in event:
             if e not in all_events:
-                LOG.error(f"'{name}' attempted to listen on '{e}' which does not exist.")
+                LOG.error(
+                    "Listener '%s' tried to subscribe to unknown event '%s'.",
+                    name,
+                    e,
+                    extra={"listener": name, "event": e},
+                )
                 continue
 
             if e not in self._listeners:
@@ -259,7 +273,7 @@ class EventBus(metaclass=Singleton):
             self._listeners[e] = [(n, listener) for n, listener in self._listeners[e] if n != name]
             self._listeners[e].append((name, EventListener(name, callback)))
 
-        LOG.debug(f"'{name}' subscribed to '{event}'.")
+        LOG.debug("Listener '%s' has subscribed.", name, extra={"listener": name, "events": event})
 
         return self
 
@@ -287,7 +301,7 @@ class EventBus(metaclass=Singleton):
                     events.append(e)
 
         if len(events) > 0:
-            LOG.debug(f"'{name}' unsubscribed from '{events}'.")
+            LOG.debug("Listener '%s' unsubscribed from '%s'.", name, events, extra={"listener": name, "events": events})
 
         return self
 
@@ -314,7 +328,17 @@ class EventBus(metaclass=Singleton):
         ev = Event(event=event, title=title, message=message, data=data)
 
         if self.debug or event not in Events.only_debug():
-            LOG.debug(f"Emitting event '{ev.id}: {ev.event}'.", extra={"data": data})
+            LOG.debug(
+                "Delivering '%s' event to %s listener(s).",
+                ev.event,
+                len(self._listeners[event]),
+                extra={
+                    "event_id": ev.id,
+                    "event": ev.event,
+                    "handler_count": len(self._listeners[event]),
+                    "data": data,
+                },
+            )
 
         try:
             loop = asyncio.get_running_loop()
@@ -327,16 +351,35 @@ class EventBus(metaclass=Singleton):
                             if asyncio.iscoroutine(coro):
                                 await coro
                             else:
-                                LOG.warning(f"Expected coroutine from async handler '{handler.name}', got {type(coro)}")
+                                LOG.warning(
+                                    "Async handler '%s' returned '%s' instead of a coroutine.",
+                                    handler.name,
+                                    type(coro).__name__,
+                                    extra={"handler": handler.name, "returned_type": type(coro).__name__},
+                                )
                         else:
                             await self._call(handler, ev, kwargs)
                     except Exception as e:
-                        LOG.exception(e)
-                        LOG.error(f"Failed to emit event '{ev.event}' to '{handler.name}'. Error message '{e!s}'.")
+                        LOG.exception(
+                            "Failed to emit '%s' event to listener '%s'.",
+                            ev.event,
+                            handler.name,
+                            extra={
+                                "event_id": ev.id,
+                                "event": ev.event,
+                                "handler": handler.name,
+                                "exception_type": type(e).__name__,
+                            },
+                        )
 
             loop.create_task(execute_handlers())
         except RuntimeError:
-            LOG.debug(f"No event loop detected - using BackgroundWorker for {len(self._listeners[event])} handlers")
+            LOG.debug(
+                "No event loop detected; offloading '%s' event to %s background listener(s).",
+                ev.event,
+                len(self._listeners[event]),
+                extra={"event_id": ev.id, "event": ev.event, "handler_count": len(self._listeners[event])},
+            )
             for _, handler in self._listeners[event]:
                 try:
                     if not self._offload:
@@ -344,8 +387,17 @@ class EventBus(metaclass=Singleton):
 
                     self._offload.submit(handler.handle, ev, **kwargs)
                 except Exception as e:
-                    LOG.exception(e)
-                    LOG.error(f"Failed to emit event '{ev.event}' to '{handler.name}'. Error message '{e!s}'.")
+                    LOG.exception(
+                        "Failed to offload '%s' event to listener '%s'.",
+                        ev.event,
+                        handler.name,
+                        extra={
+                            "event_id": ev.id,
+                            "event": ev.event,
+                            "handler": handler.name,
+                            "exception_type": type(e).__name__,
+                        },
+                    )
 
     def clear(self) -> None:
         """

app/library/HttpAPI.py

@@ -1,15 +1,19 @@
 import base64
 import hmac
 import inspect
-import logging
-from collections.abc import Awaitable
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
+from typing import Any, cast
 
 import anyio
 from aiohttp import web
-from aiohttp.web import Request, RequestHandler, Response
+from aiohttp.typedefs import Handler, Middleware
+from aiohttp.web import Request, Response
+from aiohttp.web_log import AccessLogger
+from aiohttp.web_request import BaseRequest
+from aiohttp.web_response import StreamResponse
 
+from app.library.log import get_logger
 from app.library.Services import Services
 
 from .cache import Cache
@@ -19,7 +23,33 @@ from .Events import EventBus
 from .router import RouteType, get_routes
 from .Utils import decrypt_data, encrypt_data, get_file, load_modules
 
-LOG: logging.Logger = logging.getLogger("http_api")
+LOG = get_logger("http")
+
+
+class HttpAccessLogger(AccessLogger):
+    def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
+        try:
+            fmt_info = self._format_line(request, response, time)
+
+            values: list[object] = []
+            extra: dict[str, Any] = {"elapsed_ms": round(time * 1000.0, 2)}
+            for key, value in fmt_info:
+                values.append(value)
+
+                if isinstance(key, str):
+                    extra[key] = value
+                else:
+                    parent, child = key
+                    group: dict[str, Any] = extra.get(parent, {})  # type: ignore[assignment]
+                    if not isinstance(group, dict):
+                        group = {}
+                    group[child] = value
+                    extra[parent] = group
+
+            message = self._log_format % tuple(values)
+            self.logger.info(message, extra=extra)
+        except Exception:
+            self.logger.exception("Error in logging")
 
 
 class HttpAPI:
@@ -89,7 +119,10 @@ class HttpAPI:
         try:
             app.on_response_prepare.append(on_prepare)
         except Exception as e:
-            LOG.exception(e)
+            LOG.exception(
+                "Failed to register response preparation middleware.",
+                extra={"operation": "register_on_response_prepare", "exception_type": type(e).__name__},
+            )
 
         app.on_shutdown.append(self.on_shutdown)
 
@@ -107,10 +140,10 @@ class HttpAPI:
         registered_options: list = []
 
         base_path: str = self.config.base_path.rstrip("/")
-        from app.routes.api._static import preload_static
+        from app.routes.api._static import setup_static_routes
 
         load_modules(self.rootPath, self.rootPath / "routes" / "api")
-        preload_static(self.rootPath, self.config)
+        setup_static_routes(self.rootPath, self.config)
 
         async def options_handler(_: Request) -> Response:
             return web.Response(status=204)
@@ -130,7 +163,13 @@ class HttpAPI:
                 route.path = f"{base_path}/{route.path.lstrip('/')}"
 
             if self.config.debug:
-                LOG.debug(f"Add ({route.name}) {route.method}: {route.path}.")
+                LOG.debug(
+                    "Adding route '%s' %s: %s.",
+                    route.name,
+                    route.method,
+                    route.path,
+                    extra={"route_name": route.name, "method": route.method, "path": route.path},
+                )
 
             app.router.add_route(route.method, route.path, handler=_handle(route.handler), name=route.name)
 
@@ -141,7 +180,7 @@ class HttpAPI:
             registered_options.append(route.path)
 
     @staticmethod
-    def basic_auth(username: str, password: str) -> Awaitable:
+    def basic_auth(username: str, password: str) -> Middleware:
         """
         Middleware to handle basic authentication.
 
@@ -155,7 +194,7 @@ class HttpAPI:
         """
 
         @web.middleware
-        async def auth_handler(request: Request, handler: RequestHandler) -> Response:
+        async def auth_handler(request: Request, handler: Handler) -> StreamResponse:
             # if OPTIONS request, skip auth
             if "OPTIONS" == request.method:
                 return await handler(request)
@@ -168,12 +207,19 @@ class HttpAPI:
                 auth_cookie = request.cookies.get("auth")
                 if auth_cookie is not None:
                     try:
-                        data = decrypt_data(auth_cookie, key=Config.get_instance().secret_key)
+                        data = decrypt_data(auth_cookie, key=cast("bytes", Config.get_instance().secret_key))
                         if data is not None:
                             data = base64.b64encode(data.encode()).decode()
                             auth_header = f"Basic {data}"
                     except Exception as e:
-                        LOG.exception(e)
+                        LOG.exception(
+                            "Failed to decrypt authentication cookie.",
+                            extra={
+                                "route": str(request.rel_url),
+                                "method": request.method,
+                                "exception_type": type(e).__name__,
+                            },
+                        )
 
             if auth_header is None:
                 return web.json_response(
@@ -207,7 +253,7 @@ class HttpAPI:
                     },
                 )
 
-            response: Response = await handler(request)
+            response: StreamResponse = await handler(request)
 
             contentType: str | None = response.headers.get("content-type", None)
             if contentType and not contentType.startswith("text/html"):
@@ -218,7 +264,7 @@ class HttpAPI:
                     "auth",
                     encrypt_data(
                         f"{user_name}:{user_password}",
-                        key=Config.get_instance().secret_key,
+                        key=cast("bytes", Config.get_instance().secret_key),
                     ),
                     max_age=60 * 60 * 24 * 7,
                     expires=(datetime.now(UTC) + timedelta(days=7)).strftime("%a, %d %b %Y %H:%M:%S GMT"),
@@ -226,16 +272,20 @@ class HttpAPI:
                     samesite="Strict",
                 )
             except Exception as e:
-                LOG.exception(e)
+                LOG.exception(
+                    "Failed to set authentication cookie for '%s'.",
+                    request.rel_url,
+                    extra={"route": str(request.rel_url), "method": request.method, "exception_type": type(e).__name__},
+                )
 
             return response
 
         return auth_handler
 
     @staticmethod
-    def middle_wares(app: web.Application, base_path: str, download_path: str) -> Awaitable:
+    def middle_wares(app: web.Application, base_path: str, download_path: str) -> Middleware:
         @web.middleware
-        async def middleware_handler(request: Request, handler: RequestHandler) -> Response:
+        async def middleware_handler(request: Request, handler: Handler) -> StreamResponse:
             static_path = str(app.router["download_static"].url_for(filename=""))
             if request.path.startswith(static_path):
                 realFile, status = get_file(
@@ -254,12 +304,13 @@ class HttpAPI:
                         },
                     )
 
-            response: Response = await handler(request)
+            response: StreamResponse = await handler(request)
 
-            contentType: str = response.headers.get("content-type", "")
+            contentType: str = str(response.headers.get("content-type", ""))
             if contentType.startswith("text/html") and getattr(response, "_path", None):
                 rewrite_path: str = base_path.rstrip("/")
-                async with await anyio.open_file(response._path, "rb") as f:
+                response_path = cast("Any", response)._path
+                async with await anyio.open_file(response_path, "rb") as f:
                     content = await f.read()
                     content: str = (
                         content.decode("utf-8")
@@ -277,13 +328,13 @@ class HttpAPI:
                     new_response.set_cookie(
                         morsel.key,
                         morsel.value,
-                        expires=morsel["expires"],
-                        domain=morsel["domain"],
-                        max_age=morsel["max-age"],
-                        path=morsel["path"],
-                        secure=bool(morsel["secure"]),
-                        httponly=bool(morsel["httponly"]),
-                        samesite=morsel["samesite"] or None,
+                        expires=morsel.get("expires"),
+                        domain=morsel.get("domain"),
+                        max_age=morsel.get("max-age"),
+                        path=morsel.get("path") or "/",
+                        secure=bool(morsel.get("secure")),
+                        httponly=bool(morsel.get("httponly")),
+                        samesite=morsel.get("samesite") or None,
                     )
 
                 return new_response
@@ -292,7 +343,7 @@ class HttpAPI:
 
         return middleware_handler
 
-    async def _handle(self, handler: RequestHandler, request: Request) -> Response:
+    async def _handle(self, handler: Handler, request: Request) -> StreamResponse:
         """
         Call the handler with the request and return the response.
 
@@ -314,7 +365,16 @@ class HttpAPI:
                 else:
                     response = await Services.get_instance().handle_async(handler, request=request)
             except TypeError as te:
-                LOG.exception(te)
+                LOG.exception(
+                    "Failed to inject route handler dependencies for '%s'.",
+                    getattr(handler, "__name__", handler.__class__.__name__),
+                    extra={
+                        "handler": getattr(handler, "__name__", handler.__class__.__name__),
+                        "route": str(request.rel_url),
+                        "method": request.method,
+                        "exception_type": type(te).__name__,
+                    },
+                )
                 if "missing 1 required positional argument" in str(te) and "request" in str(te):
                     response = await handler(request)
                 else:
@@ -322,7 +382,12 @@ class HttpAPI:
         except web.HTTPException as e:
             return web.json_response(data={"error": str(e)}, status=e.status_code)
         except Exception as e:
-            LOG.exception(e)
+            LOG.exception(
+                "Failed to handle request '%s %s'.",
+                request.method,
+                request.rel_url,
+                extra={"route": str(request.rel_url), "method": request.method, "exception_type": type(e).__name__},
+            )
             response = web.json_response(
                 data={"error": "Internal Server Error"},
                 status=web.HTTPInternalServerError.status_code,

app/library/HttpSocket.py

@@ -1,13 +1,13 @@
 import asyncio
 import functools
 import json
-import logging
 from pathlib import Path
 from typing import Any
 
 from aiohttp import web
 
 from app.features.core.utils import gen_random
+from app.library.log import get_logger
 from app.library.router import Route, RouteType, get_routes
 from app.library.Services import Services
 from app.library.Utils import load_modules
@@ -17,7 +17,7 @@ from .encoder import Encoder
 from .Events import Event, EventBus, Events
 from .ItemDTO import Item
 
-LOG: logging.Logger = logging.getLogger("socket_api")
+LOG = get_logger()
 
 
 class WebSocketHub:
@@ -104,19 +104,20 @@ class HttpSocket:
             }
         )
 
-        self._notify.subscribe("frontend", event_handler, f"{__class__.__name__}.emit")
+        self._notify.subscribe("frontend", event_handler, f"{HttpSocket.__name__}.emit")
 
     @staticmethod
-    def ws_event(func):  # type: ignore
+    def ws_event(func):
         """
         Decorator to mark a method as a socket event.
         """
 
-        @functools.wraps(func)  # type: ignore
+        @functools.wraps(func)
         async def wrapper(*args, **kwargs):
-            return await func(*args, **kwargs)  # type: ignore
+            return await func(*args, **kwargs)
 
-        wrapper._ws_event = func.__name__  # type: ignore
+        wrapper_with_event: Any = wrapper
+        wrapper_with_event._ws_event = func.__name__
         return wrapper
 
     async def on_shutdown(self, _: web.Application):
@@ -134,9 +135,11 @@ class HttpSocket:
             if not (data and data.data):
                 return
 
-            await Services.get_instance().get("queue").add(item=Item.format(data.data))
+            queue = Services.get_instance().get("queue")
+            if queue is not None:
+                await queue.add(item=Item.format(data.data))
 
-        self._notify.subscribe(Events.ADD_URL, event_handler, f"{__class__.__name__}.add")
+        self._notify.subscribe(Events.ADD_URL, event_handler, f"{HttpSocket.__name__}.add")
 
         load_modules(self.rootPath, self.rootPath / "routes" / "socket")
 
@@ -145,7 +148,10 @@ class HttpSocket:
         for route in socket_routes.values():
             if self.config.debug:
                 LOG.debug(
-                    f"Add ({route.name}) {route.method.value if isinstance(route.method, RouteType) else route.method}: {route.path}."
+                    "Registered socket route '%s' for %s %s.",
+                    route.name,
+                    route.method.value if isinstance(route.method, RouteType) else route.method,
+                    route.path,
                 )
 
         async def handle_message(sid: str, message: str) -> None:
@@ -161,7 +167,7 @@ class HttpSocket:
                 return
 
             if not (route := socket_routes.get(event)):
-                LOG.debug(f"Unknown websocket event '{event}'.")
+                LOG.debug("Unknown WebSocket event '%s'.", event, extra={"sid": sid, "event": event})
                 return
 
             data = payload.get("data") if isinstance(payload, dict) else None
@@ -183,7 +189,7 @@ class HttpSocket:
 
             sid: str = gen_random(14)
             self.sio.add(sid, ws)
-            LOG.debug(f"WebSocket client '{sid}' connected.")
+            LOG.debug("WebSocket client '%s' connected.", sid, extra={"sid": sid})
 
             await handle_connect(sid)
 
@@ -192,8 +198,11 @@ class HttpSocket:
                 try:
                     await handle_message(sid, message)
                 except Exception as e:
-                    LOG.exception(e)
-                    LOG.error(f"Error handling WebSocket message from client '{sid}': {e}")
+                    LOG.exception(
+                        "Failed to handle WebSocket message from client '%s'.",
+                        sid,
+                        extra={"sid": sid, "message_size": len(message), "exception_type": type(e).__name__},
+                    )
 
             try:
                 i: int = 0
@@ -202,15 +211,22 @@ class HttpSocket:
                         i = i + 1
                         asyncio.create_task(handle_message_safe(sid, msg.data), name=f"ws_msg_{sid}_{i}")
                     elif msg.type == web.WSMsgType.ERROR:
-                        LOG.error(f"WebSocket connection closed with exception {ws.exception()}")
+                        LOG.error(
+                            "WebSocket connection closed with exception %s",
+                            ws.exception(),
+                            extra={
+                                "sid": sid,
+                                "exception": str(ws.exception()) if ws.exception() else None,
+                            },
+                        )
             finally:
                 await handle_disconnect(sid, getattr(ws, "close_reason", None))
                 self.sio.remove(sid)
-                LOG.debug(f"WebSocket client '{sid}' disconnected.")
+                LOG.debug("WebSocket client '%s' disconnected.", sid, extra={"sid": sid})
 
             return ws
 
         if self.config.debug:
-            LOG.debug(f"Add (ws) GET: {ws_path}.")
+            LOG.debug("Registered WebSocket endpoint at '%s'.", ws_path, extra={"method": "GET", "path": ws_path})
 
         app.router.add_get(ws_path, ws_handler, name="ws")

app/library/ItemDTO.py

@@ -1,4 +1,3 @@
-import logging
 import re
 import time
 import uuid
@@ -10,12 +9,13 @@ from typing import TYPE_CHECKING, Any
 from app.features.ytdlp.utils import archive_add, archive_delete, archive_read, get_archive_id
 from app.features.ytdlp.ytdlp_opts import YTDLPOpts
 from app.library.encoder import Encoder
+from app.library.log import get_logger
 from app.library.Utils import clean_item, get_file, get_file_sidecar
 
 if TYPE_CHECKING:
     from app.features.presets.schemas import Preset
 
-LOG: logging.Logger = logging.getLogger("ItemDTO")
+LOG = get_logger()
 
 
 @dataclass(kw_only=True)
@@ -93,7 +93,7 @@ class Item:
             bool: True if the item has extras, False otherwise.
 
         """
-        return self.extras and len(self.extras) > 0
+        return bool(self.extras and len(self.extras) > 0)
 
     def add_extras(self, key: str, value: Any) -> None:
         """
@@ -117,7 +117,7 @@ class Item:
             bool: True if the item has command options for yt, False otherwise.
 
         """
-        return self.cli and len(self.cli) > 2
+        return bool(self.cli and len(self.cli) > 2)
 
     @staticmethod
     def _default_preset() -> str:
@@ -161,7 +161,7 @@ class Item:
             Item: The formatted item.
 
         """
-        url: str = item.get("url")
+        url = item.get("url")
 
         if not url or not isinstance(url, str):
             msg = "url param is required."
@@ -173,7 +173,7 @@ class Item:
         if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url):
             url = f"https://www.youtube.com/watch?v={url}"
 
-        data: dict[str, str] = {"url": url}
+        data: dict[str, Any] = {"url": url}
 
         preset: str | None = item.get("preset")
         if preset and isinstance(preset, str) and preset != Item._default_preset():
@@ -185,24 +185,28 @@ class Item:
 
             data["preset"] = preset
 
-        if item.get("folder") and isinstance(item.get("folder"), str):
-            data["folder"] = item.get("folder")
+        folder = item.get("folder")
+        if isinstance(folder, str) and folder:
+            data["folder"] = folder
 
-        if item.get("cookies") and isinstance(item.get("cookies"), str):
-            data["cookies"] = item.get("cookies")
+        cookies = item.get("cookies")
+        if isinstance(cookies, str) and cookies:
+            data["cookies"] = cookies
 
-        if item.get("template") and isinstance(item.get("template"), str):
-            data["template"] = item.get("template")
+        template = item.get("template")
+        if isinstance(template, str) and template:
+            data["template"] = template
 
-        if "auto_start" in item and isinstance(item.get("auto_start"), bool):
-            data["auto_start"] = bool(item.get("auto_start"))
+        if isinstance(item.get("auto_start"), bool):
+            data["auto_start"] = item["auto_start"]
 
         extras: dict | None = item.get("extras")
-        if extras and isinstance(extras, dict) and len(extras) > 0:
+        if isinstance(extras, dict) and len(extras) > 0:
             data["extras"] = extras
 
-        if item.get("requeued") and isinstance(item.get("requeued"), bool):
-            data["requeued"] = item.get("requeued")
+        requeued = item.get("requeued")
+        if isinstance(requeued, bool):
+            data["requeued"] = requeued
 
         cli: str | None = item.get("cli")
         if cli and len(cli) > 2:
@@ -381,6 +385,8 @@ class ItemDTO:
     """ The archive ID of the item. """
     sidecar: dict = field(default_factory=dict)
     """ Sidecar data associated with the item. """
+    download_skipped: bool = False
+    """ True when yt-dlp intentionally skips the primary media download. """
 
     # yt-dlp injected fields.
     tmpfilename: str | None = None
@@ -572,8 +578,10 @@ class ItemDTO:
         self.is_archivable = bool(self._archive_file)
         if not self.is_archivable:
             self.is_archived = False
-        else:
+        elif self.archive_id and self._archive_file:
             self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
+        else:
+            self.is_archived = False
 
     def get_archive_file(self) -> str | None:
         """

app/library/PackageInstaller.py

@@ -1,14 +1,15 @@
 import importlib
 import importlib.metadata
-import logging
 import os
 import subprocess
 import sys
 from pathlib import Path
 
+from app.library.log import get_logger
+
 from .httpx_client import sync_client
 
-LOG: logging.Logger = logging.getLogger("package_installer")
+LOG = get_logger()
 
 
 def parse_version(v: str) -> tuple[int, ...]:
@@ -16,7 +17,7 @@ def parse_version(v: str) -> tuple[int, ...]:
 
 
 class Packages:
-    def __init__(self, env: str | None, file: str | None, upgrade: bool = False):
+    def __init__(self, env: str | None, file: str | Path | None, upgrade: bool = False):
         from_env: list[str] = env.split() if env else []
         from_file = []
 
@@ -64,19 +65,34 @@ class PackageInstaller:
         current_version: str | None = self._get_installed_version(pkg)
 
         if current_version and version and self.compare_versions(current_version, version):
-            LOG.info(f"'{pkg}' is already installed with the specified version ({version}). Skipping installation.")
+            LOG.info(
+                "Package '%s' is already installed at version '%s'; skipping installation.",
+                pkg,
+                version,
+                extra={"package": pkg, "installed_version": current_version, "requested_version": version},
+            )
             return
 
         if upgrade and current_version and not version:
             latest_version: str | None = self._get_latest_version(pkg)
             if latest_version and parse_version(current_version) >= parse_version(latest_version):
-                LOG.info(f"'{pkg}' is already the latest version ({current_version}). Skipping upgrade.")
+                LOG.info(
+                    "Package '%s' is already at the latest version '%s'; skipping upgrade.",
+                    pkg,
+                    current_version,
+                    extra={"package": pkg, "installed_version": current_version, "latest_version": latest_version},
+                )
                 return
 
         if current_version:
-            LOG.info(f"'{pkg}' is already installed (version: {current_version}). Proceeding with upgrade...")
+            LOG.info(
+                "Package '%s' is installed at '%s'; proceeding with upgrade.",
+                pkg,
+                current_version,
+                extra={"package": pkg, "installed_version": current_version},
+            )
         else:
-            LOG.info(f"'{pkg}' is not installed. Installing...")
+            LOG.info("Package '%s' is not installed; installing it.", pkg, extra={"package": pkg})
 
         self._install_pkg(pkg, version=version)
 
@@ -93,9 +109,20 @@ class PackageInstaller:
                 resp = client.get(url)
                 if 200 == resp.status_code:
                     return resp.json()["info"]["version"]
-                LOG.warning(f"Failed to fetch '{pkg}' info: HTTP {resp.status_code}")
+                LOG.warning(
+                    "Failed to fetch PyPI metadata for '%s': HTTP %s.",
+                    pkg,
+                    resp.status_code,
+                    extra={"package": pkg, "status_code": resp.status_code, "url": url},
+                )
         except Exception as e:
-            LOG.warning(f"Error while querying PyPI for '{pkg}': {e}")
+            LOG.warning(
+                "Failed to query PyPI for '%s': %s",
+                pkg,
+                e,
+                extra={"package": pkg, "url": url, "exception_type": type(e).__name__},
+                exc_info=True,
+            )
         return None
 
     def _install_pkg(self, pkg: str, version: str | None = None) -> bool:
@@ -135,7 +162,12 @@ class PackageInstaller:
 
             return 0 == proc.returncode
         except subprocess.CalledProcessError as e:
-            LOG.error(f"Failed to install '{pkg}' (exit {e.returncode}). {e!s}")
+            LOG.exception(
+                "Failed to install package '%s' (exit %s).",
+                pkg,
+                e.returncode,
+                extra={"package": pkg, "returncode": e.returncode, "exception_type": type(e).__name__},
+            )
             self.out(out=e.stdout, err=e.stderr)
             raise
 
@@ -150,13 +182,20 @@ class PackageInstaller:
         if not pkgs.has_packages() or not self.user_site:
             return
 
-        LOG.info(f"Checking for user pip packages: {', '.join(pkgs.packages)}")
+        LOG.info("Checking configured user pip packages.", extra={"packages": pkgs.packages})
         for package in pkgs.packages:
             try:
                 self.action(package, upgrade=pkgs.allow_upgrade())
             except Exception as e:
-                LOG.error(f"Failed to install or upgrade package '{package}'. Error message: {e!s}")
-                LOG.exception(e)
+                LOG.exception(
+                    "Failed to install or upgrade package '%s'.",
+                    package,
+                    extra={
+                        "package": package,
+                        "allow_upgrade": pkgs.allow_upgrade(),
+                        "exception_type": type(e).__name__,
+                    },
+                )
 
     def compare_versions(self, current: str, target: str) -> bool:
         """

app/library/Scheduler.py

@@ -1,13 +1,16 @@
 import asyncio
-import logging
+from collections.abc import Callable
+from typing import Any
 
 from aiocron import Cron
 from aiohttp import web
 
+from app.library.log import get_logger
+
 from .Events import EventBus, Events
 from .Singleton import Singleton
 
-LOG = logging.getLogger("scheduler")
+LOG = get_logger()
 
 
 class Scheduler(metaclass=Singleton):
@@ -46,10 +49,13 @@ class Scheduler(metaclass=Singleton):
         for job in self._jobs:
             try:
                 self._jobs[job].stop()
-                LOG.debug(f"Stopped job '{job}'.")
+                LOG.debug("Stopped job '%s'.", job, extra={"job_id": job})
             except Exception as e:
-                LOG.exception(e)
-                LOG.error(f"Failed to stop job '{job}'. Error message '{e!s}'.")
+                LOG.exception(
+                    "Failed to stop scheduler job '%s'.",
+                    job,
+                    extra={"job_id": job, "exception_type": type(e).__name__},
+                )
 
         self._jobs = {}
 
@@ -62,7 +68,7 @@ class Scheduler(metaclass=Singleton):
             if data and data.data:
                 self.add(**data.data)
 
-        EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{__class__.__name__}.add")
+        EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{type(self).__name__}.add")
 
     def get_all(self) -> dict[str, Cron]:
         """Return the jobs."""
@@ -95,7 +101,7 @@ class Scheduler(metaclass=Singleton):
         return id in self._jobs
 
     def add(
-        self, timer: str, func: callable, args: tuple = (), kwargs: dict | None = None, id: str | None = None
+        self, timer: str, func: Callable[..., Any], args: tuple = (), kwargs: dict | None = None, id: str | None = None
     ) -> str:
         """
         Add a job to the schedule.
@@ -127,7 +133,7 @@ class Scheduler(metaclass=Singleton):
 
         self._jobs[job_id] = job
 
-        LOG.debug(f"Added '{job_id}' to the scheduler to run on '{timer}'.")
+        LOG.debug("Added job '%s' to run on '%s'.", job_id, timer, extra={"job_id": job_id, "timer": timer})
 
         return job_id
 
@@ -151,12 +157,15 @@ class Scheduler(metaclass=Singleton):
             try:
                 self._jobs[id].stop()
             except Exception as e:
-                LOG.exception(e)
-                LOG.error(f"Failed to stop job '{id}'. Error message '{e!s}'.")
+                LOG.exception(
+                    "Failed to stop scheduler job '%s'.",
+                    id,
+                    extra={"job_id": id, "exception_type": type(e).__name__},
+                )
                 return False
 
             del self._jobs[id]
-            LOG.debug(f"Removed job '{id}' from the scheduler.")
+            LOG.debug("Removed job '%s' from the scheduler.", id, extra={"job_id": id})
             return True
 
         return False

app/library/Services.py

@@ -1,12 +1,13 @@
 import inspect
-import logging
+from collections.abc import Callable
 from dataclasses import dataclass
 from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints
 
+from app.library.log import get_logger
 from app.library.Singleton import Singleton
 
 T = TypeVar("T")
-LOG: logging.Logger = logging.getLogger(__name__)
+LOG = get_logger()
 
 
 def _unwrap_annotation(ann: Any) -> Any:
@@ -117,7 +118,7 @@ class Services(metaclass=Singleton):
 
         return candidates[0]
 
-    def _build_call_args(self, handler: callable, overrides: dict[str, Any]) -> dict[str, Any]:
+    def _build_call_args(self, handler: Callable[..., Any], overrides: dict[str, Any]) -> dict[str, Any]:
         sig: inspect.Signature = inspect.signature(handler)
 
         try:
@@ -169,10 +170,10 @@ class Services(metaclass=Singleton):
 
         return resolved
 
-    async def handle_async(self, handler: callable, **kwargs) -> Any:
+    async def handle_async(self, handler: Callable[..., Any], **kwargs) -> Any:
         resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
         return await handler(**resolved)
 
-    def handle_sync(self, handler: callable, **kwargs) -> Any:
+    def handle_sync(self, handler: Callable[..., Any], **kwargs) -> Any:
         resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
         return handler(**resolved)

app/library/TerminalSessionManager.py

@@ -0,0 +1,768 @@
+from __future__ import annotations
+
+import asyncio
+import errno
+import json
+import os
+import shlex
+import shutil
+import time
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from aiohttp import web
+
+from app.library.config import Config
+from app.library.log import get_logger
+from app.library.Scheduler import Scheduler
+from app.library.Services import Services
+from app.library.Singleton import Singleton
+
+if TYPE_CHECKING:
+    from asyncio.events import AbstractEventLoop
+    from asyncio.subprocess import Process
+
+    from aiohttp.web import Request
+
+LOG = get_logger()
+
+ACTIVE_FILE_NAME = "active.json"
+METADATA_FILE_NAME = "metadata.json"
+TRANSCRIPT_FILE_NAME = "transcript.jsonl"
+DEFAULT_DRAIN_TTL = 30.0
+COMPLETED_SESSION_RETENTION = 86400.0
+DEFAULT_KEEPALIVE_INTERVAL = 15.0
+DEFAULT_SHUTDOWN_TIMEOUT = 5.0
+CLEANUP_SCHEDULE = "5 */1 * * *"
+
+
+class TerminalSessionConflictError(RuntimeError):
+    pass
+
+
+@dataclass(slots=True)
+class ActiveTerminalSession:
+    session_id: str
+    task: asyncio.Task[None]
+    process: Process | None = None
+    subscribers: set[asyncio.Queue[dict[str, Any] | None]] = field(default_factory=set)
+    interrupted: bool = False
+
+
+class TerminalSessionManager(metaclass=Singleton):
+    def __init__(self) -> None:
+        self.config: Config = Config.get_instance()
+        self.root_path: Path = Path(self.config.config_path) / "runtime" / "terminal"
+        self._lock = asyncio.Lock()
+        self._active: ActiveTerminalSession | None = None
+        self._drain_ttl: float = DEFAULT_DRAIN_TTL
+        self._completed_retention: float = COMPLETED_SESSION_RETENTION
+        self._keepalive_interval: float = DEFAULT_KEEPALIVE_INTERVAL
+        self._shutdown_timeout: float = DEFAULT_SHUTDOWN_TIMEOUT
+        self._cleanup_job_id: str | None = None
+
+    @staticmethod
+    def get_instance() -> TerminalSessionManager:
+        return TerminalSessionManager()
+
+    def attach(self, app: web.Application) -> None:
+        self._ensure_root()
+        Services.get_instance().add("terminal_manager", self)
+        app.on_startup.append(self.on_startup)
+        app.on_shutdown.append(self.on_shutdown)
+        self._cleanup_job_id = Scheduler.get_instance().add(
+            timer=CLEANUP_SCHEDULE,
+            func=self.cleanup,
+            id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
+        )
+
+    async def on_startup(self, _: web.Application) -> None:
+        await self.initialize()
+
+    async def on_shutdown(self, _: web.Application) -> None:
+        if self._cleanup_job_id is not None:
+            Scheduler.get_instance().remove(self._cleanup_job_id)
+            self._cleanup_job_id = None
+
+        session_id: str | None = None
+        task: asyncio.Task[None] | None = None
+
+        async with self._lock:
+            runtime = self._active
+            if runtime is None:
+                return
+
+            runtime.interrupted = True
+            session_id = runtime.session_id
+            task = runtime.task
+
+            self._signal_process(runtime.process)
+            if runtime.process is None and task.done() is False:
+                task.cancel()
+
+            for subscriber in list(runtime.subscribers):
+                subscriber.put_nowait(None)
+
+        assert session_id is not None
+        assert task is not None
+
+        try:
+            await asyncio.wait_for(asyncio.shield(task), timeout=self._shutdown_timeout)
+            return
+        except TimeoutError:
+            async with self._lock:
+                runtime = self._active
+                if runtime is not None and runtime.session_id == session_id:
+                    self._signal_process(runtime.process, force=True)
+                    if task.done() is False:
+                        task.cancel()
+
+            try:
+                await asyncio.wait_for(asyncio.shield(task), timeout=self._shutdown_timeout)
+                return
+            except TimeoutError:
+                LOG.warning("Terminal session '%s' did not finish during shutdown.", session_id)
+
+        await self._force_interrupt_session(session_id)
+
+    async def initialize(self) -> None:
+        async with self._lock:
+            self._ensure_root()
+            self._cleanup_expired_sessions(time.time())
+            self._recover_orphaned_active_session(time.time())
+
+    async def cleanup(self) -> None:
+        async with self._lock:
+            LOG.debug("Running scheduled terminal session cleanup.")
+            removed = self._cleanup_expired_sessions(time.time())
+            LOG.debug("Scheduled terminal session cleanup finished. Removed %d expired sessions.", removed)
+
+    async def list_sessions(self) -> list[dict[str, Any]]:
+        async with self._lock:
+            return self._list_sessions_locked()
+
+    async def create_session(self, command: str) -> dict[str, Any]:
+        async with self._lock:
+            now = time.time()
+
+            active_session = self._get_active_session_locked()
+            if active_session is not None:
+                msg = "A terminal session is already active."
+                raise TerminalSessionConflictError(msg)
+
+            session_id = uuid.uuid4().hex
+            session_dir = self._session_dir(session_id)
+            session_dir.mkdir(parents=True, exist_ok=True)
+
+            metadata = {
+                "session_id": session_id,
+                "command": command,
+                "status": "starting",
+                "created_at": now,
+                "started_at": now,
+                "finished_at": None,
+                "expires_at": None,
+                "exit_code": None,
+                "last_sequence": 0,
+            }
+            self._write_json(self._metadata_path(session_id), metadata)
+            self._transcript_path(session_id).touch(exist_ok=True)
+            self._set_active_marker(session_id)
+
+            task = asyncio.create_task(
+                self._run_session(session_id=session_id, command=command), name=f"terminal_{session_id}"
+            )
+            self._active = ActiveTerminalSession(session_id=session_id, task=task)
+            return dict(metadata)
+
+    async def get_active_session(self) -> dict[str, Any] | None:
+        async with self._lock:
+            metadata = self._get_active_session_locked()
+            return None if metadata is None else dict(metadata)
+
+    async def get_session(self, session_id: str) -> dict[str, Any] | None:
+        async with self._lock:
+            metadata = self._load_metadata(session_id)
+            if metadata is None or self._is_expired_metadata(metadata, time.time()):
+                return None
+            return dict(metadata)
+
+    async def cancel_session(self, session_id: str) -> dict[str, Any]:
+        async with self._lock:
+            metadata = self._load_metadata(session_id)
+            if metadata is None or self._is_expired_metadata(metadata, time.time()):
+                msg = f"Unknown terminal session '{session_id}'."
+                raise FileNotFoundError(msg)
+
+            runtime = self._active
+            if runtime is None or runtime.session_id != session_id:
+                msg = "Terminal session is not active."
+                raise RuntimeError(msg)
+
+            runtime.interrupted = True
+            self._signal_process(runtime.process)
+            if runtime.process is None and runtime.task.done() is False:
+                runtime.task.cancel()
+
+            return dict(metadata)
+
+    async def stream_session(self, session_id: str, request: Request) -> web.StreamResponse:
+        since = self._parse_since(request)
+
+        response = web.StreamResponse(
+            status=web.HTTPOk.status_code,
+            headers={
+                "Content-Type": "text/event-stream",
+                "Cache-Control": "no-cache",
+                "Connection": "keep-alive",
+            },
+        )
+        await response.prepare(request)
+
+        queue: asyncio.Queue[dict[str, Any] | None] | None = None
+        last_sent = since
+
+        try:
+            replay_events = self._read_transcript(session_id=session_id, since=since)
+            for event in replay_events:
+                if await self._emit_sse(request=request, response=response, event=event) is False:
+                    return response
+                last_sent = event["seq"]
+
+            replay_until = last_sent
+            async with self._lock:
+                metadata = self._load_metadata(session_id)
+                if metadata is not None:
+                    if self._is_expired_metadata(metadata, time.time()):
+                        return response
+
+                    replay_until = int(metadata.get("last_sequence", last_sent))
+
+                runtime = self._active
+                if runtime is not None and runtime.session_id == session_id:
+                    queue = asyncio.Queue()
+                    runtime.subscribers.add(queue)
+
+            if replay_until > last_sent:
+                replay_events = self._read_transcript(session_id=session_id, since=last_sent, until=replay_until)
+                for event in replay_events:
+                    if await self._emit_sse(request=request, response=response, event=event) is False:
+                        return response
+                    last_sent = event["seq"]
+
+            if queue is None:
+                return response
+
+            while True:
+                if self._is_request_disconnected(request):
+                    break
+
+                try:
+                    event = await asyncio.wait_for(queue.get(), timeout=self._keepalive_interval)
+                except TimeoutError:
+                    if await self._emit_keepalive(request=request, response=response) is False:
+                        break
+                    continue
+
+                if event is None:
+                    break
+
+                if event["seq"] <= last_sent:
+                    continue
+
+                if await self._emit_sse(request=request, response=response, event=event) is False:
+                    break
+                last_sent = event["seq"]
+        finally:
+            if queue is not None:
+                async with self._lock:
+                    if self._active is not None and self._active.session_id == session_id:
+                        self._active.subscribers.discard(queue)
+
+            try:
+                await response.write_eof()
+            except ConnectionResetError:
+                pass
+
+        return response
+
+    async def _run_session(self, session_id: str, command: str) -> None:
+        return_code = -1
+        final_status = "completed"
+        proc: Process | None = None
+        read_task: asyncio.Task[None] | None = None
+        master_fd: int | None = None
+
+        try:
+            LOG.info("Starting terminal session '%s' command.", session_id, extra={"session_id": session_id})
+            args = ["yt-dlp", *shlex.split(command, posix=os.name != "nt")]
+            env_vars = self._build_env()
+
+            pty_handles = self._open_pty()
+            if pty_handles is None:
+                stdin_arg = asyncio.subprocess.DEVNULL
+                stdout_arg = asyncio.subprocess.PIPE
+                stderr_arg = asyncio.subprocess.STDOUT
+                use_pty = False
+                slave_fd = None
+            else:
+                master_fd, slave_fd = pty_handles
+                stdin_arg = asyncio.subprocess.DEVNULL
+                stdout_arg = slave_fd
+                stderr_arg = slave_fd
+                use_pty = True
+
+            creationflags = 0
+            if os.name == "nt":
+                import subprocess
+
+                creationflags = subprocess.CREATE_NO_WINDOW
+
+            proc = await asyncio.create_subprocess_exec(
+                *args,
+                cwd=self.config.download_path,
+                stdin=stdin_arg,
+                stdout=stdout_arg,
+                stderr=stderr_arg,
+                env=env_vars,
+                creationflags=creationflags,
+            )
+
+            async with self._lock:
+                metadata = self._load_metadata(session_id)
+                if metadata is not None:
+                    metadata["status"] = "running"
+                    self._write_json(self._metadata_path(session_id), metadata)
+
+                if self._active is not None and self._active.session_id == session_id:
+                    self._active.process = proc
+
+            if use_pty is True and slave_fd is not None:
+                try:
+                    os.close(slave_fd)
+                except Exception as exc:
+                    LOG.exception(
+                        "Failed to close the PTY slave file descriptor.",
+                        extra={"exception_type": type(exc).__name__},
+                    )
+
+            read_task = asyncio.create_task(
+                self._read_process_output(session_id=session_id, proc=proc, use_pty=use_pty, master_fd=master_fd),
+                name=f"terminal_reader_{session_id}",
+            )
+
+            return_code = await proc.wait()
+            await read_task
+        except asyncio.CancelledError:
+            final_status = "interrupted"
+        except Exception as exc:
+            final_status = "failed"
+            LOG.exception(
+                "Terminal session '%s' command failed.",
+                session_id,
+                extra={"session_id": session_id, "use_pty": use_pty, "exception_type": type(exc).__name__},
+            )
+            await self._append_event(session_id, "output", {"type": "stderr", "line": str(exc)})
+        finally:
+            final_status = await self._resolve_final_status(session_id=session_id, status=final_status)
+
+            if final_status == "interrupted" and proc is not None and getattr(proc, "returncode", None) is None:
+                self._signal_process(proc)
+                try:
+                    return_code = await asyncio.wait_for(proc.wait(), timeout=self._shutdown_timeout)
+                except TimeoutError:
+                    self._signal_process(proc, force=True)
+                    try:
+                        return_code = await asyncio.wait_for(proc.wait(), timeout=self._shutdown_timeout)
+                    except TimeoutError:
+                        LOG.warning("Terminal session '%s' did not exit after repeated shutdown attempts.", session_id)
+
+            if proc is not None:
+                proc_returncode = getattr(proc, "returncode", None)
+                if proc_returncode is not None:
+                    return_code = int(proc_returncode)
+
+            await self._append_event(session_id, "close", {"exitcode": return_code})
+            await self._finalize_session(session_id=session_id, status=final_status, exit_code=return_code)
+
+            if read_task is not None and not read_task.done():
+                read_task.cancel()
+
+            if master_fd is not None:
+                try:
+                    os.close(master_fd)
+                except OSError:
+                    pass
+
+    async def _read_process_output(self, session_id: str, proc: Process, use_pty: bool, master_fd: int | None) -> None:
+        if use_pty is False:
+            assert proc.stdout is not None
+            async for raw_line in proc.stdout:
+                line = raw_line.rstrip(b"\n")
+                await self._append_event(
+                    session_id,
+                    "output",
+                    {"type": "stdout", "line": line.decode("utf-8", errors="replace")},
+                )
+            return
+
+        assert master_fd is not None
+        loop: AbstractEventLoop = asyncio.get_running_loop()
+        buffer = b""
+
+        while True:
+            try:
+                chunk = await loop.run_in_executor(None, lambda: os.read(master_fd, 1024))
+            except OSError as exc:
+                if exc.errno == errno.EIO:
+                    break
+                raise
+
+            if not chunk:
+                if buffer:
+                    await self._append_event(
+                        session_id,
+                        "output",
+                        {"type": "stdout", "line": buffer.decode("utf-8", errors="replace")},
+                    )
+                break
+
+            buffer += chunk
+            *lines, buffer = buffer.split(b"\n")
+            for line in lines:
+                await self._append_event(
+                    session_id,
+                    "output",
+                    {"type": "stdout", "line": line.decode("utf-8", errors="replace")},
+                )
+
+    async def _append_event(self, session_id: str, event: str, data: dict[str, Any]) -> dict[str, Any]:
+        async with self._lock:
+            metadata = self._load_metadata(session_id)
+            if metadata is None:
+                msg = f"Unknown terminal session '{session_id}'."
+                raise FileNotFoundError(msg)
+
+            next_sequence = int(metadata.get("last_sequence", 0)) + 1
+            metadata["last_sequence"] = next_sequence
+            self._write_json(self._metadata_path(session_id), metadata)
+
+            record = {"seq": next_sequence, "event": event, "data": data}
+            with self._transcript_path(session_id).open("a", encoding="utf-8") as handle:
+                handle.write(json.dumps(record) + "\n")
+
+            if self._active is not None and self._active.session_id == session_id:
+                for subscriber in list(self._active.subscribers):
+                    subscriber.put_nowait(record)
+
+            return record
+
+    async def _finalize_session(self, session_id: str, status: str, exit_code: int) -> None:
+        async with self._lock:
+            metadata = self._load_metadata(session_id)
+            if metadata is None:
+                return
+
+            now = time.time()
+            metadata["status"] = status
+            metadata["exit_code"] = exit_code
+            metadata["finished_at"] = now
+            metadata["expires_at"] = now + self._completed_retention
+            self._write_json(self._metadata_path(session_id), metadata)
+
+            if self._load_active_marker() == session_id:
+                self._clear_active_marker()
+
+            runtime = self._active
+            if runtime is not None and runtime.session_id == session_id:
+                subscribers = list(runtime.subscribers)
+                self._active = None
+                for subscriber in subscribers:
+                    subscriber.put_nowait(None)
+
+    def _get_active_session_locked(self) -> dict[str, Any] | None:
+        session_id = self._load_active_marker()
+        if session_id is None:
+            return None
+
+        metadata = self._load_metadata(session_id)
+        if metadata is None:
+            self._clear_active_marker()
+            return None
+
+        return metadata
+
+    def _list_sessions_locked(self) -> list[dict[str, Any]]:
+        self._ensure_root()
+        items: list[dict[str, Any]] = []
+
+        for path in self.root_path.iterdir():
+            if path.name == ACTIVE_FILE_NAME or path.is_dir() is False:
+                continue
+
+            metadata = self._load_metadata(path.name)
+            if metadata is None:
+                continue
+
+            if self._is_expired_metadata(metadata, time.time()):
+                continue
+
+            item = dict(metadata)
+            item["available_until"] = self._available_until(metadata)
+            items.append(item)
+
+        items.sort(
+            key=lambda item: float(item.get("finished_at") or item.get("started_at") or item.get("created_at") or 0),
+            reverse=True,
+        )
+        return items
+
+    def _recover_orphaned_active_session(self, now: float) -> None:
+        session_id = self._load_active_marker()
+        if session_id is None:
+            return
+
+        metadata = self._load_metadata(session_id)
+        if metadata is None:
+            self._clear_active_marker()
+            return
+
+        metadata["status"] = "interrupted"
+        metadata["finished_at"] = now
+        metadata["expires_at"] = now + self._completed_retention
+        metadata["exit_code"] = -1 if metadata.get("exit_code") is None else metadata["exit_code"]
+        self._write_json(self._metadata_path(session_id), metadata)
+        self._clear_active_marker()
+
+    def _cleanup_expired_sessions(self, now: float) -> int:
+        self._ensure_root()
+        active_session_id = self._load_active_marker()
+        removed = 0
+
+        for path in self.root_path.iterdir():
+            if path.name == ACTIVE_FILE_NAME or path.is_dir() is False:
+                continue
+
+            metadata = self._load_metadata(path.name)
+            if metadata is None:
+                shutil.rmtree(path, ignore_errors=True)
+                removed += 1
+                continue
+
+            if not self._is_expired_metadata(metadata, now):
+                continue
+
+            if active_session_id == path.name:
+                self._clear_active_marker()
+                active_session_id = None
+
+            shutil.rmtree(path, ignore_errors=True)
+
+            removed += 1
+
+        return removed
+
+    def _parse_since(self, request: Request) -> int:
+        values: list[int] = []
+        candidates = [request.query.get("since"), request.headers.get("Last-Event-ID")]
+        for candidate in candidates:
+            if candidate in (None, ""):
+                continue
+            try:
+                value = int(candidate)
+            except ValueError as exc:
+                msg = "Resume cursor must be an integer."
+                raise ValueError(msg) from exc
+
+            if value < 0:
+                msg = "Resume cursor must be zero or greater."
+                raise ValueError(msg)
+
+            values.append(value)
+
+        return max(values, default=0)
+
+    async def _force_interrupt_session(self, session_id: str) -> None:
+        async with self._lock:
+            metadata = self._load_metadata(session_id)
+            if metadata is None:
+                return
+
+            now = time.time()
+            metadata["status"] = "interrupted"
+            metadata["finished_at"] = now
+            metadata["expires_at"] = now + self._completed_retention
+            metadata["exit_code"] = -1 if metadata.get("exit_code") is None else metadata["exit_code"]
+            self._write_json(self._metadata_path(session_id), metadata)
+
+            if self._load_active_marker() == session_id:
+                self._clear_active_marker()
+
+            runtime = self._active
+            if runtime is not None and runtime.session_id == session_id:
+                subscribers = list(runtime.subscribers)
+                self._active = None
+                for subscriber in subscribers:
+                    subscriber.put_nowait(None)
+
+    async def _resolve_final_status(self, session_id: str, status: str) -> str:
+        async with self._lock:
+            runtime = self._active
+            if runtime is not None and runtime.session_id == session_id and runtime.interrupted:
+                return "interrupted"
+
+            metadata = self._load_metadata(session_id)
+            if metadata is not None and metadata.get("status") == "interrupted":
+                return "interrupted"
+
+        return status
+
+    def _signal_process(self, process: Process | None, *, force: bool = False) -> None:
+        if process is None or getattr(process, "returncode", None) is not None:
+            return
+
+        action = process.kill if force else process.terminate
+        try:
+            action()
+        except ProcessLookupError:
+            return
+
+    async def _emit_sse(self, request: Request, response: web.StreamResponse, event: dict[str, Any]) -> bool:
+        if self._is_request_disconnected(request):
+            return False
+
+        payload = f"id: {event['seq']}\nevent: {event['event']}\ndata: {json.dumps(event['data'])}\n\n"
+        try:
+            await response.write(payload.encode("utf-8"))
+        except ConnectionResetError:
+            return False
+
+        return True
+
+    async def _emit_keepalive(self, request: Request, response: web.StreamResponse) -> bool:
+        if self._is_request_disconnected(request):
+            return False
+
+        try:
+            await response.write(b": keepalive\n\n")
+        except ConnectionResetError:
+            return False
+
+        return True
+
+    def _read_transcript(self, session_id: str, since: int, until: int | None = None) -> list[dict[str, Any]]:
+        transcript_path = self._transcript_path(session_id)
+        if transcript_path.exists() is False:
+            return []
+
+        events: list[dict[str, Any]] = []
+        with transcript_path.open("r", encoding="utf-8") as handle:
+            for raw_line in handle:
+                line = raw_line.strip()
+                if not line:
+                    continue
+
+                event = json.loads(line)
+                sequence = int(event["seq"])
+                if sequence <= since:
+                    continue
+                if until is not None and sequence > until:
+                    break
+                events.append(event)
+
+        return events
+
+    def _available_until(self, metadata: dict[str, Any]) -> float | None:
+        expires_at = metadata.get("expires_at")
+        if expires_at is None:
+            return None
+
+        return float(expires_at)
+
+    def _is_expired_metadata(self, metadata: dict[str, Any], now: float) -> bool:
+        expires_at = self._available_until(metadata)
+        if expires_at is None:
+            return False
+
+        return expires_at <= now
+
+    def _build_env(self) -> dict[str, str]:
+        env_vars = os.environ.copy()
+        env_vars.update(
+            {
+                "PWD": self.config.download_path,
+                "FORCE_COLOR": "1",
+                "PYTHONUNBUFFERED": "1",
+            }
+        )
+
+        if os.name != "nt":
+            env_vars.update(
+                {
+                    "TERM": "xterm-256color",
+                    "LANG": "en_US.UTF-8",
+                    "LC_ALL": "en_US.UTF-8",
+                    "SHELL": "/bin/bash",
+                }
+            )
+
+        return env_vars
+
+    def _open_pty(self) -> tuple[int, int] | None:
+        try:
+            import pty
+
+            return pty.openpty()
+        except ImportError:
+            return None
+
+    def _is_request_disconnected(self, request: Request) -> bool:
+        return request.transport is None or request.transport.is_closing()
+
+    def _ensure_root(self) -> None:
+        self.root_path.mkdir(parents=True, exist_ok=True)
+
+    def _session_dir(self, session_id: str) -> Path:
+        return self.root_path / session_id
+
+    def _metadata_path(self, session_id: str) -> Path:
+        return self._session_dir(session_id) / METADATA_FILE_NAME
+
+    def _transcript_path(self, session_id: str) -> Path:
+        return self._session_dir(session_id) / TRANSCRIPT_FILE_NAME
+
+    def _active_marker_path(self) -> Path:
+        return self.root_path / ACTIVE_FILE_NAME
+
+    def _set_active_marker(self, session_id: str) -> None:
+        self._write_json(self._active_marker_path(), {"session_id": session_id})
+
+    def _clear_active_marker(self) -> None:
+        self._active_marker_path().unlink(missing_ok=True)
+
+    def _load_active_marker(self) -> str | None:
+        data = self._read_json(self._active_marker_path())
+        if not isinstance(data, dict):
+            return None
+        session_id = data.get("session_id")
+        return session_id if isinstance(session_id, str) and session_id else None
+
+    def _load_metadata(self, session_id: str) -> dict[str, Any] | None:
+        data = self._read_json(self._metadata_path(session_id))
+        return data if isinstance(data, dict) else None
+
+    def _read_json(self, path: Path) -> dict[str, Any] | None:
+        if path.exists() is False:
+            return None
+
+        with path.open("r", encoding="utf-8") as handle:
+            return json.load(handle)
+
+    def _write_json(self, path: Path, payload: dict[str, Any]) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temp_path = path.with_suffix(f"{path.suffix}.tmp")
+        with temp_path.open("w", encoding="utf-8") as handle:
+            json.dump(payload, handle)
+        temp_path.replace(path)

app/library/UpdateChecker.py

@@ -1,10 +1,11 @@
 import asyncio
-import logging
 import re
 from typing import Any
 
 from aiohttp import web
 
+from app.library.log import get_logger
+
 from .cache import Cache
 from .config import Config
 from .Events import EventBus, Events
@@ -13,7 +14,7 @@ from .Scheduler import Scheduler
 from .Singleton import Singleton
 from .version import APP_VERSION
 
-LOG: logging.Logger = logging.getLogger("update_checker")
+LOG = get_logger()
 
 
 class UpdateChecker(metaclass=Singleton):
@@ -79,7 +80,7 @@ class UpdateChecker(metaclass=Singleton):
         self._notify.subscribe(
             event=Events.STARTED,
             callback=event_handler,
-            name=f"{__class__.__name__}.{__class__.attach.__name__}",
+            name=f"{UpdateChecker.__name__}.{type(self).attach.__name__}",
         )
 
         self._schedule_check()
@@ -107,7 +108,7 @@ class UpdateChecker(metaclass=Singleton):
         self._job_id = self._scheduler.add(
             timer=timer,
             func=lambda: asyncio.create_task(self.check_for_updates()),
-            id=f"{__class__.__name__}.{self.check_for_updates.__name__}",
+            id=f"{UpdateChecker.__name__}.{self.check_for_updates.__name__}",
         )
 
     async def check_for_updates(self) -> tuple[tuple[str, str | None], tuple[str, str | None]]:
@@ -145,7 +146,11 @@ class UpdateChecker(metaclass=Singleton):
         strip_v_prefix: bool = False,
     ) -> tuple[str, str | None]:
         try:
-            LOG.info(f"Checking for {name} updates...")
+            LOG.info(
+                "Checking whether %s has an update available.",
+                name,
+                extra={"target_name": name, "api_url": api_url, "current_version": current_version},
+            )
 
             client = get_async_client(use_curl=False)
             response = await client.get(
@@ -155,32 +160,59 @@ class UpdateChecker(metaclass=Singleton):
             )
 
             if 200 != response.status_code:
-                LOG.warning(f"Failed to check for {name} updates: HTTP {response.status_code}")
+                LOG.warning(
+                    "Failed to check for %s updates: HTTP %s",
+                    name,
+                    response.status_code,
+                    extra={"target_name": name, "api_url": api_url, "status_code": response.status_code},
+                )
                 return ("error", None)
 
             data: dict[str, Any] = response.json()
 
             latest_tag: str = data.get("tag_name", "")
             if not latest_tag:
-                LOG.warning(f"No tag_name found in {name} GitHub release data.")
+                LOG.warning(
+                    "No tag_name found in %s GitHub release data.",
+                    name,
+                    extra={"target_name": name, "api_url": api_url},
+                )
                 return ("error", None)
 
             compare_current: str = current_version.lstrip("v") if strip_v_prefix else current_version
             compare_latest: str = latest_tag.lstrip("v") if strip_v_prefix else latest_tag
 
             if self._compare_versions(compare_current, compare_latest):
-                LOG.warning(f"{name} update available: {current_version} -> {latest_tag}")
+                LOG.warning(
+                    "%s has an update available: %s -> %s.",
+                    name,
+                    current_version,
+                    latest_tag,
+                    extra={"target_name": name, "current_version": current_version, "latest_version": latest_tag},
+                )
                 result = ("update_available", latest_tag)
                 await self._cache.aset(cache_key, result, self.CACHE_DURATION)
                 return result
 
-            LOG.info(f"No {name} updates available.")
+            LOG.info(
+                "%s is already up to date.",
+                name,
+                extra={"target_name": name, "current_version": current_version},
+            )
             result = ("up_to_date", None)
             await self._cache.aset(cache_key, result, self.CACHE_DURATION)
             return result
         except Exception as e:
-            LOG.exception(e)
-            LOG.error(f"Error checking for {name} updates: {e!s}")
+            LOG.exception(
+                "Failed to check whether %s has an update available.",
+                name,
+                extra={
+                    "target_name": name,
+                    "api_url": api_url,
+                    "cache_key": cache_key,
+                    "exception_type": type(e).__name__,
+                },
+            )
             return ("error", None)
 
     async def _check_app_version(self) -> tuple[str, str | None]:
@@ -203,7 +235,7 @@ class UpdateChecker(metaclass=Singleton):
     async def _check_ytdlp_version(self) -> tuple[str, str | None]:
         current_version: str = self._config._ytdlp_version()
         if not current_version or "0.0.0" == current_version:
-            LOG.warning("Could not determine yt-dlp version, skipping yt-dlp update check.")
+            LOG.warning("Skipping the yt-dlp update check because the current version could not be determined.")
             return ("error", None)
 
         status, new_version = await self._check_github_version(
@@ -238,5 +270,12 @@ class UpdateChecker(metaclass=Singleton):
 
             return parse_version(latest) > parse_version(current)
         except Exception as e:
-            LOG.warning(f"Error comparing versions '{current}' vs '{latest}': {e}")
+            LOG.warning(
+                "Failed to compare versions '%s' and '%s' because %s.",
+                current,
+                latest,
+                e,
+                extra={"current_version": current, "latest_version": latest, "exception_type": type(e).__name__},
+                exc_info=True,
+            )
             return False

app/library/Utils.py

@@ -2,11 +2,13 @@ import base64
 import copy
 import glob
 import ipaddress
+import json
 import logging
 import os
 import re
 import socket
 import time
+import traceback
 import uuid
 from datetime import UTC, datetime, timedelta
 from functools import lru_cache, wraps
@@ -16,7 +18,9 @@ from typing import Any
 
 from Crypto.Cipher import AES
 
-LOG: logging.Logger = logging.getLogger("Utils")
+from app.library.log import get_logger
+
+LOG = get_logger()
 
 ALLOWED_SUBS_EXTENSIONS: set[str] = {".srt", ".vtt", ".ass"}
 "Allowed subtitle file extensions."
@@ -36,11 +40,179 @@ TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c")
 
 
 class FileLogFormatter(logging.Formatter):
-    def formatTime(self, record, datefmt=None):  # noqa: ARG002, N802
+    def formatTime(self, record, datefmt=None):  # noqa: N802
+        _ = datefmt
         return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
 
 
-def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
+LOG_RECORD_ATTRS: set[str] = set(logging.makeLogRecord({}).__dict__) | {"asctime", "message"}
+SKIP_LOG_FIELD = object()
+
+
+class JsonLogFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        source = {
+            "path": record.pathname,
+            "file": record.filename,
+            "module": record.module,
+            "function": record.funcName,
+            "line": record.lineno,
+        }
+        data: dict[str, Any] = {
+            "id": str(uuid.uuid4()),
+            "datetime": self.formatTime(record),
+            "level": record.levelname.lower(),
+            "levelno": record.levelno,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "source": source,
+            "process": {"id": record.process, "name": record.processName},
+            "thread": {"id": record.thread, "name": record.threadName},
+            "fields": self._extra(record),
+        }
+
+        if record.exc_info:
+            exception = self._exception(record.exc_info)
+            if exception is not None:
+                data["exception"] = exception
+                data["source"].update(self._exception_source(exception))
+
+        return json.dumps(data, ensure_ascii=False, default=str)
+
+    def formatTime(self, record, datefmt=None):  # noqa: N802
+        _ = datefmt
+        return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
+
+    @staticmethod
+    def _extra(record: logging.LogRecord) -> dict[str, Any]:
+        extra: dict[str, Any] = {}
+        for key, value in record.__dict__.items():
+            if key in LOG_RECORD_ATTRS or key.startswith("_"):
+                continue
+
+            value = JsonLogFormatter._field(value)
+            if value is not SKIP_LOG_FIELD:
+                extra[key] = value
+
+        return extra
+
+    @staticmethod
+    def _field(value: Any) -> Any:
+        if isinstance(value, str | int | float | bool) or value is None:
+            return value
+
+        if isinstance(value, Path):
+            return str(value)
+
+        if isinstance(value, dict):
+            data: dict[str, Any] = {}
+            for key, item in value.items():
+                if not isinstance(key, str) or key.startswith("_"):
+                    continue
+
+                item = JsonLogFormatter._field(item)
+                if item is not SKIP_LOG_FIELD:
+                    data[key] = item
+
+            return data
+
+        if isinstance(value, list):
+            data: list[Any] = []
+            for item in value:
+                item = JsonLogFormatter._field(item)
+                if item is not SKIP_LOG_FIELD:
+                    data.append(item)
+
+            return data
+
+        return SKIP_LOG_FIELD
+
+    @staticmethod
+    def _exception(
+        exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None],
+    ) -> dict[str, Any] | None:
+        exc = exc_info[1]
+        if exc is None:
+            return None
+
+        stack = JsonLogFormatter._exception_stack(exc_info)
+        data: dict[str, Any] = {"type": JsonLogFormatter._exception_type(exc)}
+
+        message = str(exc).strip()
+        if message:
+            data["message"] = message
+
+        if stack:
+            origin = stack[-1]
+            data["file"] = origin["path"]
+            data["line"] = origin["line"]
+            data["stack"] = stack
+
+        return data
+
+    @staticmethod
+    def _exception_type(exc: BaseException) -> str:
+        module = exc.__class__.__module__
+        name = exc.__class__.__qualname__
+        return name if module == "builtins" else f"{module}.{name}"
+
+    @staticmethod
+    def _exception_stack(
+        exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None],
+    ) -> list[dict[str, Any]]:
+        exc = exc_info[1]
+        tb = exc_info[2] if len(exc_info) > 2 else None
+        if tb is None and exc is not None:
+            tb = exc.__traceback__
+
+        if tb is None:
+            return []
+
+        return [JsonLogFormatter._frame(frame) for frame in traceback.extract_tb(tb)]
+
+    @staticmethod
+    def _frame(frame: traceback.FrameSummary) -> dict[str, Any]:
+        path = frame.filename
+        file_path = Path(path)
+        return {
+            "path": path,
+            "file": file_path.name,
+            "module": file_path.stem,
+            "function": frame.name,
+            "line": frame.lineno,
+        }
+
+    @staticmethod
+    def _exception_source(exception: dict[str, Any]) -> dict[str, Any]:
+        source: dict[str, Any] = {}
+
+        path = exception.get("file")
+        if isinstance(path, str) and path:
+            file_path = Path(path)
+            source["path"] = path
+            source["file"] = file_path.name
+            source["module"] = file_path.stem
+
+        line = exception.get("line")
+        if isinstance(line, int):
+            source["line"] = line
+
+        stack = exception.get("stack")
+        if isinstance(stack, list) and stack:
+            frame = stack[-1]
+            if isinstance(frame, dict):
+                function = frame.get("function")
+                if isinstance(function, str) and function:
+                    source["function"] = function
+
+                module = frame.get("module")
+                if isinstance(module, str) and module:
+                    source["module"] = module
+
+        return source
+
+
+def timed_lru_cache(ttl_seconds: float, max_size: int = 128):
     """
     Decorator that applies an LRU cache with a time-to-live (TTL) to a function.
     Supports both synchronous and asynchronous functions.
@@ -110,8 +282,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
 
                 return _CacheInfo(hits=0, misses=len(cache), maxsize=max_size, currsize=len(cache))
 
-            async_wrapper.cache_clear = cache_clear
-            async_wrapper.cache_info = cache_info
+            async_wrapper_cache: Any = async_wrapper
+            async_wrapper_cache.cache_clear = cache_clear
+            async_wrapper_cache.cache_info = cache_info
             return async_wrapper
 
         # For sync functions, use the original implementation
@@ -139,8 +312,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
                 return result
 
         # expose cache_clear, cache_info
-        sync_wrapper.cache_clear = cached.cache_clear
-        sync_wrapper.cache_info = cached.cache_info
+        sync_wrapper_cache: Any = sync_wrapper
+        sync_wrapper_cache.cache_clear = cached.cache_clear
+        sync_wrapper_cache.cache_info = cached.cache_info
         return sync_wrapper
 
     return decorator
@@ -211,7 +385,12 @@ def _is_safe_key(key: Any) -> bool:
 
 
 def merge_dict(
-    source: dict, destination: dict, max_depth: int = 50, max_list_size: int = 10000, _depth: int = 0, _seen: set = None
+    source: dict,
+    destination: dict,
+    max_depth: int = 50,
+    max_list_size: int = 10000,
+    _depth: int = 0,
+    _seen: set | None = None,
 ) -> dict:
     """
     Merge data from source into destination.
@@ -302,7 +481,7 @@ def merge_dict(
     return destination_copy
 
 
-def check_id(file: Path) -> bool | str:
+def check_id(file: Path) -> bool | Path:
     """
     Check if we are able to get an id from the file name.
     if so check if any video file with the same id exists.
@@ -321,6 +500,8 @@ def check_id(file: Path) -> bool | str:
         return False
 
     id: str | None = match.groupdict().get("id")
+    if id is None:
+        return False
 
     try:
         if not file.parent.exists():
@@ -335,7 +516,11 @@ def check_id(file: Path) -> bool | str:
 
             return f.absolute()
     except OSError as e:
-        LOG.error(f"Error checking file '{file}': {e!s}")
+        LOG.exception(
+            "Failed to check for a matching file for '%s'.",
+            file,
+            extra={"file_path": str(file), "operation": "check_id", "exception_type": type(e).__name__},
+        )
         return False
 
     return False
@@ -371,9 +556,9 @@ def validate_url(url: str, allow_internal: bool = False) -> bool:
         from yarl import URL
 
         parsed_url = URL(url)
-    except ValueError:
+    except ValueError as exc:
         msg = "Invalid URL."
-        raise ValueError(msg)  # noqa: B904
+        raise ValueError(msg) from exc
 
     # Check allowed schemes
     if parsed_url.scheme not in ["http", "https"]:
@@ -391,7 +576,11 @@ def validate_url(url: str, allow_internal: bool = False) -> bool:
                 msg = "Access to internal urls or private networks is not allowed."
                 raise ValueError(msg)
         except socket.gaierror as e:
-            LOG.error(f"Error resolving hostname '{hostname}': {e!s}")
+            LOG.exception(
+                "Failed to resolve hostname '%s'.",
+                hostname,
+                extra={"hostname": hostname, "exception_type": type(e).__name__},
+            )
             msg = "Invalid hostname."
             raise ValueError(msg) from e
 
@@ -418,7 +607,7 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool:
 
 
 @timed_lru_cache(ttl_seconds=60, max_size=256)
-def get_file_sidecar(file: Path | None = None) -> dict[dict]:
+def get_file_sidecar(file: Path | None = None) -> dict[str, list[dict[str, Any]]]:
     """
     Get sidecar files for the given file.
 
@@ -519,16 +708,46 @@ def rename_file(old_path: Path, new_name: str) -> tuple[Path, list[tuple[Path, P
                 renamed_sidecar: Path = old_sidecar.rename(new_sidecar)
                 renamed_sidecars.append((old_sidecar, renamed_sidecar))
             except OSError as e:
-                LOG.error(f"Failed to rename sidecar '{old_sidecar}': {e}")
+                LOG.exception(
+                    "Failed to rename sidecar '%s' to '%s'.",
+                    old_sidecar,
+                    new_sidecar,
+                    extra={
+                        "old_path": str(old_sidecar),
+                        "new_path": str(new_sidecar),
+                        "operation": "rename_sidecar",
+                        "exception_type": type(e).__name__,
+                    },
+                )
                 try:
                     renamed_main.rename(old_path)
-                except OSError:
-                    LOG.error(f"Failed to rollback main file rename from '{renamed_main}' to '{old_path}'")
+                except OSError as rollback_error:
+                    LOG.exception(
+                        "Failed to roll back main file rename from '%s' to '%s'.",
+                        renamed_main,
+                        old_path,
+                        extra={
+                            "old_path": str(renamed_main),
+                            "new_path": str(old_path),
+                            "operation": "rename_rollback",
+                            "exception_type": type(rollback_error).__name__,
+                        },
+                    )
                 raise
 
         return renamed_main, renamed_sidecars
     except OSError as e:
-        LOG.error(f"Failed to rename '{old_path}' to '{new_path}': {e}")
+        LOG.exception(
+            "Failed to rename '%s' to '%s'.",
+            old_path,
+            new_path,
+            extra={
+                "old_path": str(old_path),
+                "new_path": str(new_path),
+                "operation": "rename",
+                "exception_type": type(e).__name__,
+            },
+        )
         raise
 
 
@@ -586,24 +805,62 @@ def move_file(old_path: Path, target_dir: Path) -> tuple[Path, list[tuple[Path,
                 moved_sidecar: Path = old_sidecar.rename(new_sidecar)
                 renamed_sidecars.append((old_sidecar, moved_sidecar))
             except OSError as e:
-                LOG.error(f"Failed to move sidecar '{old_sidecar}': {e}")
+                LOG.exception(
+                    "Failed to move sidecar '%s' to '%s'.",
+                    old_sidecar,
+                    new_sidecar,
+                    extra={
+                        "old_path": str(old_sidecar),
+                        "new_path": str(new_sidecar),
+                        "operation": "move_sidecar",
+                        "exception_type": type(e).__name__,
+                    },
+                )
                 try:
                     moved_main.rename(old_path)
                     # Rollback any sidecars that were already moved
                     for rolled_back_old, rolled_back_new in renamed_sidecars:
                         try:
                             rolled_back_new.rename(rolled_back_old)
-                        except OSError:
-                            LOG.error(
-                                f"Failed to rollback sidecar move from '{rolled_back_new}' to '{rolled_back_old}'"
+                        except OSError as rollback_error:
+                            LOG.exception(
+                                "Failed to roll back sidecar move from '%s' to '%s'.",
+                                rolled_back_new,
+                                rolled_back_old,
+                                extra={
+                                    "old_path": str(rolled_back_new),
+                                    "new_path": str(rolled_back_old),
+                                    "operation": "move_sidecar_rollback",
+                                    "exception_type": type(rollback_error).__name__,
+                                },
                             )
-                except OSError:
-                    LOG.error(f"Failed to rollback main file move from '{moved_main}' to '{old_path}'")
+                except OSError as rollback_error:
+                    LOG.exception(
+                        "Failed to roll back main file move from '%s' to '%s'.",
+                        moved_main,
+                        old_path,
+                        extra={
+                            "old_path": str(moved_main),
+                            "new_path": str(old_path),
+                            "operation": "move_rollback",
+                            "exception_type": type(rollback_error).__name__,
+                        },
+                    )
                 raise
 
         return moved_main, renamed_sidecars
     except OSError as e:
-        LOG.error(f"Failed to move '{old_path}' to '{new_path}': {e}")
+        LOG.exception(
+            "Failed to move '%s' to '%s'.",
+            old_path,
+            new_path,
+            extra={
+                "old_path": str(old_path),
+                "new_path": str(new_path),
+                "operation": "move",
+                "exception_type": type(e).__name__,
+            },
+        )
         raise
 
 
@@ -679,18 +936,22 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
         download_path = Path(download_path)
 
     try:
-        realFile: str = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
+        realFile: Path = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
         if realFile.exists():
             return (realFile, 200)
     except Exception as e:
-        LOG.error(f"Error calculating download path. {e!s}")
+        LOG.exception(
+            "Failed to resolve download file path for '%s'.",
+            file,
+            extra={"file_path": str(file), "download_path": str(download_path), "exception_type": type(e).__name__},
+        )
         return (Path(file), 404)
 
-    possibleFile: bool | str = check_id(file=realFile)
-    if not possibleFile:
+    possibleFile: bool | Path = check_id(file=realFile)
+    if not isinstance(possibleFile, Path):
         return (realFile, 404)
 
-    return (Path(possibleFile), 302)
+    return (possibleFile, 302)
 
 
 def encrypt_data(data: str, key: bytes) -> str:
@@ -712,7 +973,7 @@ def encrypt_data(data: str, key: bytes) -> str:
     return base64.urlsafe_b64encode(iv + ciphertext + tag).decode()
 
 
-def decrypt_data(data: str, key: bytes) -> str:
+def decrypt_data(data: str, key: bytes) -> str | None:
     """
     Decrypts AES-GCM encrypted data
 
@@ -725,8 +986,8 @@ def decrypt_data(data: str, key: bytes) -> str:
 
     """
     try:
-        data = base64.urlsafe_b64decode(data)
-        iv, ciphertext, tag = data[:12], data[12:-16], data[-16:]
+        raw: bytes = base64.urlsafe_b64decode(data)
+        iv, ciphertext, tag = raw[:12], raw[12:-16], raw[-16:]
         cipher = AES.new(key, AES.MODE_GCM, nonce=iv)
         plaintext = cipher.decrypt_and_verify(ciphertext, tag)
         return plaintext.decode()
@@ -737,7 +998,7 @@ def decrypt_data(data: str, key: bytes) -> str:
 def get(
     data: dict | list,
     path: str | list | None = None,
-    default: any = None,
+    default: Any = None,
     separator=".",
 ):
     """
@@ -854,22 +1115,42 @@ def get_files(
     try:
         dir_path = dir_path.resolve()
     except OSError as e:
-        LOG.warning(f"Failed to resolve '{dir}' - {e}")
-        return []
+        LOG.warning(
+            "Failed to resolve '%s': %s",
+            dir,
+            e,
+            extra={"requested_dir": dir, "base_path": str(base_path), "exception_type": type(e).__name__},
+            exc_info=True,
+        )
+        return [], 0
 
     try:
         dir_path.relative_to(base_path)
     except ValueError:
-        LOG.warning(f"Invalid path: '{dir_path}' - must be inside '{base_path}'.")
-        return []
+        LOG.warning(
+            "Invalid path '%s': must be inside '%s'.",
+            dir_path,
+            base_path,
+            extra={"path": str(dir_path), "base_path": str(base_path)},
+        )
+        return [], 0
 
     if not str(dir_path).startswith(str(base_path)):
-        LOG.warning(f"Invalid path: '{dir_path}' - must be inside '{base_path}'.")
-        return []
+        LOG.warning(
+            "Invalid path '%s': must be inside '%s'.",
+            dir_path,
+            base_path,
+            extra={"path": str(dir_path), "base_path": str(base_path)},
+        )
+        return [], 0
 
     if not dir_path.is_dir():
-        LOG.warning(f"Invalid path: '{dir_path}' - must be a directory.")
-        return []
+        LOG.warning(
+            "Invalid path '%s': must be a directory.",
+            dir_path,
+            extra={"path": str(dir_path), "base_path": str(base_path)},
+        )
+        return [], 0
 
     contents: list = []
     for file in dir_path.iterdir():
@@ -882,13 +1163,28 @@ def get_files(
                 test: Path = file.resolve()
                 test.relative_to(base_path)
                 if not str(test).startswith(str(base_path)):
-                    LOG.warning(f"Invalid symlink: '{file}' - must resolve inside '{base_path}'.")
+                    LOG.warning(
+                        "Invalid symlink '%s': must resolve inside '%s'.",
+                        file,
+                        base_path,
+                        extra={"path": str(file), "resolved_path": str(test), "base_path": str(base_path)},
+                    )
                     continue
             except ValueError:
-                LOG.warning(f"Invalid symlink: '{file}' - must resolve inside '{base_path}'.")
+                LOG.warning(
+                    "Invalid symlink '%s': must resolve inside '%s'.",
+                    file,
+                    base_path,
+                    extra={"path": str(file), "base_path": str(base_path)},
+                )
                 continue
-            except OSError:
-                LOG.warning(f"Skipping broken symlink: {file}")
+            except OSError as e:
+                LOG.warning(
+                    "Skipping broken symlink '%s'.",
+                    file,
+                    extra={"path": str(file), "base_path": str(base_path), "exception_type": type(e).__name__},
+                    exc_info=True,
+                )
                 continue
 
         content_type = None
@@ -1013,7 +1309,7 @@ def load_cookies(file: str | Path) -> tuple[bool, MozillaCookieJar]:
     try:
         from http.cookiejar import MozillaCookieJar
 
-        cookies = MozillaCookieJar(str(file), None, None)
+        cookies = MozillaCookieJar(str(file), delayload=False, policy=None)
         cookies.load()
 
         return (True, cookies)
@@ -1078,7 +1374,11 @@ def delete_dir(dir: Path) -> bool:
         dir.rmdir()
         return True
     except Exception as e:
-        LOG.error(f"Failed to delete directory '{dir}': {e}")
+        LOG.exception(
+            "Failed to delete directory '%s'.",
+            dir,
+            extra={"dir": str(dir), "exception_type": type(e).__name__},
+        )
         return False
 
 
@@ -1126,7 +1426,11 @@ def load_modules(root_path: Path, directory: Path):
         try:
             importlib.import_module(full_name)
         except ImportError as e:
-            LOG.error(f"Failed to import module '{full_name}': {e}")
+            LOG.exception(
+                "Failed to import module '%s'.",
+                full_name,
+                extra={"module_name": full_name, "exception_type": type(e).__name__},
+            )
 
 
 def parse_tags(text: str) -> tuple[str, dict[str, str | bool]]:
@@ -1184,35 +1488,6 @@ def str_to_dt(time_str: str, now=None) -> datetime:
     return dt
 
 
-def list_folders(path: Path, base: Path, depth_limit: int) -> list[str]:
-    """
-    List all folders relative to a base path, up to a specified depth limit.
-
-    Args:
-        path (Path): The path to start listing folders from.
-        base (Path): The base path to which the folders should be relative.
-        depth_limit (int): The maximum depth to traverse from the base path.
-
-    Returns:
-        list[str]: A list of folder paths relative to the base path, up to the specified
-
-    """
-    if "/" == str(path):
-        return []
-
-    rel_depth: int = len(path.relative_to(base).parts)
-    if rel_depth > depth_limit:
-        return []
-
-    folders: list[str] = []
-    for entry in path.iterdir():
-        if entry.is_dir():
-            folders.append(str(entry.relative_to(base)))
-            folders.extend(list_folders(entry, base, depth_limit))
-
-    return folders
-
-
 def get_channel_images(thumbnails: list[dict]) -> dict:
     """
     Extract channel images from a list of thumbnail dictionaries.

app/library/ag_utils.py

@@ -1,6 +1,20 @@
+import sys
 from typing import Any
 
 
+def _dict_value(data: dict[Any, Any], key: Any) -> tuple[bool, Any]:
+    if key not in data or data[key] is None:
+        return False, None
+    return True, data[key]
+
+
+def _dict_delete(data: dict[Any, Any], key: Any) -> bool:
+    if key not in data:
+        return False
+    del data[key]
+    return True
+
+
 def get_value(value: Any) -> Any:
     """
     Return the result of calling `value` if it is callable, else return `value`.
@@ -91,14 +105,19 @@ def ag(
                 return val
         return get_value(default)
     key = path
-    if isinstance(array, dict) and key in array and array[key] is not None:
-        return array[key]
+    if isinstance(array, dict):
+        found, value = _dict_value(array, key)
+        if found:
+            return value
     if not isinstance(key, str) or separator not in key:
         return get_value(default)
     current = array
     for segment in key.split(separator):
-        if isinstance(current, dict) and segment in current and current[segment] is not None:
-            current = current[segment]
+        if isinstance(current, dict):
+            found, value = _dict_value(current, segment)
+            if not found:
+                return get_value(default)
+            current = value
         elif isinstance(current, list):
             try:
                 idx = int(segment)
@@ -150,7 +169,8 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
         except TypeError:
             return False
     if isinstance(data, dict):
-        if path in data and data[path] is not None:
+        found, _ = _dict_value(data, path)
+        if found:
             return True
     elif isinstance(data, list) and isinstance(path, int):
         return 0 <= path < len(data) and data[path] is not None
@@ -158,8 +178,11 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
     segments = path_str.split(separator)
     current = data
     for seg in segments:
-        if isinstance(current, dict) and seg in current and current[seg] is not None:
-            current = current[seg]
+        if isinstance(current, dict):
+            found, value = _dict_value(current, seg)
+            if not found:
+                return False
+            current = value
         elif isinstance(current, list):
             try:
                 idx = int(seg)
@@ -189,17 +212,16 @@ def ag_delete(
         The modified structure.
 
     """
-    if isinstance(path, list | tuple):
-        for p in path:
-            ag_delete(data, p, separator)
-        return data
     if not isinstance(data, dict | list):
         try:
             data = vars(data)
         except TypeError:
             return data  # type: ignore
-    if isinstance(data, dict) and path in data:
-        del data[path]
+    if isinstance(path, list | tuple):
+        for p in path:
+            ag_delete(data, p, separator)
+        return data
+    if isinstance(data, dict) and _dict_delete(data, path):
         return data
     if isinstance(data, list) and isinstance(path, int):
         if 0 <= path < len(data):
@@ -210,8 +232,12 @@ def ag_delete(
     last = segments.pop()
     current = data
     for seg in segments:
-        if isinstance(current, dict) and seg in current:
-            current = current[seg]
+        if isinstance(current, dict):
+            found, value = _dict_value(current, seg)
+            if not found:
+                current = None
+                break
+            current = value
         elif isinstance(current, list):
             try:
                 idx = int(seg)
@@ -228,8 +254,8 @@ def ag_delete(
             break
     if current is None:
         return data
-    if isinstance(current, dict) and last in current:
-        del current[last]
+    if isinstance(current, dict):
+        _dict_delete(current, last)
     elif isinstance(current, list):
         try:
             idx = int(last)
@@ -246,10 +272,12 @@ def run_tests():
         assert ag_set(d, "a.c.d", 42) == {"a": {"b": 1, "c": {"d": 42}}}
 
     def test_ag_set_overwrite_error():
+        error = ""
         try:
             ag_set({"a": 1}, "a.b", 2)
         except RuntimeError as e:
-            assert "Cannot set value at path" in str(e)  # noqa: PT017
+            error = str(e)
+        assert "Cannot set value at path" in error
 
     def test_ag_get_value_callable_and_value():
         assert get_value(5) == 5
@@ -320,9 +348,9 @@ def run_tests():
     ]:
         try:
             test()
-            print(f"PASS: {test.__name__}")  # noqa: T201
+            sys.stdout.write(f"PASS: {test.__name__}\n")
         except AssertionError:
-            print(f"FAIL {test.__name__}")  # noqa: T201
+            sys.stdout.write(f"FAIL {test.__name__}\n")
 
 
 if __name__ == "__main__":