Compare commits
309 commits
create-pul
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c68646e266 | ||
|
|
3c571429e7 | ||
|
|
0ba52fa38e | ||
|
|
19f0b7819e | ||
|
|
3381ca3805 | ||
|
|
72f2373c42 | ||
|
|
44f5f2eee0 | ||
|
|
13424d1bcf | ||
|
|
3631947234 | ||
|
|
aedeaf9a0a | ||
|
|
18f54a28c6 | ||
|
|
db8d8663eb | ||
|
|
a614a48108 | ||
|
|
4f5ac8d8a0 | ||
|
|
c7fbe6b484 | ||
|
|
8c54ed6c0a | ||
|
|
9ada8b5a91 | ||
|
|
a346ed69d9 | ||
|
|
a194b9bf28 | ||
|
|
6df1250a43 | ||
|
|
ef8642ace7 | ||
|
|
8e14258c7a | ||
|
|
f65cdddbfe | ||
|
|
cee3bcf48d | ||
|
|
e7772b361e | ||
|
|
d3a3b5d95f | ||
|
|
522e6a75ef | ||
|
|
6cf2eb053c | ||
|
|
b35ac1831a | ||
|
|
cf4386e1a5 | ||
|
|
cb1c355e58 | ||
|
|
4ee7537733 | ||
|
|
fdc08989ae | ||
|
|
ed77ea508c | ||
|
|
8729f030c7 | ||
|
|
0eabc2197b | ||
|
|
28ec496430 | ||
|
|
2139e20332 | ||
|
|
1087596486 | ||
|
|
dbf1f59efb | ||
|
|
235438af51 | ||
|
|
4bea8a315a | ||
|
|
932f51c583 | ||
|
|
2f435a3a64 | ||
|
|
b462048679 | ||
|
|
ffbd82096c | ||
|
|
705973219c | ||
|
|
d9c9a85939 | ||
|
|
0c7ebb66ed | ||
|
|
c36295d76f | ||
|
|
ee0942b37a | ||
|
|
1cc6172862 | ||
|
|
0473c86fd6 | ||
|
|
d17aa6c29e | ||
|
|
8659f74a3b | ||
|
|
412ad9565d | ||
|
|
be6b68efdd | ||
|
|
3b49678272 | ||
|
|
2435144cb1 | ||
|
|
1189d31f5c | ||
|
|
163be08803 | ||
|
|
0a70c02b3e | ||
|
|
d5ae882df0 | ||
|
|
9eb78ecdc0 | ||
|
|
a924ddb718 | ||
|
|
67f3a24580 | ||
|
|
b9421f442a | ||
|
|
3b55c4d677 | ||
|
|
b8934d40eb | ||
|
|
2f6dcbb4d4 | ||
|
|
18673f0a55 | ||
|
|
714b6d95ba | ||
|
|
c9ba2b0f4c | ||
|
|
3d14a50f07 | ||
|
|
b5276feba0 | ||
|
|
54dedb6696 | ||
|
|
20e6c129ba | ||
|
|
47ca5461f1 | ||
|
|
93ce7a730b | ||
|
|
2214bc66de | ||
|
|
3ece619834 | ||
|
|
af84a03ad1 | ||
|
|
7e6b6f0544 | ||
|
|
91f527887c | ||
|
|
69dc19ad4f | ||
|
|
9272e0b173 | ||
|
|
c6d89dde9c | ||
|
|
89a92028e0 | ||
|
|
70d015a2a5 | ||
|
|
d13ac83425 | ||
|
|
83a416ec4c | ||
|
|
477dc41bb6 | ||
|
|
59fe171a9d | ||
|
|
46a3abebde | ||
|
|
b94d62accd | ||
|
|
d4c14c7991 | ||
|
|
c1b9034ef8 | ||
|
|
8ff6582316 | ||
|
|
d1a20b3972 | ||
|
|
f0b5e20af8 | ||
|
|
8273847d46 | ||
|
|
f4e5dfb2f6 | ||
|
|
03e8166afe | ||
|
|
1db5e26843 | ||
|
|
ee9e2a1d0b | ||
|
|
71153d6fb4 | ||
|
|
a81c73a469 | ||
|
|
70c42252a6 | ||
|
|
477c7e7c98 | ||
|
|
b95b6eb6be | ||
|
|
238deddb9d | ||
|
|
618ce4d387 | ||
|
|
5bf97047af | ||
|
|
2a13c47bb6 | ||
|
|
f91811ad50 | ||
|
|
f51b11c421 | ||
|
|
fa4da50403 | ||
|
|
851124a047 | ||
|
|
1103b93f00 | ||
|
|
8a852bcd24 | ||
|
|
b105749121 | ||
|
|
dd3059d01e | ||
|
|
5c862eb613 | ||
|
|
871f9e803e | ||
|
|
0de61566fb | ||
|
|
2323739c49 | ||
|
|
049b9a4e38 | ||
|
|
c968ce3ed2 | ||
|
|
125a1a81a0 | ||
|
|
71ba8ea7f7 | ||
|
|
bb37f7b6ba | ||
|
|
2c7ba8e184 | ||
|
|
ab8545392e | ||
|
|
c528e2b557 | ||
|
|
a5d91e58fa | ||
|
|
984d47e946 | ||
|
|
2a1cd0a632 | ||
|
|
7185bae58d | ||
|
|
c5e3899c26 | ||
|
|
aaab5a72e8 | ||
|
|
3de3471bd5 | ||
|
|
8b9077d476 | ||
|
|
963fa5870c | ||
|
|
b48daf11c8 | ||
|
|
c3d8aa8348 | ||
|
|
1dd464d6cc | ||
|
|
c91ca38035 | ||
|
|
46725e6f8e | ||
|
|
43e0128c3b | ||
|
|
8473618726 | ||
|
|
d2ce9940e8 | ||
|
|
4c9c9eab84 | ||
|
|
b4505d2fcd | ||
|
|
f6d1618c91 | ||
|
|
650ba810a8 | ||
|
|
286d8ce818 | ||
|
|
2cdfa2503b | ||
|
|
6aab6fd062 | ||
|
|
3447ac8032 | ||
|
|
3f11262afb | ||
|
|
ffd99e7c09 | ||
|
|
4c54f80b87 | ||
|
|
c310ae9abc | ||
|
|
4c5ebf2f39 | ||
|
|
cc3057f432 | ||
|
|
517d4e7528 | ||
|
|
a939654715 | ||
|
|
9b5c1d9ae7 | ||
|
|
33fbf4bda8 | ||
|
|
b4e5990b83 | ||
|
|
fd75d3cb8f | ||
|
|
6ba83ac00b | ||
|
|
35aa158f40 | ||
|
|
b7cb89f656 | ||
|
|
10182e1373 | ||
|
|
e398321901 | ||
|
|
f9b70fec5f | ||
|
|
6e84b78f22 | ||
|
|
78002ef99a | ||
|
|
043b019bde | ||
|
|
f66980c3bf | ||
|
|
f4ec06cef9 | ||
|
|
4c11e178b2 | ||
|
|
49cce85cfe | ||
|
|
034ff25e76 | ||
|
|
a324ed8ffd | ||
|
|
1169eea47b | ||
|
|
cc4b9032b0 | ||
|
|
0edac6c93e | ||
|
|
27e70b6df8 | ||
|
|
eb3ac026ba | ||
|
|
6607a233c0 | ||
|
|
b3bd02afc8 | ||
|
|
26455b62bf | ||
|
|
6c44c587c7 | ||
|
|
8e3e7dda80 | ||
|
|
3fe9188b47 | ||
|
|
b5bb2a202c | ||
|
|
1ce9acd316 | ||
|
|
de4b88eef3 | ||
|
|
3f3f63f0e2 | ||
|
|
8fe68f36ee | ||
|
|
4bf65f08c4 | ||
|
|
6e05efb146 | ||
|
|
c07465a676 | ||
|
|
4c76aa7123 | ||
|
|
b65008b628 | ||
|
|
2eab819e87 | ||
|
|
670c99510e | ||
|
|
f1f79afea1 | ||
|
|
38d99c0e1e | ||
|
|
4a26dc1a7f | ||
|
|
3bbb1509e2 | ||
|
|
cf2fb5fac8 | ||
|
|
2a60a848d6 | ||
|
|
a434e3a6aa | ||
|
|
8dfdf7f615 | ||
|
|
fd27ae924e | ||
|
|
adb191bb6d | ||
|
|
4398b30b38 | ||
|
|
f7f5ef0e59 | ||
|
|
cbc6277dc6 | ||
|
|
853a443b88 | ||
|
|
12b4b12344 | ||
|
|
1deba05902 | ||
|
|
9f0d88375f | ||
|
|
d385e41f6a | ||
|
|
1bbaf461b2 | ||
|
|
b29dc00bd8 | ||
|
|
53d3b3c81d | ||
|
|
a2f8cb5523 | ||
|
|
3425657dbf | ||
|
|
fb2aa7396b | ||
|
|
2beb6a6b08 | ||
|
|
53da40b5f5 | ||
|
|
31f64e67da | ||
|
|
ec1f2f448a | ||
|
|
c118522fa2 | ||
|
|
62b41bfdfa | ||
|
|
e79d182daa | ||
|
|
18fe5409ce | ||
|
|
2ed0bde90a | ||
|
|
6cf0b9120e | ||
|
|
8305518a01 | ||
|
|
80737f7d90 | ||
|
|
dc3c2b41e3 | ||
|
|
5bac7b8d20 | ||
|
|
daad7752b3 | ||
|
|
15e4232fc8 | ||
|
|
6d77de76cd | ||
|
|
5f7bbb5eba | ||
|
|
df699dd0c5 | ||
|
|
fd93472bd0 | ||
|
|
810a63d485 | ||
|
|
68473a1b0d | ||
|
|
28d902b2a9 | ||
|
|
00f3f27b8b | ||
|
|
25da932d21 | ||
|
|
87e93a5df6 | ||
|
|
9445e3ef86 | ||
|
|
0ed8aa5a30 | ||
|
|
3161fc0bb8 | ||
|
|
98970da663 | ||
|
|
19bdd84dd5 | ||
|
|
9adb6efbc5 | ||
|
|
674c53893f | ||
|
|
d5efb994ec | ||
|
|
10aeff4bfa | ||
|
|
9427f7b2a0 | ||
|
|
7a2dc8a452 | ||
|
|
243d4554d7 | ||
|
|
7b395d717d | ||
|
|
c944c36028 | ||
|
|
e52bef3781 | ||
|
|
73f97b80f3 | ||
|
|
327136bf09 | ||
|
|
48c849da5a | ||
|
|
a608d02d67 | ||
|
|
65dbfc775e | ||
|
|
ad561f7e73 | ||
|
|
806b795b38 | ||
|
|
a1f8245086 | ||
|
|
d135de5088 | ||
|
|
f554871b9f | ||
|
|
f790257b4f | ||
|
|
f0ac16cea7 | ||
|
|
ff540addff | ||
|
|
82d1401c27 | ||
|
|
12e18ee745 | ||
|
|
c54e9107b3 | ||
|
|
bcf5593523 | ||
|
|
9ab357983b | ||
|
|
501e97aabd | ||
|
|
084def9346 | ||
|
|
5be33c6a9a | ||
|
|
5b72e45432 | ||
|
|
0614a09231 | ||
|
|
1cc4d73051 | ||
|
|
ec8cd6b88a | ||
|
|
2e99ff1f5e | ||
|
|
9adf48e404 | ||
|
|
fa2fd2970a | ||
|
|
4094e98feb | ||
|
|
5301e134ce | ||
|
|
02ba97b8ab | ||
|
|
e415edd761 | ||
|
|
0682f9d443 | ||
|
|
0230f00a8c | ||
|
|
14d598eb10 |
147 changed files with 5867 additions and 4187 deletions
2
.github/workflows/build-pr.yml
vendored
2
.github/workflows/build-pr.yml
vendored
|
|
@ -15,7 +15,7 @@ on:
|
||||||
|
|
||||||
env:
|
env:
|
||||||
BUN_VERSION: latest
|
BUN_VERSION: latest
|
||||||
NODE_VERSION: 20
|
NODE_VERSION: 22
|
||||||
PYTHON_VERSION: "3.13"
|
PYTHON_VERSION: "3.13"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
|
|
||||||
2
.github/workflows/main.yml
vendored
2
.github/workflows/main.yml
vendored
|
|
@ -29,7 +29,7 @@ env:
|
||||||
DOCKERHUB_SLUG: arabcoders/ytptube
|
DOCKERHUB_SLUG: arabcoders/ytptube
|
||||||
GHCR_SLUG: ghcr.io/arabcoders/ytptube
|
GHCR_SLUG: ghcr.io/arabcoders/ytptube
|
||||||
BUN_VERSION: latest
|
BUN_VERSION: latest
|
||||||
NODE_VERSION: 20
|
NODE_VERSION: 22
|
||||||
PYTHON_VERSION: "3.13"
|
PYTHON_VERSION: "3.13"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
|
|
||||||
2
.github/workflows/native-build.yml
vendored
2
.github/workflows/native-build.yml
vendored
|
|
@ -39,7 +39,7 @@ jobs:
|
||||||
env:
|
env:
|
||||||
PYTHON_VERSION: "3.13"
|
PYTHON_VERSION: "3.13"
|
||||||
BUN_VERSION: latest
|
BUN_VERSION: latest
|
||||||
NODE_VERSION: 20
|
NODE_VERSION: 22
|
||||||
TAG_NAME: ${{ github.event.inputs.tag || github.ref_name }}
|
TAG_NAME: ${{ github.event.inputs.tag || github.ref_name }}
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
|
|
|
||||||
72
API.md
72
API.md
|
|
@ -101,6 +101,7 @@ This document describes the available endpoints and their usage. All endpoints r
|
||||||
- [POST /api/notifications/test](#post-apinotificationstest)
|
- [POST /api/notifications/test](#post-apinotificationstest)
|
||||||
- [GET /api/yt-dlp/options](#get-apiyt-dlpoptions)
|
- [GET /api/yt-dlp/options](#get-apiyt-dlpoptions)
|
||||||
- [GET /api/system/configuration](#get-apisystemconfiguration)
|
- [GET /api/system/configuration](#get-apisystemconfiguration)
|
||||||
|
- [GET /api/system/folders](#get-apisystemfolders)
|
||||||
- [GET /api/system/diagnostics](#get-apisystemdiagnostics)
|
- [GET /api/system/diagnostics](#get-apisystemdiagnostics)
|
||||||
- [GET /api/system/limits](#get-apisystemlimits)
|
- [GET /api/system/limits](#get-apisystemlimits)
|
||||||
- [POST /api/system/terminal](#post-apisystemterminal)
|
- [POST /api/system/terminal](#post-apisystemterminal)
|
||||||
|
|
@ -699,10 +700,16 @@ GET /api/history?type=queue&status=pending&order=ASC
|
||||||
|
|
||||||
This endpoint returns the current state of active downloads from memory.
|
This endpoint returns the current state of active downloads from memory.
|
||||||
|
|
||||||
|
**Query Parameters**:
|
||||||
|
- `limit` (optional): Override the configured queue display limit for this request. `0` means unlimited.
|
||||||
|
|
||||||
**Response**:
|
**Response**:
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"history_count": 0, // total number of completed items in history
|
"history_count": 0, // total number of completed items in history
|
||||||
|
"queue_count": 250, // total number of queued items
|
||||||
|
"queue_loaded": 100, // number of queued items included in this response
|
||||||
|
"queue_limit": 100, // effective display limit, 0 means unlimited
|
||||||
"queue":{
|
"queue":{
|
||||||
"id": "abc123",
|
"id": "abc123",
|
||||||
"url": "https://example.com/video",
|
"url": "https://example.com/video",
|
||||||
|
|
@ -2628,55 +2635,38 @@ or an error:
|
||||||
---
|
---
|
||||||
|
|
||||||
### GET /api/system/configuration
|
### GET /api/system/configuration
|
||||||
**Purpose**: Retrieve comprehensive system configuration including app settings, presets, download fields, queue status, and folder structure.
|
**Purpose**: Retrieve system configuration.
|
||||||
|
|
||||||
**Response**:
|
**Response**:
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"app": {
|
"app": {...},
|
||||||
"version": "...",
|
"presets": [...],
|
||||||
"download_path": "/path/to/downloads",
|
"dl_fields": [...],
|
||||||
"base_path": "/",
|
|
||||||
...
|
|
||||||
},
|
|
||||||
"presets": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"name": "default",
|
|
||||||
"description": "...",
|
|
||||||
...
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"dl_fields": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"name": "Title",
|
|
||||||
"field": "title",
|
|
||||||
"kind": "text",
|
|
||||||
...
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"paused": false,
|
"paused": false,
|
||||||
"folders": [
|
"history_count": 150
|
||||||
{"name": "folder1", "path": "folder1"},
|
}
|
||||||
{"name": "folder2", "path": "folder2"}
|
```
|
||||||
],
|
|
||||||
"history_count": 150,
|
---
|
||||||
"queue": [
|
|
||||||
{
|
### GET /api/system/folders
|
||||||
"id": "abc123",
|
**Purpose**: List child directories for a given relative path within the download directory.
|
||||||
"url": "https://example.com/video",
|
|
||||||
"status": "pending",
|
**Query Parameters**:
|
||||||
...
|
- `path=<relative-path>` (optional, default: root) - Relative path within the download directory.
|
||||||
}
|
|
||||||
]
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"path": "videos",
|
||||||
|
"folders": ["archive", "shorts", "2024"]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**Notes**:
|
**Notes**:
|
||||||
- This endpoint combines multiple data sources into a single response for efficient initialization
|
- Results are cached server-side for a short time.
|
||||||
- The `folders` array includes available download folders up to the configured depth limit
|
- Non-existent paths return an empty folder list.
|
||||||
- The `queue` array contains active download items
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -3478,7 +3468,7 @@ Emitted when a download item is moved between queue and history.
|
||||||
"data": {
|
"data": {
|
||||||
"id": "abc123",
|
"id": "abc123",
|
||||||
"from": "queue",
|
"from": "queue",
|
||||||
"to": "done"
|
"to": "history"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
|
||||||
27
FAQ.md
27
FAQ.md
|
|
@ -3,6 +3,9 @@
|
||||||
Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line,
|
Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line,
|
||||||
or the `environment:` section in `compose.yaml` file.
|
or the `environment:` section in `compose.yaml` file.
|
||||||
|
|
||||||
|
<details>
|
||||||
|
<summary>Click to expand</summary>
|
||||||
|
|
||||||
| Environment Variable | Description | Default |
|
| Environment Variable | Description | Default |
|
||||||
| ------------------------------- | ------------------------------------------------------------------- | --------------------- |
|
| ------------------------------- | ------------------------------------------------------------------- | --------------------- |
|
||||||
| TZ | The timezone to use for the application | `(not_set)` |
|
| TZ | The timezone to use for the application | `(not_set)` |
|
||||||
|
|
@ -44,10 +47,10 @@ or the `environment:` section in `compose.yaml` file.
|
||||||
| YTP_FLARESOLVERR_CACHE_TTL | The cache TTL (in seconds) for FlareSolverr solutions | `600` |
|
| YTP_FLARESOLVERR_CACHE_TTL | The cache TTL (in seconds) for FlareSolverr solutions | `600` |
|
||||||
| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` |
|
| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` |
|
||||||
| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` |
|
| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` |
|
||||||
|
| YTP_QUEUE_DISPLAY_LIMIT | Max queued downloads returned to the UI. `0` = unlimited | `100` |
|
||||||
| YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` |
|
| YTP_LIVE_PREMIERE_BUFFER | buffer time in minutes to add to video duration | `5` |
|
||||||
| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` |
|
| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` |
|
||||||
| YTP_TEMP_DISABLED | Disable temp files handling. | `false` |
|
| YTP_TEMP_DISABLED | Disable temp files handling. | `false` |
|
||||||
| YTP_DOWNLOAD_PATH_DEPTH | How many subdirectories to show in auto complete. | `1` |
|
|
||||||
| YTP_ALLOW_INTERNAL_URLS | Allow requests to internal URLs | `false` |
|
| YTP_ALLOW_INTERNAL_URLS | Allow requests to internal URLs | `false` |
|
||||||
| YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` |
|
| YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` |
|
||||||
| YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` |
|
| YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` |
|
||||||
|
|
@ -61,19 +64,19 @@ or the `environment:` section in `compose.yaml` file.
|
||||||
| YTP_THUMB_GENERATE | Enable ffmpeg thumbnail generation when no local thumbnail exists. | `true` |
|
| YTP_THUMB_GENERATE | Enable ffmpeg thumbnail generation when no local thumbnail exists. | `true` |
|
||||||
| YTP_THUMB_SIDECAR | Save generated thumbnails next to media instead of temp cache. | `false` |
|
| YTP_THUMB_SIDECAR | Save generated thumbnails next to media instead of temp cache. | `false` |
|
||||||
| YTP_DISABLE_EXEC | Strip some dangerous yt-dlp options. | `false` |
|
| YTP_DISABLE_EXEC | Strip some dangerous yt-dlp options. | `false` |
|
||||||
|
</details>
|
||||||
|
|
||||||
> [!NOTE]
|
> [!NOTE]
|
||||||
> To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`.
|
> To raise the worker limit for a specific extractor, set an env variable using this format: `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`
|
||||||
> The extractor name must be in uppercase, to know the extractor name, check the log for the specific extractor used for the download.
|
> The extractor name must be uppercase. You can find the extractor name in the download logs. This value cannot be
|
||||||
> The limit should not exceed the `YTP_MAX_WORKERS` value as it will be ignored.
|
> higher than `YTP_MAX_WORKERS`; higher values are ignored.
|
||||||
|
>
|
||||||
> [!IMPORTANT]
|
> `YTP_SIMPLE_MODE=true` only applies when the browser has no saved layout choice yet. Users can still choose a layout in
|
||||||
> The env variable `YTP_SIMPLE_MODE` only control what being displayed for first time visitor, the users can still switch between the two modes via the WebUI settings page.
|
> WebUI Settings. `/?simple=1` forces and saves Simple for that browser.
|
||||||
|
>
|
||||||
## Notes about YTP_AUTO_CLEAR_HISTORY_DAYS
|
> `YTP_AUTO_CLEAR_HISTORY_DAYS` `0` days means no automatic clearing of the download history. lowest value that will
|
||||||
|
> trigger the clearing is `1` day. This setting will **NOT** delete the downloaded files, it will only clear the
|
||||||
- `0` days means no automatic clearing of the download history. lowest value that will trigger the clearing is `1` day.
|
> history from the database.
|
||||||
- This setting will **NOT** delete the downloaded files, it will only clear the history from the database.
|
|
||||||
|
|
||||||
# Browser extensions & bookmarklets
|
# Browser extensions & bookmarklets
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,13 +6,16 @@ from app.features.conditions.migration import Migration
|
||||||
from app.library.Singleton import Singleton
|
from app.library.Singleton import Singleton
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import AsyncGenerator, Iterable
|
from collections.abc import Callable, Iterable
|
||||||
|
from contextlib import AbstractAsyncContextManager
|
||||||
|
|
||||||
from sqlalchemy.engine.result import Result
|
from sqlalchemy.engine.result import Result
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy.sql.elements import ColumnElement
|
from sqlalchemy.sql.elements import ColumnElement
|
||||||
from sqlalchemy.sql.selectable import Select
|
from sqlalchemy.sql.selectable import Select
|
||||||
|
|
||||||
|
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
|
||||||
|
|
||||||
from sqlalchemy import delete, func, or_, select
|
from sqlalchemy import delete, func, or_, select
|
||||||
|
|
||||||
from app.features.conditions.models import ConditionModel
|
from app.features.conditions.models import ConditionModel
|
||||||
|
|
@ -22,10 +25,26 @@ from app.library.log import get_logger
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
|
|
||||||
|
|
||||||
|
def _model_from_payload(payload: dict[str, Any]) -> ConditionModel:
|
||||||
|
model = ConditionModel()
|
||||||
|
for key, value in payload.items():
|
||||||
|
if not hasattr(model, key):
|
||||||
|
msg = f"'{key}' is an invalid keyword argument for ConditionModel"
|
||||||
|
raise TypeError(msg)
|
||||||
|
setattr(model, key, value)
|
||||||
|
return model
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_model(payload: ConditionModel | dict[str, Any]) -> ConditionModel:
|
||||||
|
if isinstance(payload, ConditionModel):
|
||||||
|
return payload
|
||||||
|
return _model_from_payload(payload)
|
||||||
|
|
||||||
|
|
||||||
class ConditionsRepository(metaclass=Singleton):
|
class ConditionsRepository(metaclass=Singleton):
|
||||||
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
|
def __init__(self, session: SessionFactory | None = None) -> None:
|
||||||
self._migrated = False
|
self._migrated = False
|
||||||
self.session: AsyncGenerator[AsyncSession] = session or get_session
|
self.session: SessionFactory = session or get_session
|
||||||
|
|
||||||
async def run_migrations(self) -> None:
|
async def run_migrations(self) -> None:
|
||||||
if self._migrated:
|
if self._migrated:
|
||||||
|
|
@ -38,7 +57,7 @@ class ConditionsRepository(metaclass=Singleton):
|
||||||
def get_instance() -> ConditionsRepository:
|
def get_instance() -> ConditionsRepository:
|
||||||
return ConditionsRepository()
|
return ConditionsRepository()
|
||||||
|
|
||||||
async def list(self) -> list[ConditionModel]:
|
async def all(self) -> list[ConditionModel]:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
result: Result[tuple[ConditionModel]] = await session.execute(
|
result: Result[tuple[ConditionModel]] = await session.execute(
|
||||||
select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc())
|
select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc())
|
||||||
|
|
@ -93,11 +112,11 @@ class ConditionsRepository(metaclass=Singleton):
|
||||||
result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1))
|
result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1))
|
||||||
return result.scalar_one_or_none()
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
async def create(self, payload: ConditionModel | dict) -> ConditionModel:
|
async def create(self, payload: ConditionModel | dict[str, Any]) -> ConditionModel:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
model: ConditionModel = ConditionModel(**payload) if isinstance(payload, dict) else payload
|
model = _coerce_model(payload)
|
||||||
if model.id is not None:
|
if model.id is not None:
|
||||||
model.id = None
|
model.id = None # ty: ignore
|
||||||
|
|
||||||
if await self.get_by_name(name=model.name) is not None:
|
if await self.get_by_name(name=model.name) is not None:
|
||||||
msg: str = f"Condition with name '{model.name}' already exists."
|
msg: str = f"Condition with name '{model.name}' already exists."
|
||||||
|
|
@ -160,13 +179,11 @@ class ConditionsRepository(metaclass=Singleton):
|
||||||
await session.commit()
|
await session.commit()
|
||||||
return model
|
return model
|
||||||
|
|
||||||
async def replace_all(self, items: Iterable[dict | ConditionModel]) -> list[ConditionModel]:
|
async def replace_all(self, items: Iterable[dict[str, Any] | ConditionModel]) -> list[ConditionModel]:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
try:
|
try:
|
||||||
await session.execute(delete(ConditionModel))
|
await session.execute(delete(ConditionModel))
|
||||||
models: list[ConditionModel] = [
|
models: list[ConditionModel] = [_coerce_model(item) for item in items]
|
||||||
ConditionModel(**item) if isinstance(item, dict) else item for item in items
|
|
||||||
]
|
|
||||||
session.add_all(models)
|
session.add_all(models)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|
|
||||||
|
|
@ -131,6 +131,12 @@ async def conditions_test(request: Request, encoder: Encoder, cache: Cache, conf
|
||||||
status=web.HTTPInternalServerError.status_code,
|
status=web.HTTPInternalServerError.status_code,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return web.json_response(
|
||||||
|
data={"error": "Failed to extract video info."},
|
||||||
|
status=web.HTTPInternalServerError.status_code,
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from app.features.ytdlp.mini_filter import match_str
|
from app.features.ytdlp.mini_filter import match_str
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from numbers import Number
|
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
|
||||||
from app.features.conditions.models import ConditionModel
|
from app.features.conditions.models import ConditionModel
|
||||||
from app.features.conditions.repository import ConditionsRepository
|
from app.features.conditions.repository import ConditionsRepository
|
||||||
|
from app.features.conditions.schemas import Condition
|
||||||
from app.features.ytdlp.mini_filter import match_str
|
from app.features.ytdlp.mini_filter import match_str
|
||||||
from app.library.Events import EventBus, Events
|
from app.library.Events import EventBus, Events
|
||||||
from app.library.log import get_logger
|
from app.library.log import get_logger
|
||||||
|
|
@ -13,7 +13,7 @@ from app.library.Singleton import Singleton
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
|
|
||||||
|
|
||||||
def _ignored_identifiers(ignore_conditions: Iterable[str | Number] | None) -> tuple[set[str], bool]:
|
def _ignored_identifiers(ignore_conditions: Iterable[str | int | float] | None) -> tuple[set[str], bool]:
|
||||||
ignored: set[str] = set()
|
ignored: set[str] = set()
|
||||||
ignore_all = False
|
ignore_all = False
|
||||||
|
|
||||||
|
|
@ -21,7 +21,7 @@ def _ignored_identifiers(ignore_conditions: Iterable[str | Number] | None) -> tu
|
||||||
return ignored, ignore_all
|
return ignored, ignore_all
|
||||||
|
|
||||||
for value in ignore_conditions:
|
for value in ignore_conditions:
|
||||||
if isinstance(value, bool) or not isinstance(value, (str, Number)):
|
if isinstance(value, bool) or not isinstance(value, (str, int, float)):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
identifier = str(value).strip()
|
identifier = str(value).strip()
|
||||||
|
|
@ -55,7 +55,7 @@ class Conditions(metaclass=Singleton):
|
||||||
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations")
|
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations")
|
||||||
|
|
||||||
async def get_all(self) -> list[ConditionModel]:
|
async def get_all(self) -> list[ConditionModel]:
|
||||||
return await self._repo.list()
|
return await self._repo.all()
|
||||||
|
|
||||||
async def save(self, item: ConditionModel | dict) -> ConditionModel:
|
async def save(self, item: ConditionModel | dict) -> ConditionModel:
|
||||||
"""
|
"""
|
||||||
|
|
@ -80,7 +80,7 @@ class Conditions(metaclass=Singleton):
|
||||||
if item.id is None or 0 == item.id:
|
if item.id is None or 0 == item.id:
|
||||||
model = await repo.create(item)
|
model = await repo.create(item)
|
||||||
else:
|
else:
|
||||||
model = await repo.update(item.id, item.serialize())
|
model = await repo.update(item.id, Condition.model_validate(item).model_dump())
|
||||||
except KeyError as exc:
|
except KeyError as exc:
|
||||||
raise ValueError(str(exc)) from exc
|
raise ValueError(str(exc)) from exc
|
||||||
|
|
||||||
|
|
@ -113,13 +113,15 @@ class Conditions(metaclass=Singleton):
|
||||||
repo = self._repo
|
repo = self._repo
|
||||||
return await repo.get(identifier)
|
return await repo.get(identifier)
|
||||||
|
|
||||||
async def match(self, info: dict, ignore_conditions: Iterable[str | Number] | None = None) -> ConditionModel | None:
|
async def match(
|
||||||
|
self, info: dict, ignore_conditions: Iterable[str | int | float] | None = None
|
||||||
|
) -> ConditionModel | None:
|
||||||
"""
|
"""
|
||||||
Check if any condition matches the info dict.
|
Check if any condition matches the info dict.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
info (dict): The info dict to check.
|
info (dict): The info dict to check.
|
||||||
ignore_conditions (Iterable[str | Number] | None): Condition ids or names to skip for this match.
|
ignore_conditions (Iterable[str | int | float] | None): Condition ids or names to skip for this match.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Condition|None: The condition if found, None otherwise.
|
Condition|None: The condition if found, None otherwise.
|
||||||
|
|
@ -133,7 +135,7 @@ class Conditions(metaclass=Singleton):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
repo = self._repo
|
repo = self._repo
|
||||||
items: list[ConditionModel] = await repo.list()
|
items: list[ConditionModel] = await repo.all()
|
||||||
if len(items) < 1:
|
if len(items) < 1:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
@ -187,7 +189,7 @@ class Conditions(metaclass=Singleton):
|
||||||
if not info or not isinstance(info, dict) or len(info) < 1:
|
if not info or not isinstance(info, dict) or len(info) < 1:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if not (item := await self.get(identifier)) or not item.enabled or not item.filter:
|
if not (item := await self.get(str(identifier))) or not item.enabled or not item.filter:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return item if match_str(item.filter, info) else None
|
return item if match_str(item.filter, info) else None
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
import pytest
|
import pytest
|
||||||
|
from typing import Any
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
from aiohttp.test_utils import make_mocked_request
|
from aiohttp.test_utils import make_mocked_request
|
||||||
|
|
||||||
|
|
@ -39,13 +40,13 @@ async def repo():
|
||||||
|
|
||||||
|
|
||||||
def _json_request(path: str, payload: object) -> web.Request:
|
def _json_request(path: str, payload: object) -> web.Request:
|
||||||
request = make_mocked_request("POST", path)
|
mock_request: Any = make_mocked_request("POST", path)
|
||||||
|
|
||||||
async def _json() -> object:
|
async def _json() -> object:
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
request.json = _json # type: ignore[attr-defined]
|
mock_request.json = _json
|
||||||
return request
|
return mock_request
|
||||||
|
|
||||||
|
|
||||||
class TestAllowInternalUrlsScope:
|
class TestAllowInternalUrlsScope:
|
||||||
|
|
@ -208,7 +209,7 @@ class TestConditionsRepository:
|
||||||
await repo.create({"name": "A", "filter": "test", "priority": 1})
|
await repo.create({"name": "A", "filter": "test", "priority": 1})
|
||||||
await repo.create({"name": "C", "filter": "test", "priority": 2})
|
await repo.create({"name": "C", "filter": "test", "priority": 2})
|
||||||
|
|
||||||
items = await repo.list()
|
items = await repo.all()
|
||||||
|
|
||||||
assert items[0].name == "C", "Highest priority should be first"
|
assert items[0].name == "C", "Highest priority should be first"
|
||||||
assert items[1].name == "A", "Same priority sorted alphabetically"
|
assert items[1].name == "A", "Same priority sorted alphabetically"
|
||||||
|
|
@ -240,6 +241,6 @@ class TestConditionsRepository:
|
||||||
|
|
||||||
assert len(result) == 2, "Should create 2 new conditions"
|
assert len(result) == 2, "Should create 2 new conditions"
|
||||||
|
|
||||||
all_items = await repo.list()
|
all_items = await repo.all()
|
||||||
assert len(all_items) == 2, "Should only have new conditions"
|
assert len(all_items) == 2, "Should only have new conditions"
|
||||||
assert all_items[0].name in ["New 1", "New 2"], "Should only have new items"
|
assert all_items[0].name in ["New 1", "New 2"], "Should only have new items"
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
from sqlalchemy import DateTime as SQLADateTime
|
from sqlalchemy import DateTime as SQLADateTime
|
||||||
from sqlalchemy import TypeDecorator
|
from sqlalchemy import Dialect, TypeDecorator
|
||||||
from sqlalchemy.orm import DeclarativeBase
|
from sqlalchemy.orm import DeclarativeBase
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -20,16 +20,18 @@ class UTCDateTime(TypeDecorator):
|
||||||
impl = SQLADateTime
|
impl = SQLADateTime
|
||||||
cache_ok = True
|
cache_ok = True
|
||||||
|
|
||||||
def process_bind_param(self, value: datetime | None, _dialect) -> datetime | None:
|
def process_bind_param(self, value: datetime | None, dialect: Dialect) -> datetime | None:
|
||||||
"""Convert datetime to UTC before storing."""
|
"""Convert datetime to UTC before storing."""
|
||||||
|
_ = dialect
|
||||||
if value is not None:
|
if value is not None:
|
||||||
if value.tzinfo is None:
|
if value.tzinfo is None:
|
||||||
return value.replace(tzinfo=UTC)
|
return value.replace(tzinfo=UTC)
|
||||||
return value.astimezone(UTC).replace(tzinfo=None)
|
return value.astimezone(UTC).replace(tzinfo=None)
|
||||||
return value
|
return value
|
||||||
|
|
||||||
def process_result_value(self, value: datetime | None, _dialect) -> datetime | None:
|
def process_result_value(self, value: datetime | None, dialect: Dialect) -> datetime | None:
|
||||||
"""Ensure datetime is timezone-aware (UTC) when loading."""
|
"""Ensure datetime is timezone-aware (UTC) when loading."""
|
||||||
|
_ = dialect
|
||||||
if value is not None and value.tzinfo is None:
|
if value is not None and value.tzinfo is None:
|
||||||
return value.replace(tzinfo=UTC)
|
return value.replace(tzinfo=UTC)
|
||||||
return value
|
return value
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any, cast
|
||||||
|
|
||||||
from sqlalchemy import delete, func, or_, select
|
from sqlalchemy import delete, func, or_, select
|
||||||
|
|
||||||
|
|
@ -11,20 +11,39 @@ from app.library.log import get_logger
|
||||||
from app.library.Singleton import Singleton
|
from app.library.Singleton import Singleton
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import AsyncGenerator, Iterable
|
from collections.abc import Callable, Iterable
|
||||||
|
from contextlib import AbstractAsyncContextManager
|
||||||
|
|
||||||
from sqlalchemy.engine.result import Result
|
from sqlalchemy.engine.result import Result
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy.sql.elements import ColumnElement
|
from sqlalchemy.sql.elements import ColumnElement
|
||||||
from sqlalchemy.sql.selectable import Select
|
from sqlalchemy.sql.selectable import Select
|
||||||
|
|
||||||
|
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
|
||||||
|
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
|
|
||||||
|
|
||||||
|
def _model_from_payload(payload: dict[str, Any]) -> DLFieldModel:
|
||||||
|
model = DLFieldModel()
|
||||||
|
for key, value in payload.items():
|
||||||
|
if not hasattr(model, key):
|
||||||
|
msg = f"'{key}' is an invalid keyword argument for DLFieldModel"
|
||||||
|
raise TypeError(msg)
|
||||||
|
setattr(model, key, value)
|
||||||
|
return model
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_model(payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
|
||||||
|
if isinstance(payload, DLFieldModel):
|
||||||
|
return payload
|
||||||
|
return _model_from_payload(payload)
|
||||||
|
|
||||||
|
|
||||||
class DLFieldsRepository(metaclass=Singleton):
|
class DLFieldsRepository(metaclass=Singleton):
|
||||||
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
|
def __init__(self, session: SessionFactory | None = None) -> None:
|
||||||
self._migrated = False
|
self._migrated = False
|
||||||
self.session: AsyncGenerator[AsyncSession] = session or get_session
|
self.session: SessionFactory = session or get_session
|
||||||
|
|
||||||
async def run_migrations(self) -> None:
|
async def run_migrations(self) -> None:
|
||||||
if self._migrated:
|
if self._migrated:
|
||||||
|
|
@ -37,7 +56,7 @@ class DLFieldsRepository(metaclass=Singleton):
|
||||||
def get_instance() -> DLFieldsRepository:
|
def get_instance() -> DLFieldsRepository:
|
||||||
return DLFieldsRepository()
|
return DLFieldsRepository()
|
||||||
|
|
||||||
async def list(self) -> list[DLFieldModel]:
|
async def all(self) -> list[DLFieldModel]:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
result: Result[tuple[DLFieldModel]] = await session.execute(
|
result: Result[tuple[DLFieldModel]] = await session.execute(
|
||||||
select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc())
|
select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc())
|
||||||
|
|
@ -90,9 +109,9 @@ class DLFieldsRepository(metaclass=Singleton):
|
||||||
result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1))
|
result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1))
|
||||||
return result.scalar_one_or_none()
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
async def create(self, payload: DLFieldModel | dict) -> DLFieldModel:
|
async def create(self, payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
model: DLFieldModel = DLFieldModel(**payload) if isinstance(payload, dict) else payload
|
model = _coerce_model(payload)
|
||||||
|
|
||||||
if await self.get_by_name(name=model.name) is not None:
|
if await self.get_by_name(name=model.name) is not None:
|
||||||
msg: str = f"DL field with name '{model.name}' already exists."
|
msg: str = f"DL field with name '{model.name}' already exists."
|
||||||
|
|
@ -153,13 +172,16 @@ class DLFieldsRepository(metaclass=Singleton):
|
||||||
await session.commit()
|
await session.commit()
|
||||||
return model
|
return model
|
||||||
|
|
||||||
async def replace_all(self, items: Iterable[dict | DLFieldModel]) -> list[DLFieldModel]:
|
async def replace_all(self, items: Iterable[dict[str, Any] | DLFieldModel]) -> list[DLFieldModel]:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
try:
|
try:
|
||||||
await session.execute(delete(DLFieldModel))
|
await session.execute(delete(DLFieldModel))
|
||||||
models: list[DLFieldModel] = [
|
models: list[DLFieldModel] = []
|
||||||
DLFieldModel(**item) if isinstance(item, dict) else item for item in items
|
for item in items:
|
||||||
]
|
if isinstance(item, dict):
|
||||||
|
models.append(DLFieldModel(**cast("dict[str, Any]", item)))
|
||||||
|
else:
|
||||||
|
models.append(item)
|
||||||
session.add_all(models)
|
session.add_all(models)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|
|
||||||
|
|
@ -33,10 +33,10 @@ class DLFields(metaclass=Singleton):
|
||||||
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations")
|
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations")
|
||||||
|
|
||||||
async def get_all(self) -> list[DLFieldModel]:
|
async def get_all(self) -> list[DLFieldModel]:
|
||||||
return await self._repo.list()
|
return await self._repo.all()
|
||||||
|
|
||||||
async def get_all_serialized(self) -> list[dict[str, Any]]:
|
async def get_all_serialized(self) -> list[dict[str, Any]]:
|
||||||
items = await self._repo.list()
|
items = await self._repo.all()
|
||||||
return [DLField.model_validate(item).model_dump() for item in items]
|
return [DLField.model_validate(item).model_dump() for item in items]
|
||||||
|
|
||||||
async def save(self, item: DLField | dict) -> DLFieldModel:
|
async def save(self, item: DLField | dict) -> DLFieldModel:
|
||||||
|
|
|
||||||
|
|
@ -204,7 +204,7 @@ class TestDLFieldsRepository:
|
||||||
await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1})
|
await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1})
|
||||||
await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0})
|
await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0})
|
||||||
|
|
||||||
items = await repo.list()
|
items = await repo.all()
|
||||||
|
|
||||||
assert items[0].name == "c", "Lowest order should be first"
|
assert items[0].name == "c", "Lowest order should be first"
|
||||||
assert items[1].name == "a", "Same order sorted alphabetically"
|
assert items[1].name == "a", "Same order sorted alphabetically"
|
||||||
|
|
@ -243,6 +243,6 @@ class TestDLFieldsRepository:
|
||||||
|
|
||||||
assert len(result) == 2, "Should create 2 new fields"
|
assert len(result) == 2, "Should create 2 new fields"
|
||||||
|
|
||||||
all_items = await repo.list()
|
all_items = await repo.all()
|
||||||
assert len(all_items) == 2, "Should only have new fields"
|
assert len(all_items) == 2, "Should only have new fields"
|
||||||
assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items"
|
assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items"
|
||||||
|
|
|
||||||
|
|
@ -11,20 +11,39 @@ from app.library.log import get_logger
|
||||||
from app.library.Singleton import Singleton
|
from app.library.Singleton import Singleton
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import Callable
|
||||||
|
from contextlib import AbstractAsyncContextManager
|
||||||
|
|
||||||
from sqlalchemy.engine.result import Result
|
from sqlalchemy.engine.result import Result
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy.sql.elements import ColumnElement
|
from sqlalchemy.sql.elements import ColumnElement
|
||||||
from sqlalchemy.sql.selectable import Select
|
from sqlalchemy.sql.selectable import Select
|
||||||
|
|
||||||
|
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
|
||||||
|
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
|
|
||||||
|
|
||||||
|
def _model_from_payload(payload: dict[str, Any]) -> NotificationModel:
|
||||||
|
model = NotificationModel()
|
||||||
|
for key, value in payload.items():
|
||||||
|
if not hasattr(model, key):
|
||||||
|
msg = f"'{key}' is an invalid keyword argument for NotificationModel"
|
||||||
|
raise TypeError(msg)
|
||||||
|
setattr(model, key, value)
|
||||||
|
return model
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_model(payload: NotificationModel | dict[str, Any]) -> NotificationModel:
|
||||||
|
if isinstance(payload, NotificationModel):
|
||||||
|
return payload
|
||||||
|
return _model_from_payload(payload)
|
||||||
|
|
||||||
|
|
||||||
class NotificationsRepository(metaclass=Singleton):
|
class NotificationsRepository(metaclass=Singleton):
|
||||||
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
|
def __init__(self, session: SessionFactory | None = None) -> None:
|
||||||
self._migrated = False
|
self._migrated = False
|
||||||
self.session: AsyncGenerator[AsyncSession] = session or get_session
|
self.session: SessionFactory = session or get_session
|
||||||
|
|
||||||
async def run_migrations(self) -> None:
|
async def run_migrations(self) -> None:
|
||||||
if self._migrated:
|
if self._migrated:
|
||||||
|
|
@ -37,7 +56,7 @@ class NotificationsRepository(metaclass=Singleton):
|
||||||
def get_instance() -> NotificationsRepository:
|
def get_instance() -> NotificationsRepository:
|
||||||
return NotificationsRepository()
|
return NotificationsRepository()
|
||||||
|
|
||||||
async def list(self) -> list[NotificationModel]:
|
async def all(self) -> list[NotificationModel]:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
result: Result[tuple[NotificationModel]] = await session.execute(
|
result: Result[tuple[NotificationModel]] = await session.execute(
|
||||||
select(NotificationModel).order_by(NotificationModel.name.asc())
|
select(NotificationModel).order_by(NotificationModel.name.asc())
|
||||||
|
|
@ -92,11 +111,11 @@ class NotificationsRepository(metaclass=Singleton):
|
||||||
result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1))
|
result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1))
|
||||||
return result.scalar_one_or_none()
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
async def create(self, payload: NotificationModel | dict) -> NotificationModel:
|
async def create(self, payload: NotificationModel | dict[str, Any]) -> NotificationModel:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
model: NotificationModel = NotificationModel(**payload) if isinstance(payload, dict) else payload
|
model = _coerce_model(payload)
|
||||||
if model.id is not None:
|
if model.id is not None:
|
||||||
model.id = None
|
model.id = None # ty: ignore
|
||||||
|
|
||||||
if await self.get_by_name(name=model.name) is not None:
|
if await self.get_by_name(name=model.name) is not None:
|
||||||
msg: str = f"Notification target with name '{model.name}' already exists."
|
msg: str = f"Notification target with name '{model.name}' already exists."
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any, cast
|
||||||
|
|
||||||
from app.features.notifications.models import NotificationModel
|
from app.features.notifications.models import NotificationModel
|
||||||
from app.features.notifications.repository import NotificationsRepository
|
from app.features.notifications.repository import NotificationsRepository
|
||||||
|
|
@ -64,10 +64,10 @@ class Notifications(metaclass=Singleton):
|
||||||
await self._repo.run_migrations()
|
await self._repo.run_migrations()
|
||||||
|
|
||||||
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations")
|
EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations")
|
||||||
EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{__class__.__name__}.emit")
|
EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{type(self).__name__}.emit")
|
||||||
|
|
||||||
async def list(self) -> list[NotificationModel]:
|
async def all(self) -> list[NotificationModel]:
|
||||||
return await self._repo.list()
|
return await self._repo.all()
|
||||||
|
|
||||||
async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]:
|
async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]:
|
||||||
return await self._repo.list_paginated(page, per_page)
|
return await self._repo.list_paginated(page, per_page)
|
||||||
|
|
@ -92,8 +92,8 @@ class Notifications(metaclass=Singleton):
|
||||||
async def delete(self, identifier: int | str) -> NotificationModel:
|
async def delete(self, identifier: int | str) -> NotificationModel:
|
||||||
return await self._repo.delete(identifier)
|
return await self._repo.delete(identifier)
|
||||||
|
|
||||||
async def send(self, ev: Event, wait: bool = True) -> list[dict] | list[Awaitable[dict]]:
|
async def send(self, ev: Event, wait: bool = True) -> list[dict | Awaitable[dict]]:
|
||||||
targets = await self._repo.list()
|
targets = await self._repo.all()
|
||||||
if len(targets) < 1:
|
if len(targets) < 1:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
@ -126,7 +126,7 @@ class Notifications(metaclass=Singleton):
|
||||||
if wait:
|
if wait:
|
||||||
return await asyncio.gather(*tasks)
|
return await asyncio.gather(*tasks)
|
||||||
|
|
||||||
return tasks
|
return cast("list[dict | Awaitable[dict]]", tasks)
|
||||||
|
|
||||||
def emit(self, e: Event, _, **__) -> None:
|
def emit(self, e: Event, _, **__) -> None:
|
||||||
if not NotificationEvents.is_valid(e.event):
|
if not NotificationEvents.is_valid(e.event):
|
||||||
|
|
@ -271,7 +271,7 @@ class Notifications(metaclass=Singleton):
|
||||||
|
|
||||||
if not status:
|
if not status:
|
||||||
msg = "Apprise failed to send notification."
|
msg = "Apprise failed to send notification."
|
||||||
raise RuntimeError(msg) # noqa: TRY301
|
self._raise_apprise_error(msg)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
LOG.exception(
|
LOG.exception(
|
||||||
"Failed to send Apprise notification for event '%s'.",
|
"Failed to send Apprise notification for event '%s'.",
|
||||||
|
|
@ -288,6 +288,10 @@ class Notifications(metaclass=Singleton):
|
||||||
|
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _raise_apprise_error(msg: str) -> None:
|
||||||
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
async def _send(self, target: Notification, ev: Event) -> dict:
|
async def _send(self, target: Notification, ev: Event) -> dict:
|
||||||
try:
|
try:
|
||||||
LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name)
|
LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name)
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,7 @@ class TestNotificationsRepository:
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_list_empty(self, repo):
|
async def test_list_empty(self, repo):
|
||||||
"""List returns empty when no notifications exist."""
|
"""List returns empty when no notifications exist."""
|
||||||
notifications = await repo.list()
|
notifications = await repo.all()
|
||||||
assert notifications == [], "Should return empty list when no notifications exist"
|
assert notifications == [], "Should return empty list when no notifications exist"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,7 @@ class Migration(FeatureMigration):
|
||||||
return
|
return
|
||||||
|
|
||||||
inserted = 0
|
inserted = 0
|
||||||
seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.list()}
|
seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.all()}
|
||||||
|
|
||||||
for index, item in enumerate(items):
|
for index, item in enumerate(items):
|
||||||
if not (normalized := self._normalize(item, index, seen_names)):
|
if not (normalized := self._normalize(item, index, seen_names)):
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,30 @@ if TYPE_CHECKING:
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
|
|
||||||
|
|
||||||
|
def _payload_data(payload: PresetModel | dict[str, Any]) -> dict[str, Any]:
|
||||||
|
if isinstance(payload, dict):
|
||||||
|
data: dict[str, Any] = {}
|
||||||
|
for key, value in payload.items():
|
||||||
|
if not isinstance(key, str):
|
||||||
|
msg = "Preset payload keys must be strings."
|
||||||
|
raise TypeError(msg)
|
||||||
|
data[key] = value
|
||||||
|
return data
|
||||||
|
|
||||||
|
return {
|
||||||
|
"name": payload.name,
|
||||||
|
"description": payload.description,
|
||||||
|
"folder": payload.folder,
|
||||||
|
"template": payload.template,
|
||||||
|
"cookies": payload.cookies,
|
||||||
|
"cli": payload.cli,
|
||||||
|
"default": payload.default,
|
||||||
|
"priority": payload.priority,
|
||||||
|
"created_at": payload.created_at,
|
||||||
|
"updated_at": payload.updated_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class PresetsRepository(metaclass=Singleton):
|
class PresetsRepository(metaclass=Singleton):
|
||||||
SORT_FIELDS: dict[str, Any] = {
|
SORT_FIELDS: dict[str, Any] = {
|
||||||
"id": PresetModel.id,
|
"id": PresetModel.id,
|
||||||
|
|
@ -73,15 +97,15 @@ class PresetsRepository(metaclass=Singleton):
|
||||||
LOG.debug("Refreshing presets cache due to configuration update.")
|
LOG.debug("Refreshing presets cache due to configuration update.")
|
||||||
await self._update_cache()
|
await self._update_cache()
|
||||||
|
|
||||||
Services.get_instance().add(__class__.__name__, self)
|
Services.get_instance().add(PresetsRepository.__name__, self)
|
||||||
EventBus.get_instance().subscribe(
|
EventBus.get_instance().subscribe(
|
||||||
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
|
Events.STARTED, handle_event, f"{PresetsRepository.__name__}.run_migrations"
|
||||||
).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache")
|
).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache")
|
||||||
|
|
||||||
async def _update_cache(self) -> None:
|
async def _update_cache(self) -> None:
|
||||||
from app.features.presets.service import Presets
|
from app.features.presets.service import Presets
|
||||||
|
|
||||||
await Presets.get_instance().refresh_cache(await self.list())
|
await Presets.get_instance().refresh_cache(await self.all())
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_instance() -> PresetsRepository:
|
def get_instance() -> PresetsRepository:
|
||||||
|
|
@ -97,7 +121,7 @@ class PresetsRepository(metaclass=Singleton):
|
||||||
LOG.error("Default preset '%s' not found, using 'default' preset.", default_name)
|
LOG.error("Default preset '%s' not found, using 'default' preset.", default_name)
|
||||||
config.default_preset = "default"
|
config.default_preset = "default"
|
||||||
|
|
||||||
async def list(self) -> list[PresetModel]:
|
async def all(self) -> list[PresetModel]:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
result: Result[tuple[PresetModel]] = await session.execute(
|
result: Result[tuple[PresetModel]] = await session.execute(
|
||||||
select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc())
|
select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc())
|
||||||
|
|
@ -238,24 +262,9 @@ class PresetsRepository(metaclass=Singleton):
|
||||||
result: Result[tuple[PresetModel]] = await session.execute(query.limit(1))
|
result: Result[tuple[PresetModel]] = await session.execute(query.limit(1))
|
||||||
return result.scalar_one_or_none()
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
async def create(self, payload: PresetModel | dict) -> PresetModel:
|
async def create(self, payload: PresetModel | dict[str, Any]) -> PresetModel:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
data: dict[str, Any]
|
data = _payload_data(payload)
|
||||||
if isinstance(payload, dict):
|
|
||||||
data = dict(payload)
|
|
||||||
else:
|
|
||||||
data = {
|
|
||||||
"name": payload.name,
|
|
||||||
"description": payload.description,
|
|
||||||
"folder": payload.folder,
|
|
||||||
"template": payload.template,
|
|
||||||
"cookies": payload.cookies,
|
|
||||||
"cli": payload.cli,
|
|
||||||
"default": payload.default,
|
|
||||||
"priority": payload.priority,
|
|
||||||
"created_at": payload.created_at,
|
|
||||||
"updated_at": payload.updated_at,
|
|
||||||
}
|
|
||||||
|
|
||||||
data.pop("id", None)
|
data.pop("id", None)
|
||||||
model = PresetModel(**data)
|
model = PresetModel(**data)
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ class Presets(metaclass=Singleton):
|
||||||
def __init__(self, repo: PresetsRepository | None = None) -> None:
|
def __init__(self, repo: PresetsRepository | None = None) -> None:
|
||||||
self._repo: PresetsRepository = repo or PresetsRepository.get_instance()
|
self._repo: PresetsRepository = repo or PresetsRepository.get_instance()
|
||||||
self._cache: list[tuple[int, str, Preset]] = []
|
self._cache: list[tuple[int, str, Preset]] = []
|
||||||
Services.get_instance().add(__class__.__name__, self)
|
Services.get_instance().add(type(self).__name__, self)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_instance() -> Presets:
|
def get_instance() -> Presets:
|
||||||
|
|
|
||||||
|
|
@ -57,7 +57,7 @@ class TestPresetsRepository:
|
||||||
await repo.create({"name": "A", "priority": 1})
|
await repo.create({"name": "A", "priority": 1})
|
||||||
await repo.create({"name": "C", "priority": 2})
|
await repo.create({"name": "C", "priority": 2})
|
||||||
|
|
||||||
items = await repo.list()
|
items = await repo.all()
|
||||||
|
|
||||||
assert items[0].name == "c", "Highest priority should be first"
|
assert items[0].name == "c", "Highest priority should be first"
|
||||||
assert items[1].name == "a", "Same priority should sort by name"
|
assert items[1].name == "a", "Same priority should sort by name"
|
||||||
|
|
|
||||||
|
|
@ -10,8 +10,6 @@ import os
|
||||||
import subprocess # qa: ignore
|
import subprocess # qa: ignore
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import anyio
|
|
||||||
|
|
||||||
from app.features.streaming.types import FFProbeError
|
from app.features.streaming.types import FFProbeError
|
||||||
from app.library.log import get_logger
|
from app.library.log import get_logger
|
||||||
from app.library.Utils import timed_lru_cache
|
from app.library.Utils import timed_lru_cache
|
||||||
|
|
@ -39,21 +37,26 @@ class FFStream:
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
if "codec_long_name" not in self.__dict__:
|
if "codec_long_name" not in self.__dict__:
|
||||||
self.codec_long_name = self.__dict__.get("codec_name", "")
|
self.__dict__["codec_long_name"] = self.__dict__.get("codec_name", "")
|
||||||
|
|
||||||
|
index = self.__dict__.get("index", "?")
|
||||||
|
codec_type = self.__dict__.get("codec_type", "unknown")
|
||||||
|
codec_long_name = self.__dict__.get("codec_long_name", "")
|
||||||
|
|
||||||
if self.is_video():
|
if self.is_video():
|
||||||
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, {self.framerate}, ({self.width}x{self.height})>"
|
return f"<Stream: #{index} [{codec_type}] {codec_long_name}, {self.__dict__.get('framerate')}, ({self.__dict__.get('width')}x{self.__dict__.get('height')})>"
|
||||||
|
|
||||||
if self.is_audio():
|
if self.is_audio():
|
||||||
return (
|
return (
|
||||||
f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}, channels: {self.channels} ({self.channel_layout}), "
|
f"<Stream: #{index} [{codec_type}] {codec_long_name}, "
|
||||||
"{sample_rate}Hz> "
|
f"channels: {self.__dict__.get('channels')} ({self.__dict__.get('channel_layout')}), "
|
||||||
|
f"{self.__dict__.get('sample_rate')}Hz> "
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.is_subtitle() or self.is_attachment():
|
if self.is_subtitle() or self.is_attachment():
|
||||||
return f"<Stream: #{self.index} [{self.codec_type}] {self.codec_long_name}>"
|
return f"<Stream: #{index} [{codec_type}] {codec_long_name}>"
|
||||||
|
|
||||||
return f"<Stream: #{self.index} [{self.codec_type}]>"
|
return f"<Stream: #{index} [{codec_type}]>"
|
||||||
|
|
||||||
def is_audio(self):
|
def is_audio(self):
|
||||||
"""
|
"""
|
||||||
|
|
@ -252,14 +255,14 @@ async def ffprobe(file: Path | str) -> FFProbeResult:
|
||||||
raise OSError(msg)
|
raise OSError(msg)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with await anyio.open_file(os.devnull, "w") as tempf:
|
proc = await asyncio.create_subprocess_exec(
|
||||||
await asyncio.create_subprocess_exec(
|
|
||||||
"ffprobe",
|
"ffprobe",
|
||||||
"-h",
|
"-h",
|
||||||
stdout=tempf,
|
stdout=asyncio.subprocess.DEVNULL,
|
||||||
stderr=tempf,
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
|
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
|
||||||
)
|
)
|
||||||
|
await proc.wait()
|
||||||
except FileNotFoundError as e:
|
except FileNotFoundError as e:
|
||||||
msg = "ffprobe not found."
|
msg = "ffprobe not found."
|
||||||
raise OSError(msg) from e
|
raise OSError(msg) from e
|
||||||
|
|
|
||||||
|
|
@ -39,7 +39,7 @@ class M3u8:
|
||||||
m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
|
m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
|
||||||
m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
|
m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
|
||||||
|
|
||||||
segmentSize: float = f"{self.duration:.6f}"
|
segmentSize: str = f"{self.duration:.6f}"
|
||||||
splits: int = math.ceil(duration / self.duration)
|
splits: int = math.ceil(duration / self.duration)
|
||||||
|
|
||||||
segmentParams: dict = {}
|
segmentParams: dict = {}
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ class Playlist:
|
||||||
"The path where files are downloaded."
|
"The path where files are downloaded."
|
||||||
|
|
||||||
async def make(self, file: Path) -> str:
|
async def make(self, file: Path) -> str:
|
||||||
ref: str = Path(str(file.relative_to(self.download_path)).strip("/"))
|
ref: Path = Path(str(file.relative_to(self.download_path)).strip("/"))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
ff = await ffprobe(file)
|
ff = await ffprobe(file)
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
# flake8: noqa: ARG002
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
|
|
@ -16,9 +16,13 @@ def detect_qsv_capabilities() -> dict[str, dict[str, bool]]:
|
||||||
Returns a dict where keys are codec names (e.g. "h264", "hevc", "vp9") and
|
Returns a dict where keys are codec names (e.g. "h264", "hevc", "vp9") and
|
||||||
values are dicts with keys "full" and "lp" booleans.
|
values are dicts with keys "full" and "lp" booleans.
|
||||||
"""
|
"""
|
||||||
|
vainfo = shutil.which("vainfo")
|
||||||
|
if not vainfo:
|
||||||
|
return {}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result: subprocess.CompletedProcess[str] = subprocess.run(
|
result: subprocess.CompletedProcess[str] = subprocess.run(
|
||||||
["vainfo"], # noqa: S607
|
[vainfo],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
check=False,
|
check=False,
|
||||||
|
|
@ -87,9 +91,13 @@ def ffmpeg_encoders() -> set[str]:
|
||||||
"""
|
"""
|
||||||
from app.library.config import SUPPORTED_CODECS
|
from app.library.config import SUPPORTED_CODECS
|
||||||
|
|
||||||
|
ffmpeg = shutil.which("ffmpeg")
|
||||||
|
if not ffmpeg:
|
||||||
|
return set()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result: subprocess.CompletedProcess[str] = subprocess.run(
|
result: subprocess.CompletedProcess[str] = subprocess.run(
|
||||||
["ffmpeg", "-hide_banner", "-loglevel", "error", "-encoders"], # noqa: S607
|
[ffmpeg, "-hide_banner", "-loglevel", "error", "-encoders"],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
check=False,
|
check=False,
|
||||||
|
|
@ -149,9 +157,11 @@ class _BaseBuilder:
|
||||||
codec_name: str
|
codec_name: str
|
||||||
|
|
||||||
def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
|
def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
|
||||||
|
_ = ctx
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
|
def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
|
||||||
|
_ = ctx
|
||||||
return [*args, "-codec:v", self.codec_name]
|
return [*args, "-codec:v", self.codec_name]
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -312,4 +322,4 @@ def encoder_fallback_chain(codec: str) -> tuple[str, ...]:
|
||||||
"libx264": [],
|
"libx264": [],
|
||||||
}
|
}
|
||||||
|
|
||||||
return chains.get(codec, chains["libx264"])
|
return tuple(chains.get(codec, chains["libx264"]))
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import subprocess # type: ignore
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
@ -178,6 +178,7 @@ class Segments:
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
assert proc.stdout is not None
|
||||||
while True:
|
while True:
|
||||||
chunk: bytes = await proc.stdout.read(1024 * 64)
|
chunk: bytes = await proc.stdout.read(1024 * 64)
|
||||||
if not chunk:
|
if not chunk:
|
||||||
|
|
@ -240,7 +241,9 @@ class Segments:
|
||||||
return (wrote_any, rc, client_disconnected_local, stderr_buf.decode("utf-8", errors="ignore").strip())
|
return (wrote_any, rc, client_disconnected_local, stderr_buf.decode("utf-8", errors="ignore").strip())
|
||||||
|
|
||||||
async def stream(self, file: Path, resp: web.StreamResponse) -> None:
|
async def stream(self, file: Path, resp: web.StreamResponse) -> None:
|
||||||
codec: str = Segments._cached_vcodec if Segments._cache_initialized else self.vcodec
|
codec: str = self.vcodec
|
||||||
|
if Segments._cache_initialized and Segments._cached_vcodec:
|
||||||
|
codec = Segments._cached_vcodec
|
||||||
if not codec or codec not in SUPPORTED_CODECS:
|
if not codec or codec not in SUPPORTED_CODECS:
|
||||||
codec: str = select_encoder(self.vcodec or "")
|
codec: str = select_encoder(self.vcodec or "")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import anyio
|
import anyio
|
||||||
import pysubs2
|
import pysubs2
|
||||||
|
|
@ -48,7 +49,8 @@ def ms_to_timestamp(ms: int) -> str:
|
||||||
return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}"
|
return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}"
|
||||||
|
|
||||||
|
|
||||||
SubstationFormat.ms_to_timestamp = ms_to_timestamp
|
_substation_format: Any = SubstationFormat
|
||||||
|
_substation_format.ms_to_timestamp = ms_to_timestamp
|
||||||
|
|
||||||
|
|
||||||
class Subtitle:
|
class Subtitle:
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,8 @@ THUMBNAIL_MISS_TTL = 3600.0
|
||||||
|
|
||||||
_LOCK = asyncio.Lock()
|
_LOCK = asyncio.Lock()
|
||||||
_IN_PROCESS: dict[str, asyncio.Task[Path | None]] = {}
|
_IN_PROCESS: dict[str, asyncio.Task[Path | None]] = {}
|
||||||
|
|
||||||
|
|
||||||
_SEM: asyncio.Semaphore | None = None
|
_SEM: asyncio.Semaphore | None = None
|
||||||
_SEM_LIMIT: int | None = None
|
_SEM_LIMIT: int | None = None
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ from pathlib import Path
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
from aiohttp.web import Request, Response
|
from aiohttp.web import Request, Response
|
||||||
|
from aiohttp.web_response import StreamResponse
|
||||||
|
|
||||||
from app.features.streaming.library.m3u8 import M3u8
|
from app.features.streaming.library.m3u8 import M3u8
|
||||||
from app.features.streaming.library.playlist import Playlist
|
from app.features.streaming.library.playlist import Playlist
|
||||||
|
|
@ -32,7 +33,7 @@ async def playlist_create(request: Request, config: Config, app: web.Application
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
file: str = request.match_info.get("file")
|
file: str | None = request.match_info.get("file")
|
||||||
|
|
||||||
if not file:
|
if not file:
|
||||||
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
||||||
|
|
@ -83,8 +84,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
file: str = request.match_info.get("file")
|
file: str | None = request.match_info.get("file")
|
||||||
mode: str = request.match_info.get("mode")
|
mode: str | None = request.match_info.get("mode")
|
||||||
|
|
||||||
if mode not in ["video", "subtitle"]:
|
if mode not in ["video", "subtitle"]:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
|
|
@ -94,13 +95,14 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
|
||||||
if not file:
|
if not file:
|
||||||
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
||||||
|
|
||||||
duration = request.query.get("duration", None)
|
duration: float | None = None
|
||||||
|
duration_arg = request.query.get("duration", None)
|
||||||
|
|
||||||
if "subtitle" in mode:
|
if "subtitle" in mode:
|
||||||
if not duration:
|
if not duration_arg:
|
||||||
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
|
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
|
||||||
|
|
||||||
duration = float(duration)
|
duration = float(duration_arg)
|
||||||
|
|
||||||
base_path: str = config.base_path.rstrip("/")
|
base_path: str = config.base_path.rstrip("/")
|
||||||
|
|
||||||
|
|
@ -124,6 +126,8 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
|
||||||
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
|
return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
|
||||||
|
|
||||||
if "subtitle" in mode:
|
if "subtitle" in mode:
|
||||||
|
if duration is None:
|
||||||
|
return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
|
||||||
text = await cls.make_subtitle(file=realFile, duration=duration)
|
text = await cls.make_subtitle(file=realFile, duration=duration)
|
||||||
else:
|
else:
|
||||||
text = await cls.make_stream(file=realFile)
|
text = await cls.make_stream(file=realFile)
|
||||||
|
|
@ -149,7 +153,7 @@ async def m3u8_create(request: Request, config: Config, app: web.Application) ->
|
||||||
|
|
||||||
|
|
||||||
@route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream")
|
@route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream")
|
||||||
async def segments_stream(request: Request, config: Config, app: web.Application) -> Response:
|
async def segments_stream(request: Request, config: Config, app: web.Application) -> StreamResponse:
|
||||||
"""
|
"""
|
||||||
Get the segments.
|
Get the segments.
|
||||||
|
|
||||||
|
|
@ -162,9 +166,9 @@ async def segments_stream(request: Request, config: Config, app: web.Application
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
file: str = request.match_info.get("file")
|
file: str | None = request.match_info.get("file")
|
||||||
segment: int = request.match_info.get("segment")
|
segment: str | None = request.match_info.get("segment")
|
||||||
sd: int = request.query.get("sd")
|
sd: str | None = request.query.get("sd")
|
||||||
vc: int = int(request.query.get("vc", 0))
|
vc: int = int(request.query.get("vc", 0))
|
||||||
ac: int = int(request.query.get("ac", 0))
|
ac: int = int(request.query.get("ac", 0))
|
||||||
|
|
||||||
|
|
@ -241,7 +245,7 @@ async def subtitles_get(request: Request, config: Config, app: web.Application)
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
file: str = request.match_info.get("file")
|
file: str | None = request.match_info.get("file")
|
||||||
|
|
||||||
if not file:
|
if not file:
|
||||||
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
||||||
|
|
@ -299,7 +303,7 @@ async def subtitles_manifest_get(request: Request, config: Config, app: web.Appl
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
file: str = request.match_info.get("file")
|
file: str | None = request.match_info.get("file")
|
||||||
|
|
||||||
if not file:
|
if not file:
|
||||||
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
|
||||||
|
|
@ -354,7 +358,7 @@ async def subtitles_track_get(request: Request, config: Config, app: web.Applica
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
file: str = request.match_info.get("file")
|
file: str | None = request.match_info.get("file")
|
||||||
source_format: str | None = request.match_info.get("source_format")
|
source_format: str | None = request.match_info.get("source_format")
|
||||||
|
|
||||||
if not file:
|
if not file:
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from aiohttp import web
|
||||||
|
|
||||||
from app.features.streaming.library.segments import Segments
|
from app.features.streaming.library.segments import Segments
|
||||||
from app.tests.helpers import get_test_system_temp_root
|
from app.tests.helpers import get_test_system_temp_root
|
||||||
|
|
@ -148,18 +149,18 @@ class _FakeProc:
|
||||||
self.killed = True
|
self.killed = True
|
||||||
|
|
||||||
|
|
||||||
class _FakeResp:
|
class _FakeResp(web.StreamResponse):
|
||||||
def __init__(self, fail_with: Exception | None = None) -> None:
|
def __init__(self, fail_with: BaseException | None = None) -> None:
|
||||||
self.data: bytearray = bytearray()
|
self.data: bytearray = bytearray()
|
||||||
self.eof = False
|
self.eof = False
|
||||||
self._exc = fail_with
|
self._exc = fail_with
|
||||||
|
|
||||||
async def write(self, data: bytes) -> None:
|
async def write(self, data: bytes | bytearray | memoryview, *_args: Any, **_kwargs: Any) -> None:
|
||||||
if self._exc:
|
if self._exc:
|
||||||
raise self._exc
|
raise self._exc
|
||||||
self.data.extend(data)
|
self.data.extend(data)
|
||||||
|
|
||||||
async def write_eof(self) -> None:
|
async def write_eof(self, *_args: Any, **_kwargs: Any) -> None:
|
||||||
self.eof = True
|
self.eof = True
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
from unittest.mock import AsyncMock
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
@ -260,7 +261,7 @@ async def test_retry_no_seek(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
output = temp_dir / ".jpg"
|
output = temp_dir / ".jpg"
|
||||||
media.write_text("video")
|
media.write_text("video")
|
||||||
|
|
||||||
ff_info = FFProbeResult()
|
ff_info: Any = FFProbeResult()
|
||||||
ff_info.metadata = {"duration": "120.0"}
|
ff_info.metadata = {"duration": "120.0"}
|
||||||
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
|
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
|
||||||
|
|
||||||
|
|
@ -321,7 +322,7 @@ async def test_limit_wait(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
media1.write_text("video")
|
media1.write_text("video")
|
||||||
media2.write_text("video")
|
media2.write_text("video")
|
||||||
|
|
||||||
ff_info = FFProbeResult()
|
ff_info: Any = FFProbeResult()
|
||||||
ff_info.metadata = {"duration": "60.0"}
|
ff_info.metadata = {"duration": "60.0"}
|
||||||
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
|
ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
|
||||||
monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))
|
monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,3 @@
|
||||||
# flake8: noqa: ARG004
|
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult
|
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult
|
||||||
|
|
@ -12,6 +11,7 @@ if TYPE_CHECKING:
|
||||||
class BaseHandler:
|
class BaseHandler:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def can_handle(task: HandleTask) -> bool:
|
async def can_handle(task: HandleTask) -> bool:
|
||||||
|
_ = task
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
@ -24,6 +24,7 @@ class BaseHandler:
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def parse(url: str) -> Any | None:
|
def parse(url: str) -> Any | None:
|
||||||
|
_ = url
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
|
||||||
|
|
@ -64,7 +64,7 @@ class GenericTaskHandler(BaseHandler):
|
||||||
from app.features.tasks.definitions.utils import model_to_schema
|
from app.features.tasks.definitions.utils import model_to_schema
|
||||||
|
|
||||||
repo = TaskDefinitionsRepository.get_instance()
|
repo = TaskDefinitionsRepository.get_instance()
|
||||||
models = await repo.list()
|
models = await repo.all()
|
||||||
|
|
||||||
cls._definitions = [model_to_schema(model) for model in models]
|
cls._definitions = [model_to_schema(model) for model in models]
|
||||||
return cls._definitions
|
return cls._definitions
|
||||||
|
|
@ -141,7 +141,8 @@ class GenericTaskHandler(BaseHandler):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004
|
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
|
||||||
|
_ = config
|
||||||
definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
|
definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
|
||||||
if not definition:
|
if not definition:
|
||||||
return TaskFailure(message="No generic task definition matched the provided URL.")
|
return TaskFailure(message="No generic task definition matched the provided URL.")
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ from app.features.tasks.definitions.results import HandleTask, TaskFailure, Task
|
||||||
from app.features.ytdlp.extractor import fetch_info
|
from app.features.ytdlp.extractor import fetch_info
|
||||||
from app.features.ytdlp.utils import get_archive_id
|
from app.features.ytdlp.utils import get_archive_id
|
||||||
from app.library.cache import Cache
|
from app.library.cache import Cache
|
||||||
|
from app.library.config import Config
|
||||||
from app.library.log import get_logger
|
from app.library.log import get_logger
|
||||||
|
|
||||||
from ._base_handler import BaseHandler
|
from ._base_handler import BaseHandler
|
||||||
|
|
@ -160,12 +161,14 @@ class RssGenericHandler(BaseHandler):
|
||||||
return feed_url, items, real_count
|
return feed_url, items, real_count
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
|
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
|
||||||
|
_ = config
|
||||||
"""
|
"""
|
||||||
Extract items from an RSS/Atom feed.
|
Extract items from an RSS/Atom feed.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
task (Task): The task containing the feed URL.
|
task (Task): The task containing the feed URL.
|
||||||
|
config (Config | None): Optional handler configuration.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
TaskResult | TaskFailure: Extraction result with parsed items or failure information.
|
TaskResult | TaskFailure: Extraction result with parsed items or failure information.
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import httpx
|
||||||
|
|
||||||
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
||||||
from app.features.ytdlp.utils import get_archive_id
|
from app.features.ytdlp.utils import get_archive_id
|
||||||
|
from app.library.config import Config
|
||||||
from app.library.log import get_logger
|
from app.library.log import get_logger
|
||||||
|
|
||||||
from ._base_handler import BaseHandler
|
from ._base_handler import BaseHandler
|
||||||
|
|
@ -177,7 +178,8 @@ class TverHandler(BaseHandler):
|
||||||
return feed_url, items, has_items
|
return feed_url, items, has_items
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
|
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
|
||||||
|
_ = config
|
||||||
series_id: str | None = TverHandler.parse(task.url)
|
series_id: str | None = TverHandler.parse(task.url)
|
||||||
if not series_id:
|
if not series_id:
|
||||||
return TaskFailure(message="Unrecognized Tver series URL.")
|
return TaskFailure(message="Unrecognized Tver series URL.")
|
||||||
|
|
@ -208,10 +210,11 @@ class TverHandler(BaseHandler):
|
||||||
if not (url := entry.get("url")):
|
if not (url := entry.get("url")):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
archive_id: str = entry.get("archive_id")
|
archive_id: str | None = entry.get("archive_id")
|
||||||
task_items.append(
|
metadata = entry.get("metadata", {})
|
||||||
TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=entry.get("metadata", {}))
|
if not isinstance(metadata, dict):
|
||||||
)
|
metadata = {}
|
||||||
|
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
|
||||||
|
|
||||||
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})
|
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import httpx
|
||||||
|
|
||||||
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
||||||
from app.features.ytdlp.utils import get_archive_id
|
from app.features.ytdlp.utils import get_archive_id
|
||||||
|
from app.library.config import Config
|
||||||
from app.library.log import get_logger
|
from app.library.log import get_logger
|
||||||
|
|
||||||
from ._base_handler import BaseHandler
|
from ._base_handler import BaseHandler
|
||||||
|
|
@ -92,7 +93,8 @@ class TwitchHandler(BaseHandler):
|
||||||
return feed_url, items, has_items
|
return feed_url, items, has_items
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
|
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
|
||||||
|
_ = config
|
||||||
handle_name: str | None = TwitchHandler.parse(task.url)
|
handle_name: str | None = TwitchHandler.parse(task.url)
|
||||||
if not handle_name:
|
if not handle_name:
|
||||||
return TaskFailure(message="Unrecognized Twitch channel URL.")
|
return TaskFailure(message="Unrecognized Twitch channel URL.")
|
||||||
|
|
@ -122,7 +124,7 @@ class TwitchHandler(BaseHandler):
|
||||||
if not (url := entry.get("url")):
|
if not (url := entry.get("url")):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
archive_id: str = entry.get("archive_id")
|
archive_id: str | None = entry.get("archive_id")
|
||||||
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id))
|
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id))
|
||||||
|
|
||||||
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})
|
return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import httpx
|
||||||
|
|
||||||
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
||||||
from app.features.ytdlp.utils import get_archive_id
|
from app.features.ytdlp.utils import get_archive_id
|
||||||
|
from app.library.config import Config
|
||||||
from app.library.log import get_logger
|
from app.library.log import get_logger
|
||||||
|
|
||||||
from ._base_handler import BaseHandler
|
from ._base_handler import BaseHandler
|
||||||
|
|
@ -103,7 +104,8 @@ class YoutubeHandler(BaseHandler):
|
||||||
return feed_url, items, real_count
|
return feed_url, items, real_count
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def extract(task: HandleTask) -> TaskResult | TaskFailure:
|
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
|
||||||
|
_ = config
|
||||||
parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
|
parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
|
||||||
if not parsed:
|
if not parsed:
|
||||||
return TaskFailure(message="Unrecognized YouTube channel or playlist URL.")
|
return TaskFailure(message="Unrecognized YouTube channel or playlist URL.")
|
||||||
|
|
@ -133,7 +135,7 @@ class YoutubeHandler(BaseHandler):
|
||||||
if not (url := entry.get("url")):
|
if not (url := entry.get("url")):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
archive_id: str = entry.get("archive_id")
|
archive_id: str | None = entry.get("archive_id")
|
||||||
metadata: dict[str, Any] = {"published": entry.get("published")}
|
metadata: dict[str, Any] = {"published": entry.get("published")}
|
||||||
|
|
||||||
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
|
task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
|
||||||
|
|
|
||||||
|
|
@ -14,20 +14,23 @@ from app.library.Services import Services
|
||||||
from app.library.Singleton import Singleton
|
from app.library.Singleton import Singleton
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import Callable
|
||||||
|
from contextlib import AbstractAsyncContextManager
|
||||||
|
|
||||||
from sqlalchemy.engine.result import Result
|
from sqlalchemy.engine.result import Result
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy.sql.elements import ColumnElement
|
from sqlalchemy.sql.elements import ColumnElement
|
||||||
from sqlalchemy.sql.selectable import Select
|
from sqlalchemy.sql.selectable import Select
|
||||||
|
|
||||||
|
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
|
||||||
|
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
|
|
||||||
|
|
||||||
class TaskDefinitionsRepository(metaclass=Singleton):
|
class TaskDefinitionsRepository(metaclass=Singleton):
|
||||||
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
|
def __init__(self, session: SessionFactory | None = None) -> None:
|
||||||
self._migrated = False
|
self._migrated = False
|
||||||
self.session: AsyncGenerator[AsyncSession] = session or get_session
|
self.session: SessionFactory = session or get_session
|
||||||
|
|
||||||
async def run_migrations(self) -> None:
|
async def run_migrations(self) -> None:
|
||||||
if self._migrated:
|
if self._migrated:
|
||||||
|
|
@ -51,12 +54,12 @@ class TaskDefinitionsRepository(metaclass=Singleton):
|
||||||
LOG.debug("Refreshing task definitions due to configuration update.")
|
LOG.debug("Refreshing task definitions due to configuration update.")
|
||||||
await GenericTaskHandler.refresh_definitions(force=True)
|
await GenericTaskHandler.refresh_definitions(force=True)
|
||||||
|
|
||||||
Services.get_instance().add(__class__.__name__, self)
|
Services.get_instance().add(TaskDefinitionsRepository.__name__, self)
|
||||||
EventBus.get_instance().subscribe(
|
EventBus.get_instance().subscribe(
|
||||||
Events.STARTED, handle_event, f"{__class__.__name__}.run_migrations"
|
Events.STARTED, handle_event, f"{TaskDefinitionsRepository.__name__}.run_migrations"
|
||||||
).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions")
|
).subscribe(Events.CONFIG_UPDATE, handler, "GenericTaskHandler.refresh_definitions")
|
||||||
|
|
||||||
async def list(self) -> list[TaskDefinitionModel]:
|
async def all(self) -> list[TaskDefinitionModel]:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
result: Result[tuple[TaskDefinitionModel]] = await session.execute(
|
result: Result[tuple[TaskDefinitionModel]] = await session.execute(
|
||||||
select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc())
|
select(TaskDefinitionModel).order_by(TaskDefinitionModel.priority.asc(), TaskDefinitionModel.name.asc())
|
||||||
|
|
@ -115,9 +118,9 @@ class TaskDefinitionsRepository(metaclass=Singleton):
|
||||||
|
|
||||||
async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel:
|
async def create(self, payload: dict[str, Any]) -> TaskDefinitionModel:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
model: TaskDefinitionModel = TaskDefinitionModel(**payload) if isinstance(payload, dict) else payload
|
model: TaskDefinitionModel = TaskDefinitionModel(**payload)
|
||||||
if model.id is not None:
|
if model.id is not None:
|
||||||
model.id = None
|
model.id = None # ty: ignore
|
||||||
|
|
||||||
if await self.get_by_name(name=model.name) is not None:
|
if await self.get_by_name(name=model.name) is not None:
|
||||||
msg: str = f"Task definition with name '{model.name}' already exists."
|
msg: str = f"Task definition with name '{model.name}' already exists."
|
||||||
|
|
|
||||||
|
|
@ -48,10 +48,13 @@ class HandleTask(TaskSchema):
|
||||||
if isinstance(ret, tuple):
|
if isinstance(ret, tuple):
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
archive_file: Path = ret.get("file")
|
archive_file = ret.get("file")
|
||||||
items: set[str] = ret.get("items", set())
|
items = ret.get("items", set())
|
||||||
|
if not isinstance(archive_file, Path) or not isinstance(items, set):
|
||||||
|
return (False, "Failed to get archive information.")
|
||||||
|
archive_items = [item for item in items if isinstance(item, str)]
|
||||||
|
|
||||||
if len(items) < 1 or not archive_add(archive_file, list(items)):
|
if len(archive_items) < 1 or not archive_add(archive_file, archive_items):
|
||||||
return (True, "No new items to mark as downloaded.")
|
return (True, "No new items to mark as downloaded.")
|
||||||
|
|
||||||
return (True, f"Task '{self.name}' items marked as downloaded.")
|
return (True, f"Task '{self.name}' items marked as downloaded.")
|
||||||
|
|
@ -70,10 +73,13 @@ class HandleTask(TaskSchema):
|
||||||
if isinstance(ret, tuple):
|
if isinstance(ret, tuple):
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
archive_file: Path = ret.get("file")
|
archive_file = ret.get("file")
|
||||||
items: set[str] = ret.get("items", set())
|
items = ret.get("items", set())
|
||||||
|
if not isinstance(archive_file, Path) or not isinstance(items, set):
|
||||||
|
return (False, "Failed to get archive information.")
|
||||||
|
archive_items = [item for item in items if isinstance(item, str)]
|
||||||
|
|
||||||
if len(items) < 1 or not archive_delete(archive_file, list(items)):
|
if len(archive_items) < 1 or not archive_delete(archive_file, archive_items):
|
||||||
return (True, "No items to remove from archive file.")
|
return (True, "No items to remove from archive file.")
|
||||||
|
|
||||||
return (True, f"Removed '{self.name}' items from archive file.")
|
return (True, f"Removed '{self.name}' items from archive file.")
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ import random
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
from app.features.tasks.definitions.handlers._base_handler import BaseHandler
|
||||||
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
||||||
from app.features.tasks.models import TaskModel
|
from app.features.tasks.models import TaskModel
|
||||||
from app.features.ytdlp.utils import archive_read
|
from app.features.ytdlp.utils import archive_read
|
||||||
|
|
@ -27,7 +28,7 @@ LOG = get_logger()
|
||||||
|
|
||||||
class TaskHandle:
|
class TaskHandle:
|
||||||
def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None:
|
def __init__(self, scheduler: Scheduler, tasks: TasksRepository, config: Config) -> None:
|
||||||
self._handlers: list[type] = []
|
self._handlers: list[type[BaseHandler]] = []
|
||||||
"The available handlers."
|
"The available handlers."
|
||||||
self._repo: TasksRepository = tasks
|
self._repo: TasksRepository = tasks
|
||||||
"The tasks manager."
|
"The tasks manager."
|
||||||
|
|
@ -35,7 +36,7 @@ class TaskHandle:
|
||||||
"The scheduler."
|
"The scheduler."
|
||||||
self._config: Config = config
|
self._config: Config = config
|
||||||
"The configuration."
|
"The configuration."
|
||||||
self._task_name: str = f"{__class__.__name__}._dispatcher"
|
self._task_name: str = f"{TaskHandle.__name__}._dispatcher"
|
||||||
"The task name for the scheduler."
|
"The task name for the scheduler."
|
||||||
self._queued: dict[str, set[str]] = {}
|
self._queued: dict[str, set[str]] = {}
|
||||||
"Queued archive IDs per handler."
|
"Queued archive IDs per handler."
|
||||||
|
|
@ -45,11 +46,11 @@ class TaskHandle:
|
||||||
EventBus.get_instance().subscribe(
|
EventBus.get_instance().subscribe(
|
||||||
Events.ITEM_ERROR,
|
Events.ITEM_ERROR,
|
||||||
self._handle_item_error,
|
self._handle_item_error,
|
||||||
f"{__class__.__name__}.item_error",
|
f"{TaskHandle.__name__}.item_error",
|
||||||
)
|
)
|
||||||
|
|
||||||
def load(self) -> None:
|
def load(self) -> None:
|
||||||
self._handlers: list[type] = self._discover()
|
self._handlers: list[type[BaseHandler]] = self._discover()
|
||||||
|
|
||||||
timer: str = self._config.tasks_handler_timer
|
timer: str = self._config.tasks_handler_timer
|
||||||
try:
|
try:
|
||||||
|
|
@ -72,16 +73,16 @@ class TaskHandle:
|
||||||
self._scheduler.add(
|
self._scheduler.add(
|
||||||
timer=timer,
|
timer=timer,
|
||||||
func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"),
|
func=lambda: asyncio.create_task(self._dispatcher(), name="task-handler-dispatcher"),
|
||||||
id=f"{__class__.__name__}._dispatcher",
|
id=f"{TaskHandle.__name__}._dispatcher",
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _dispatcher(self):
|
async def _dispatcher(self):
|
||||||
s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []}
|
s: dict[str, list[str]] = {"h": [], "d": [], "u": [], "f": []}
|
||||||
|
|
||||||
handler_groups: dict[str, list[tuple[HandleTask, type]]] = {}
|
handler_groups: dict[str, list[tuple[HandleTask, type[BaseHandler]]]] = {}
|
||||||
dispatches: list[tuple[HandleTask, type, asyncio.Task[TaskResult | TaskFailure | None]]] = []
|
dispatches: list[tuple[HandleTask, type[BaseHandler], asyncio.Task[TaskResult | TaskFailure | None]]] = []
|
||||||
|
|
||||||
tasks: list[TaskModel] = await self._repo.list()
|
tasks: list[TaskModel] = await self._repo.all()
|
||||||
|
|
||||||
for task_model in tasks:
|
for task_model in tasks:
|
||||||
task: HandleTask = HandleTask.model_validate(task_model)
|
task: HandleTask = HandleTask.model_validate(task_model)
|
||||||
|
|
@ -100,7 +101,7 @@ class TaskHandle:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
handler: type | None = await self._find_handler(task)
|
handler: type[BaseHandler] | None = await self._find_handler(task)
|
||||||
if handler is None:
|
if handler is None:
|
||||||
s["u"].append(task.name)
|
s["u"].append(task.name)
|
||||||
continue
|
continue
|
||||||
|
|
@ -181,7 +182,9 @@ class TaskHandle:
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _dispatch(self, task: HandleTask, handler: type, delay: float) -> TaskResult | TaskFailure | None:
|
async def _dispatch(
|
||||||
|
self, task: HandleTask, handler: type[BaseHandler], delay: float
|
||||||
|
) -> TaskResult | TaskFailure | None:
|
||||||
"""
|
"""
|
||||||
Dispatch a task after a random delay to avoid rate limiting.
|
Dispatch a task after a random delay to avoid rate limiting.
|
||||||
|
|
||||||
|
|
@ -215,7 +218,7 @@ class TaskHandle:
|
||||||
exc_info=(type(exc), exc, exc.__traceback__),
|
exc_info=(type(exc), exc, exc.__traceback__),
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _find_handler(self, task: HandleTask) -> type | None:
|
async def _find_handler(self, task: HandleTask) -> type[BaseHandler] | None:
|
||||||
for cls in self._handlers:
|
for cls in self._handlers:
|
||||||
try:
|
try:
|
||||||
if await Services.get_instance().handle_async(handler=cls.can_handle, task=task):
|
if await Services.get_instance().handle_async(handler=cls.can_handle, task=task):
|
||||||
|
|
@ -239,9 +242,10 @@ class TaskHandle:
|
||||||
async def dispatch(
|
async def dispatch(
|
||||||
self,
|
self,
|
||||||
task: HandleTask,
|
task: HandleTask,
|
||||||
handler: type | None = None,
|
handler: type[BaseHandler] | None = None,
|
||||||
**kwargs, # noqa: ARG002
|
**kwargs,
|
||||||
) -> TaskResult | TaskFailure | None:
|
) -> TaskResult | TaskFailure | None:
|
||||||
|
_ = kwargs
|
||||||
"""
|
"""
|
||||||
Dispatch a task to the appropriate handler.
|
Dispatch a task to the appropriate handler.
|
||||||
|
|
||||||
|
|
@ -582,11 +586,11 @@ class TaskHandle:
|
||||||
|
|
||||||
return TaskResult(items=list(extraction.items), metadata=combined_metadata)
|
return TaskResult(items=list(extraction.items), metadata=combined_metadata)
|
||||||
|
|
||||||
def _discover(self) -> list[type]:
|
def _discover(self) -> list[type[BaseHandler]]:
|
||||||
"""Discover all available task handlers."""
|
"""Discover all available task handlers."""
|
||||||
import app.features.tasks.definitions.handlers as handlers_pkg
|
import app.features.tasks.definitions.handlers as handlers_pkg
|
||||||
|
|
||||||
handlers: list[type] = []
|
handlers: list[type[BaseHandler]] = []
|
||||||
|
|
||||||
for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__):
|
for _, module_name, _ in pkgutil.iter_modules(handlers_pkg.__path__):
|
||||||
if module_name.startswith("_"):
|
if module_name.startswith("_"):
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ from app.features.tasks.definitions.results import TaskFailure, TaskResult
|
||||||
from app.features.tasks.definitions.schemas import (
|
from app.features.tasks.definitions.schemas import (
|
||||||
Definition,
|
Definition,
|
||||||
EngineConfig,
|
EngineConfig,
|
||||||
|
Parse,
|
||||||
RequestConfig,
|
RequestConfig,
|
||||||
ResponseConfig,
|
ResponseConfig,
|
||||||
TaskDefinition,
|
TaskDefinition,
|
||||||
|
|
@ -30,10 +31,12 @@ def test_build_def_payload():
|
||||||
created_at=datetime.now(),
|
created_at=datetime.now(),
|
||||||
updated_at=datetime.now(),
|
updated_at=datetime.now(),
|
||||||
definition=Definition(
|
definition=Definition(
|
||||||
parse={
|
parse=Parse.model_validate(
|
||||||
|
{
|
||||||
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
|
"link": {"type": "css", "expression": ".article a.link::attr(href)"},
|
||||||
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
|
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
|
||||||
},
|
}
|
||||||
|
),
|
||||||
engine=EngineConfig(),
|
engine=EngineConfig(),
|
||||||
request=RequestConfig(),
|
request=RequestConfig(),
|
||||||
response=ResponseConfig(),
|
response=ResponseConfig(),
|
||||||
|
|
@ -54,7 +57,8 @@ def test_build_def_container():
|
||||||
created_at=datetime.now(),
|
created_at=datetime.now(),
|
||||||
updated_at=datetime.now(),
|
updated_at=datetime.now(),
|
||||||
definition=Definition(
|
definition=Definition(
|
||||||
parse={
|
parse=Parse.model_validate(
|
||||||
|
{
|
||||||
"items": {
|
"items": {
|
||||||
"selector": ".cards .card",
|
"selector": ".cards .card",
|
||||||
"fields": {
|
"fields": {
|
||||||
|
|
@ -62,7 +66,8 @@ def test_build_def_container():
|
||||||
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
|
"title": {"type": "css", "expression": ".card-header a", "attribute": "text"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
|
),
|
||||||
engine=EngineConfig(),
|
engine=EngineConfig(),
|
||||||
request=RequestConfig(),
|
request=RequestConfig(),
|
||||||
response=ResponseConfig(),
|
response=ResponseConfig(),
|
||||||
|
|
@ -82,7 +87,8 @@ def test_build_def_json():
|
||||||
created_at=datetime.now(),
|
created_at=datetime.now(),
|
||||||
updated_at=datetime.now(),
|
updated_at=datetime.now(),
|
||||||
definition=Definition(
|
definition=Definition(
|
||||||
parse={
|
parse=Parse.model_validate(
|
||||||
|
{
|
||||||
"items": {
|
"items": {
|
||||||
"type": "jsonpath",
|
"type": "jsonpath",
|
||||||
"selector": "items",
|
"selector": "items",
|
||||||
|
|
@ -91,7 +97,8 @@ def test_build_def_json():
|
||||||
"title": {"type": "jsonpath", "expression": "title"},
|
"title": {"type": "jsonpath", "expression": "title"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
|
),
|
||||||
engine=EngineConfig(),
|
engine=EngineConfig(),
|
||||||
request=RequestConfig(),
|
request=RequestConfig(),
|
||||||
response=ResponseConfig(type="json"),
|
response=ResponseConfig(type="json"),
|
||||||
|
|
@ -112,11 +119,13 @@ def test_parse_items_basic():
|
||||||
created_at=datetime.now(),
|
created_at=datetime.now(),
|
||||||
updated_at=datetime.now(),
|
updated_at=datetime.now(),
|
||||||
definition=Definition(
|
definition=Definition(
|
||||||
parse={
|
parse=Parse.model_validate(
|
||||||
|
{
|
||||||
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
|
"link": {"type": "css", "expression": ".article a.link::attr(href)", "attribute": None},
|
||||||
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
|
"title": {"type": "css", "expression": ".article .title", "attribute": "text"},
|
||||||
"id": {"type": "css", "expression": ".article", "attribute": "data-id"},
|
"id": {"type": "css", "expression": ".article", "attribute": "data-id"},
|
||||||
},
|
}
|
||||||
|
),
|
||||||
engine=EngineConfig(),
|
engine=EngineConfig(),
|
||||||
request=RequestConfig(),
|
request=RequestConfig(),
|
||||||
response=ResponseConfig(),
|
response=ResponseConfig(),
|
||||||
|
|
@ -154,7 +163,8 @@ def test_parse_items_cards():
|
||||||
created_at=datetime.now(),
|
created_at=datetime.now(),
|
||||||
updated_at=datetime.now(),
|
updated_at=datetime.now(),
|
||||||
definition=Definition(
|
definition=Definition(
|
||||||
parse={
|
parse=Parse.model_validate(
|
||||||
|
{
|
||||||
"items": {
|
"items": {
|
||||||
"type": "css",
|
"type": "css",
|
||||||
"selector": ".columns .card",
|
"selector": ".columns .card",
|
||||||
|
|
@ -181,7 +191,8 @@ def test_parse_items_cards():
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
|
),
|
||||||
engine=EngineConfig(),
|
engine=EngineConfig(),
|
||||||
request=RequestConfig(),
|
request=RequestConfig(),
|
||||||
response=ResponseConfig(),
|
response=ResponseConfig(),
|
||||||
|
|
@ -249,7 +260,8 @@ def test_parse_items_json():
|
||||||
created_at=datetime.now(),
|
created_at=datetime.now(),
|
||||||
updated_at=datetime.now(),
|
updated_at=datetime.now(),
|
||||||
definition=Definition(
|
definition=Definition(
|
||||||
parse={
|
parse=Parse.model_validate(
|
||||||
|
{
|
||||||
"items": {
|
"items": {
|
||||||
"type": "jsonpath",
|
"type": "jsonpath",
|
||||||
"selector": "entries",
|
"selector": "entries",
|
||||||
|
|
@ -259,7 +271,8 @@ def test_parse_items_json():
|
||||||
"id": {"type": "jsonpath", "expression": "id"},
|
"id": {"type": "jsonpath", "expression": "id"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
|
),
|
||||||
engine=EngineConfig(),
|
engine=EngineConfig(),
|
||||||
request=RequestConfig(),
|
request=RequestConfig(),
|
||||||
response=ResponseConfig(type="json"),
|
response=ResponseConfig(type="json"),
|
||||||
|
|
@ -297,7 +310,8 @@ async def test_generic_task_handler_inspect(monkeypatch):
|
||||||
created_at=datetime.now(),
|
created_at=datetime.now(),
|
||||||
updated_at=datetime.now(),
|
updated_at=datetime.now(),
|
||||||
definition=Definition(
|
definition=Definition(
|
||||||
parse={
|
parse=Parse.model_validate(
|
||||||
|
{
|
||||||
"items": {
|
"items": {
|
||||||
"type": "jsonpath",
|
"type": "jsonpath",
|
||||||
"selector": "items",
|
"selector": "items",
|
||||||
|
|
@ -306,7 +320,8 @@ async def test_generic_task_handler_inspect(monkeypatch):
|
||||||
"title": {"type": "jsonpath", "expression": "title"},
|
"title": {"type": "jsonpath", "expression": "title"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
|
),
|
||||||
engine=EngineConfig(),
|
engine=EngineConfig(),
|
||||||
request=RequestConfig(),
|
request=RequestConfig(),
|
||||||
response=ResponseConfig(type="json"),
|
response=ResponseConfig(type="json"),
|
||||||
|
|
@ -351,7 +366,8 @@ def test_parse_items_json_list():
|
||||||
created_at=datetime.now(),
|
created_at=datetime.now(),
|
||||||
updated_at=datetime.now(),
|
updated_at=datetime.now(),
|
||||||
definition=Definition(
|
definition=Definition(
|
||||||
parse={
|
parse=Parse.model_validate(
|
||||||
|
{
|
||||||
"items": {
|
"items": {
|
||||||
"type": "jsonpath",
|
"type": "jsonpath",
|
||||||
"selector": "[]",
|
"selector": "[]",
|
||||||
|
|
@ -360,7 +376,8 @@ def test_parse_items_json_list():
|
||||||
"title": {"type": "jsonpath", "expression": "title"},
|
"title": {"type": "jsonpath", "expression": "title"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
|
),
|
||||||
engine=EngineConfig(),
|
engine=EngineConfig(),
|
||||||
request=RequestConfig(),
|
request=RequestConfig(),
|
||||||
response=ResponseConfig(type="json"),
|
response=ResponseConfig(type="json"),
|
||||||
|
|
|
||||||
|
|
@ -69,7 +69,7 @@ class TestTaskDefinitionsRepository:
|
||||||
await repo.create(_sample_definition("Alpha", priority=2))
|
await repo.create(_sample_definition("Alpha", priority=2))
|
||||||
await repo.create(_sample_definition("Beta", priority=1))
|
await repo.create(_sample_definition("Beta", priority=1))
|
||||||
|
|
||||||
items = await repo.list()
|
items = await repo.all()
|
||||||
assert len(items) == 2, "Should return two task definitions"
|
assert len(items) == 2, "Should return two task definitions"
|
||||||
assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name"
|
assert [item.name for item in items] == ["Beta", "Alpha"], "Should sort by priority then name"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,17 @@
|
||||||
from typing import Any
|
from typing import Any, Literal, overload
|
||||||
|
|
||||||
from app.features.tasks.definitions.models import TaskDefinitionModel
|
from app.features.tasks.definitions.models import TaskDefinitionModel
|
||||||
from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary
|
from app.features.tasks.definitions.schemas import Definition, TaskDefinition, TaskDefinitionSummary
|
||||||
|
|
||||||
|
|
||||||
|
@overload
|
||||||
|
def model_to_schema(model: TaskDefinitionModel, summary: Literal[False] = False) -> TaskDefinition: ...
|
||||||
|
|
||||||
|
|
||||||
|
@overload
|
||||||
|
def model_to_schema(model: TaskDefinitionModel, summary: Literal[True]) -> TaskDefinitionSummary: ...
|
||||||
|
|
||||||
|
|
||||||
def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary:
|
def model_to_schema(model: TaskDefinitionModel, summary: bool = False) -> TaskDefinition | TaskDefinitionSummary:
|
||||||
"""
|
"""
|
||||||
Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema.
|
Convert a TaskDefinitionModel to a TaskDefinition or TaskDefinitionSummary schema.
|
||||||
|
|
|
||||||
|
|
@ -10,20 +10,39 @@ from app.library.log import get_logger
|
||||||
from app.library.Singleton import Singleton
|
from app.library.Singleton import Singleton
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import Callable
|
||||||
|
from contextlib import AbstractAsyncContextManager
|
||||||
|
|
||||||
from sqlalchemy.engine.result import Result
|
from sqlalchemy.engine.result import Result
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy.sql.elements import ColumnElement
|
from sqlalchemy.sql.elements import ColumnElement
|
||||||
from sqlalchemy.sql.selectable import Select
|
from sqlalchemy.sql.selectable import Select
|
||||||
|
|
||||||
|
SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
|
||||||
|
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
|
|
||||||
|
|
||||||
|
def _model_from_payload(payload: dict[str, Any]) -> TaskModel:
|
||||||
|
model = TaskModel()
|
||||||
|
for key, value in payload.items():
|
||||||
|
if not hasattr(model, key):
|
||||||
|
msg = f"'{key}' is an invalid keyword argument for TaskModel"
|
||||||
|
raise TypeError(msg)
|
||||||
|
setattr(model, key, value)
|
||||||
|
return model
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_model(payload: TaskModel | dict[str, Any]) -> TaskModel:
|
||||||
|
if isinstance(payload, TaskModel):
|
||||||
|
return payload
|
||||||
|
return _model_from_payload(payload)
|
||||||
|
|
||||||
|
|
||||||
class TasksRepository(metaclass=Singleton):
|
class TasksRepository(metaclass=Singleton):
|
||||||
def __init__(self, session: AsyncGenerator[AsyncSession] | None = None) -> None:
|
def __init__(self, session: SessionFactory | None = None) -> None:
|
||||||
self._migrated = False
|
self._migrated = False
|
||||||
self.session = session or get_session
|
self.session: SessionFactory = session or get_session
|
||||||
|
|
||||||
async def run_migrations(self) -> None:
|
async def run_migrations(self) -> None:
|
||||||
if self._migrated:
|
if self._migrated:
|
||||||
|
|
@ -38,7 +57,7 @@ class TasksRepository(metaclass=Singleton):
|
||||||
def get_instance() -> TasksRepository:
|
def get_instance() -> TasksRepository:
|
||||||
return TasksRepository()
|
return TasksRepository()
|
||||||
|
|
||||||
async def list(self) -> list[TaskModel]:
|
async def all(self) -> list[TaskModel]:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc()))
|
result: Result[tuple[TaskModel]] = await session.execute(select(TaskModel).order_by(TaskModel.name.asc()))
|
||||||
return list(result.scalars().all())
|
return list(result.scalars().all())
|
||||||
|
|
@ -90,7 +109,7 @@ class TasksRepository(metaclass=Singleton):
|
||||||
"""Get all enabled tasks."""
|
"""Get all enabled tasks."""
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
result: Result[tuple[TaskModel]] = await session.execute(
|
result: Result[tuple[TaskModel]] = await session.execute(
|
||||||
select(TaskModel).where(TaskModel.enabled == True).order_by(TaskModel.name.asc()) # noqa: E712
|
select(TaskModel).where(TaskModel.enabled).order_by(TaskModel.name.asc())
|
||||||
)
|
)
|
||||||
return list(result.scalars().all())
|
return list(result.scalars().all())
|
||||||
|
|
||||||
|
|
@ -102,11 +121,11 @@ class TasksRepository(metaclass=Singleton):
|
||||||
)
|
)
|
||||||
return list(result.scalars().all())
|
return list(result.scalars().all())
|
||||||
|
|
||||||
async def create(self, payload: TaskModel | dict) -> TaskModel:
|
async def create(self, payload: TaskModel | dict[str, Any]) -> TaskModel:
|
||||||
async with self.session() as session:
|
async with self.session() as session:
|
||||||
model: TaskModel = TaskModel(**payload) if isinstance(payload, dict) else payload
|
model = _coerce_model(payload)
|
||||||
if model.id is not None:
|
if model.id is not None:
|
||||||
model.id = None
|
model.id = None # ty: ignore
|
||||||
|
|
||||||
if await self.get_by_name(name=model.name) is not None:
|
if await self.get_by_name(name=model.name) is not None:
|
||||||
msg: str = f"Task with name '{model.name}' already exists."
|
msg: str = f"Task with name '{model.name}' already exists."
|
||||||
|
|
|
||||||
|
|
@ -705,8 +705,9 @@ async def task_metadata(request: Request, repo: TasksRepository, config: Config,
|
||||||
xml_content += f' <uniqueid type="{NFOMakerPP._escape_text(info.get("id_type"))}" default="true">{NFOMakerPP._escape_text(info.get("id"))}</uniqueid>\n'
|
xml_content += f' <uniqueid type="{NFOMakerPP._escape_text(info.get("id_type"))}" default="true">{NFOMakerPP._escape_text(info.get("id"))}</uniqueid>\n'
|
||||||
if info.get("uploader"):
|
if info.get("uploader"):
|
||||||
xml_content += f" <studio>{NFOMakerPP._escape_text(info.get('uploader'))}</studio>\n"
|
xml_content += f" <studio>{NFOMakerPP._escape_text(info.get('uploader'))}</studio>\n"
|
||||||
if info.get("tags", []):
|
tags = info.get("tags", [])
|
||||||
for tag in info.get("tags", []):
|
if isinstance(tags, list):
|
||||||
|
for tag in tags:
|
||||||
xml_content += f" <tag>{NFOMakerPP._escape_text(tag)}</tag>\n"
|
xml_content += f" <tag>{NFOMakerPP._escape_text(tag)}</tag>\n"
|
||||||
if info.get("year"):
|
if info.get("year"):
|
||||||
xml_content += f" <year>{info.get('year')}</year>\n"
|
xml_content += f" <year>{info.get('year')}</year>\n"
|
||||||
|
|
|
||||||
|
|
@ -51,7 +51,7 @@ class Tasks(metaclass=Singleton):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def _load_tasks(self) -> None:
|
async def _load_tasks(self) -> None:
|
||||||
tasks = await self._repo.list()
|
tasks = await self._repo.all()
|
||||||
|
|
||||||
for task in tasks:
|
for task in tasks:
|
||||||
if not task.timer or not task.enabled:
|
if not task.timer or not task.enabled:
|
||||||
|
|
@ -134,9 +134,11 @@ class Tasks(metaclass=Singleton):
|
||||||
task_id = task.id
|
task_id = task.id
|
||||||
task_name = task.name
|
task_name = task.name
|
||||||
try:
|
try:
|
||||||
if not (task := await self._repo.get(task_id)):
|
current_task = await self._repo.get(task_id)
|
||||||
|
if not current_task:
|
||||||
LOG.info("Task '%s' no longer exists.", task_name, extra={"task_id": task_id, "task_name": task_name})
|
LOG.info("Task '%s' no longer exists.", task_name, extra={"task_id": task_id, "task_name": task_name})
|
||||||
return
|
return
|
||||||
|
task = current_task
|
||||||
|
|
||||||
if not task.enabled:
|
if not task.enabled:
|
||||||
LOG.debug(
|
LOG.debug(
|
||||||
|
|
|
||||||
|
|
@ -260,7 +260,7 @@ class TestTasksRepository:
|
||||||
await repo.create({"name": "Alice", "url": "https://example.com"})
|
await repo.create({"name": "Alice", "url": "https://example.com"})
|
||||||
await repo.create({"name": "Bob", "url": "https://example.com"})
|
await repo.create({"name": "Bob", "url": "https://example.com"})
|
||||||
|
|
||||||
items = await repo.list()
|
items = await repo.all()
|
||||||
|
|
||||||
assert items[0].name == "Alice", "Should be alphabetically first"
|
assert items[0].name == "Alice", "Should be alphabetically first"
|
||||||
assert items[1].name == "Bob", "Should be alphabetically second"
|
assert items[1].name == "Bob", "Should be alphabetically second"
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
|
|
@ -41,13 +42,13 @@ def patch_get_info(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
|
||||||
|
|
||||||
def _json_request(method: str, path: str, payload: object, *, match_info: dict[str, str] | None = None) -> web.Request:
|
def _json_request(method: str, path: str, payload: object, *, match_info: dict[str, str] | None = None) -> web.Request:
|
||||||
request = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace())
|
mock_request: Any = make_mocked_request(method, path, match_info=match_info or {}, app=SimpleNamespace())
|
||||||
|
|
||||||
async def _json() -> object:
|
async def _json() -> object:
|
||||||
return payload
|
return payload
|
||||||
|
|
||||||
request.json = _json # type: ignore[attr-defined]
|
mock_request.json = _json
|
||||||
return request
|
return mock_request
|
||||||
|
|
||||||
|
|
||||||
class _Notify:
|
class _Notify:
|
||||||
|
|
@ -85,7 +86,7 @@ async def test_add_requires_timer_without_handler(repo) -> None:
|
||||||
|
|
||||||
assert response.status == web.HTTPBadRequest.status_code
|
assert response.status == web.HTTPBadRequest.status_code
|
||||||
assert b"requires a timer" in response.body
|
assert b"requires a timer" in response.body
|
||||||
assert await repo.list() == []
|
assert await repo.all() == []
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
@ -109,7 +110,7 @@ async def test_add_all_or_nothing(repo) -> None:
|
||||||
|
|
||||||
assert response.status == web.HTTPBadRequest.status_code
|
assert response.status == web.HTTPBadRequest.status_code
|
||||||
assert b"requires a timer" in response.body
|
assert b"requires a timer" in response.body
|
||||||
assert await repo.list() == []
|
assert await repo.all() == []
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
@ -123,7 +124,7 @@ async def test_add_allows_handler_only(repo) -> None:
|
||||||
response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=True))
|
response = await router.tasks_add(request, repo, Encoder(), _Notify(), _Handler(matched=True))
|
||||||
|
|
||||||
assert response.status == web.HTTPOk.status_code
|
assert response.status == web.HTTPOk.status_code
|
||||||
items = await repo.list()
|
items = await repo.all()
|
||||||
assert len(items) == 1
|
assert len(items) == 1
|
||||||
assert items[0].name == "Handler Only"
|
assert items[0].name == "Handler Only"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -120,7 +120,7 @@ class ExtractorPool(metaclass=Singleton):
|
||||||
def attach(self, app: web.Application) -> None:
|
def attach(self, app: web.Application) -> None:
|
||||||
"""Attach the extractor pool to the application (no-op for now)."""
|
"""Attach the extractor pool to the application (no-op for now)."""
|
||||||
app.on_shutdown.append(self.shutdown)
|
app.on_shutdown.append(self.shutdown)
|
||||||
Services.get_instance().add(__class__.__name__, self)
|
Services.get_instance().add(type(self).__name__, self)
|
||||||
|
|
||||||
def _ensure_initialized(self, config: ExtractorConfig) -> None:
|
def _ensure_initialized(self, config: ExtractorConfig) -> None:
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -286,7 +286,7 @@ async def convert(request: Request) -> Response:
|
||||||
return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code)
|
return web.json_response(data={"error": "args param is required."}, status=web.HTTPBadRequest.status_code)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = {"opts": {}, "output_template": None, "download_path": None}
|
response: dict[str, Any] = {"opts": {}, "output_template": None, "download_path": None}
|
||||||
|
|
||||||
data = arg_converter(args, dumps=True)
|
data = arg_converter(args, dumps=True)
|
||||||
|
|
||||||
|
|
@ -384,6 +384,8 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
|
||||||
|
|
||||||
if cache.has(key) and not request.query.get("force", False):
|
if cache.has(key) and not request.query.get("force", False):
|
||||||
data: Any | None = cache.get(key)
|
data: Any | None = cache.get(key)
|
||||||
|
if data is None:
|
||||||
|
data = {}
|
||||||
data["_cached"] = {
|
data["_cached"] = {
|
||||||
"status": "hit",
|
"status": "hit",
|
||||||
"preset": preset,
|
"preset": preset,
|
||||||
|
|
@ -393,7 +395,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
|
||||||
"ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(),
|
"ttl_left": data.get("_cached", {}).get("expires", time.time() + 300) - time.time(),
|
||||||
"expires": data.get("_cached", {}).get("expires", time.time() + 300),
|
"expires": data.get("_cached", {}).get("expires", time.time() + 300),
|
||||||
}
|
}
|
||||||
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
|
return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
|
||||||
|
|
||||||
ytdlp_opts: dict = opts.get_all()
|
ytdlp_opts: dict = opts.get_all()
|
||||||
|
|
||||||
|
|
@ -450,7 +452,7 @@ async def get_info(request: Request, cache: Cache, config: Config) -> Response:
|
||||||
|
|
||||||
cache.set(key=key, value=data, ttl=300)
|
cache.set(key=key, value=data, ttl=300)
|
||||||
|
|
||||||
return web.json_response(body=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
|
return web.json_response(text=json.dumps(data, indent=4, default=str), status=web.HTTPOk.status_code)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
LOG.exception(
|
LOG.exception(
|
||||||
"Failed to get video info for '%s': %s.",
|
"Failed to get video info for '%s': %s.",
|
||||||
|
|
@ -486,7 +488,7 @@ async def get_options() -> Response:
|
||||||
"""
|
"""
|
||||||
from app.features.ytdlp.ytdlp import ytdlp_options
|
from app.features.ytdlp.ytdlp import ytdlp_options
|
||||||
|
|
||||||
return web.json_response(body=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
|
return web.json_response(text=json.dumps(ytdlp_options(), indent=4, default=str), status=web.HTTPOk.status_code)
|
||||||
|
|
||||||
|
|
||||||
@route("POST", "api/yt-dlp/archive_id/", "get_archive_ids")
|
@route("POST", "api/yt-dlp/archive_id/", "get_archive_ids")
|
||||||
|
|
@ -508,7 +510,11 @@ async def get_archive_ids(request: Request, config: Config) -> Response:
|
||||||
response = []
|
response = []
|
||||||
|
|
||||||
for i, url in enumerate(data):
|
for i, url in enumerate(data):
|
||||||
dct = {"index": i, "url": url}
|
dct: dict[str, Any] = {"index": i, "url": url}
|
||||||
|
if not isinstance(url, str):
|
||||||
|
dct.update({"id": None, "ie_key": None, "archive_id": None, "error": "URL must be a string."})
|
||||||
|
response.append(dct)
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
|
await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
|
||||||
dct.update(get_archive_id(url))
|
dct.update(get_archive_id(url))
|
||||||
|
|
|
||||||
|
|
@ -122,6 +122,7 @@ class TestExtractInfo:
|
||||||
{}, "https://example.com/video", debug=True, capture_logs=logging.WARNING
|
{}, "https://example.com/video", debug=True, capture_logs=logging.WARNING
|
||||||
)
|
)
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
assert result["id"] == "test123"
|
assert result["id"] == "test123"
|
||||||
assert logs == ["[generic_browser] Browser fallback warning"]
|
assert logs == ["[generic_browser] Browser fallback warning"]
|
||||||
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
|
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
|
||||||
|
|
@ -155,6 +156,7 @@ class TestExtractInfo:
|
||||||
{}, "https://example.com/video", debug=False, capture_logs=logging.WARNING
|
{}, "https://example.com/video", debug=False, capture_logs=logging.WARNING
|
||||||
)
|
)
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
assert result["id"] == "test123"
|
assert result["id"] == "test123"
|
||||||
assert logs == ["[generic_browser] Browser fallback warning"]
|
assert logs == ["[generic_browser] Browser fallback warning"]
|
||||||
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
|
assert (logging.INFO, "[generic_browser] Using remote browser for https://example.com/video") in seen
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import importlib
|
import importlib
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
from unittest.mock import MagicMock, Mock, patch
|
from unittest.mock import MagicMock, Mock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
@ -219,7 +220,8 @@ class TestYTDLP:
|
||||||
"""Test _delete_downloaded_files skips cleanup when _interrupted is True."""
|
"""Test _delete_downloaded_files skips cleanup when _interrupted is True."""
|
||||||
with patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__", return_value=None):
|
with patch("app.features.ytdlp.ytdlp.yt_dlp.YoutubeDL.__init__", return_value=None):
|
||||||
ytdlp = YTDLP(params={})
|
ytdlp = YTDLP(params={})
|
||||||
ytdlp.to_screen = Mock()
|
mock_obj: Any = ytdlp
|
||||||
|
mock_obj.to_screen = Mock()
|
||||||
|
|
||||||
# Set interrupted flag
|
# Set interrupted flag
|
||||||
ytdlp._interrupted = True
|
ytdlp._interrupted = True
|
||||||
|
|
@ -229,7 +231,7 @@ class TestYTDLP:
|
||||||
# Should not call super method
|
# Should not call super method
|
||||||
mock_super_delete.assert_not_called()
|
mock_super_delete.assert_not_called()
|
||||||
# Should show message
|
# Should show message
|
||||||
ytdlp.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
|
mock_obj.to_screen.assert_called_once_with("[info] Cancelled — skipping temp cleanup.")
|
||||||
# Should return None
|
# Should return None
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
|
|
@ -349,6 +351,14 @@ class TestYTDLP:
|
||||||
assert len(result) == 8
|
assert len(result) == 8
|
||||||
assert result.isalnum()
|
assert result.isalnum()
|
||||||
|
|
||||||
|
def test_exec_init(self) -> None:
|
||||||
|
YTDLP(
|
||||||
|
params={
|
||||||
|
"compat_opts": set(),
|
||||||
|
"postprocessors": [{"key": "Exec", "exec_cmd": "echo %(title)q"}],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
def test_outtmpl_reuses_value(self) -> None:
|
def test_outtmpl_reuses_value(self) -> None:
|
||||||
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
|
ytdlp = YTDLP(params={"outtmpl": {"default": "%(title)s"}})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -776,7 +776,7 @@ class TestYTDLPCli:
|
||||||
from app.features.ytdlp.ytdlp_opts import YTDLPCli
|
from app.features.ytdlp.ytdlp_opts import YTDLPCli
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="Expected Item instance"):
|
with pytest.raises(ValueError, match="Expected Item instance"):
|
||||||
YTDLPCli(item="not an item") # type: ignore
|
YTDLPCli(item="not an item")
|
||||||
|
|
||||||
@patch("app.features.presets.service.Presets")
|
@patch("app.features.presets.service.Presets")
|
||||||
@patch("app.features.ytdlp.ytdlp_opts.Config")
|
@patch("app.features.ytdlp.ytdlp_opts.Config")
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,8 @@ class TestLogWrapper:
|
||||||
def test_add_target_type_validation(self) -> None:
|
def test_add_target_type_validation(self) -> None:
|
||||||
lw = LogWrapper()
|
lw = LogWrapper()
|
||||||
with pytest.raises(TypeError, match=r"Target must be a logging\.Logger instance or a callable"):
|
with pytest.raises(TypeError, match=r"Target must be a logging\.Logger instance or a callable"):
|
||||||
lw.add_target(123) # type: ignore[arg-type]
|
bad: Any = 123
|
||||||
|
lw.add_target(bad)
|
||||||
|
|
||||||
def test_add_target_names(self) -> None:
|
def test_add_target_names(self) -> None:
|
||||||
lw = LogWrapper()
|
lw = LogWrapper()
|
||||||
|
|
@ -128,7 +129,7 @@ class TestExtractYtdlpLogs:
|
||||||
def test_extract_ytdlp_logs_with_filters(self):
|
def test_extract_ytdlp_logs_with_filters(self):
|
||||||
"""Test log extraction with filters."""
|
"""Test log extraction with filters."""
|
||||||
logs = ["INFO: Downloading", "ERROR: Failed", "WARNING: Deprecated"]
|
logs = ["INFO: Downloading", "ERROR: Failed", "WARNING: Deprecated"]
|
||||||
filters = [re.compile(r"ERROR")]
|
filters: list[str | re.Pattern[str]] = [re.compile(r"ERROR")]
|
||||||
result = extract_ytdlp_logs(logs, filters)
|
result = extract_ytdlp_logs(logs, filters)
|
||||||
assert result == ["ERROR: Failed"]
|
assert result == ["ERROR: Failed"]
|
||||||
|
|
||||||
|
|
@ -312,9 +313,12 @@ class TestGetThumbnail:
|
||||||
def test_non_list(self):
|
def test_non_list(self):
|
||||||
"""Test that None is returned for non-list input."""
|
"""Test that None is returned for non-list input."""
|
||||||
|
|
||||||
assert get_thumbnail(None) is None
|
bad_none: Any = None
|
||||||
assert get_thumbnail("not a list") is None
|
bad_str: Any = "not a list"
|
||||||
assert get_thumbnail({"not": "list"}) is None
|
bad_dict: Any = {"not": "list"}
|
||||||
|
assert get_thumbnail(bad_none) is None
|
||||||
|
assert get_thumbnail(bad_str) is None
|
||||||
|
assert get_thumbnail(bad_dict) is None
|
||||||
|
|
||||||
def test_thumbnail_preference(self):
|
def test_thumbnail_preference(self):
|
||||||
"""Test that the thumbnail with highest preference is returned."""
|
"""Test that the thumbnail with highest preference is returned."""
|
||||||
|
|
@ -349,6 +353,7 @@ class TestGetThumbnail:
|
||||||
]
|
]
|
||||||
|
|
||||||
result = get_thumbnail(thumbnails)
|
result = get_thumbnail(thumbnails)
|
||||||
|
assert result is not None
|
||||||
assert result["url"] == "with_pref.jpg"
|
assert result["url"] == "with_pref.jpg"
|
||||||
|
|
||||||
def test_all_equal(self):
|
def test_all_equal(self):
|
||||||
|
|
@ -365,12 +370,15 @@ class TestGetThumbnail:
|
||||||
class TestGetExtras:
|
class TestGetExtras:
|
||||||
def test_none(self):
|
def test_none(self):
|
||||||
"""Test that empty dict is returned for None input."""
|
"""Test that empty dict is returned for None input."""
|
||||||
assert get_extras(None) == {}
|
bad: Any = None
|
||||||
|
assert get_extras(bad) == {}
|
||||||
|
|
||||||
def test_non_dict(self):
|
def test_non_dict(self):
|
||||||
"""Test that empty dict is returned for non-dict input."""
|
"""Test that empty dict is returned for non-dict input."""
|
||||||
assert get_extras("not a dict") == {}
|
bad_str: Any = "not a dict"
|
||||||
assert get_extras([]) == {}
|
bad_list: Any = []
|
||||||
|
assert get_extras(bad_str) == {}
|
||||||
|
assert get_extras(bad_list) == {}
|
||||||
|
|
||||||
def test_extracts_video_information(self):
|
def test_extracts_video_information(self):
|
||||||
"""Test extracting information from a video entry."""
|
"""Test extracting information from a video entry."""
|
||||||
|
|
|
||||||
|
|
@ -101,12 +101,12 @@ class LogWrapper:
|
||||||
name (str|None): The name of the logging target. Defaults to None.
|
name (str|None): The name of the logging target. Defaults to None.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
if not isinstance(target, logging.Logger | Callable):
|
if not isinstance(target, logging.Logger) and not callable(target):
|
||||||
msg = "Target must be a logging.Logger instance or a callable."
|
msg = "Target must be a logging.Logger instance or a callable."
|
||||||
raise TypeError(msg)
|
raise TypeError(msg)
|
||||||
|
|
||||||
if name is None:
|
if name is None:
|
||||||
name = target.name if isinstance(target, logging.Logger) else target.__name__
|
name = target.name if isinstance(target, logging.Logger) else getattr(target, "__name__", "callable")
|
||||||
|
|
||||||
self.targets.append(
|
self.targets.append(
|
||||||
LogTarget(
|
LogTarget(
|
||||||
|
|
@ -131,11 +131,11 @@ class LogWrapper:
|
||||||
if level < target.level:
|
if level < target.level:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if target.logger:
|
if isinstance(target.target, logging.Logger):
|
||||||
log_kwargs = {**kwargs}
|
log_kwargs: dict[str, Any] = {**kwargs}
|
||||||
log_kwargs.setdefault("stacklevel", 3)
|
log_kwargs.setdefault("stacklevel", 3)
|
||||||
target.target.log(level, msg, *args, **log_kwargs)
|
target.target.log(level, msg, *args, **log_kwargs)
|
||||||
else:
|
elif callable(target.target):
|
||||||
target.target(level, msg, *args, **kwargs)
|
target.target(level, msg, *args, **kwargs)
|
||||||
|
|
||||||
def debug(self, msg, *args, **kwargs):
|
def debug(self, msg, *args, **kwargs):
|
||||||
|
|
@ -179,13 +179,17 @@ def arg_converter(
|
||||||
|
|
||||||
create_parser = yt_dlp.options.create_parser
|
create_parser = yt_dlp.options.create_parser
|
||||||
|
|
||||||
def _default_opts(args: str):
|
def _default_opts(args: str | list[str]):
|
||||||
patched_parser = create_parser()
|
patched_parser = create_parser()
|
||||||
|
|
||||||
|
def patched_create_parser():
|
||||||
|
return patched_parser
|
||||||
|
|
||||||
try:
|
try:
|
||||||
yt_dlp.options.create_parser = lambda: patched_parser
|
yt_dlp.options.__dict__["create_parser"] = patched_create_parser
|
||||||
return yt_dlp.parse_options(args)
|
return yt_dlp.parse_options(args)
|
||||||
finally:
|
finally:
|
||||||
yt_dlp.options.create_parser = create_parser
|
yt_dlp.options.__dict__["create_parser"] = create_parser
|
||||||
|
|
||||||
apply_ytdlp_patches()
|
apply_ytdlp_patches()
|
||||||
|
|
||||||
|
|
@ -238,7 +242,7 @@ def arg_converter(
|
||||||
return diff
|
return diff
|
||||||
|
|
||||||
|
|
||||||
def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) -> list[str]:
|
def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern[str]] | None = None) -> list[str]:
|
||||||
"""
|
"""
|
||||||
Extract yt-dlp log lines matching built-in filters plus any extras.
|
Extract yt-dlp log lines matching built-in filters plus any extras.
|
||||||
|
|
||||||
|
|
@ -332,7 +336,7 @@ def get_ytdlp(params: dict | None = None) -> YTDLP:
|
||||||
return _DATA.YTDLP_INFO_CLS
|
return _DATA.YTDLP_INFO_CLS
|
||||||
|
|
||||||
|
|
||||||
def get_thumbnail(thumbnails: list) -> str | None:
|
def get_thumbnail(thumbnails: list) -> dict | None:
|
||||||
"""
|
"""
|
||||||
Extract thumbnail URL from a yt-dlp entry.
|
Extract thumbnail URL from a yt-dlp entry.
|
||||||
|
|
||||||
|
|
@ -340,7 +344,7 @@ def get_thumbnail(thumbnails: list) -> str | None:
|
||||||
thumbnails (list): The list of thumbnail dictionaries from yt-dlp entry.
|
thumbnails (list): The list of thumbnail dictionaries from yt-dlp entry.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str | None: The thumbnail URL if available, otherwise None.
|
dict | None: The best thumbnail dict if available, otherwise None.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
if not thumbnails or not isinstance(thumbnails, list):
|
if not thumbnails or not isinstance(thumbnails, list):
|
||||||
|
|
@ -381,7 +385,7 @@ def get_extras(entry: dict, kind: str = "video") -> dict:
|
||||||
extras[f"playlist_{property}"] = val
|
extras[f"playlist_{property}"] = val
|
||||||
|
|
||||||
if thumbnail := get_thumbnail(entry.get("thumbnails", [])):
|
if thumbnail := get_thumbnail(entry.get("thumbnails", [])):
|
||||||
extras["thumbnail"] = thumbnail.get("url")
|
extras["thumbnail"] = thumbnail.get("url") if isinstance(thumbnail, dict) else thumbnail
|
||||||
elif thumbnail := entry.get("thumbnail"):
|
elif thumbnail := entry.get("thumbnail"):
|
||||||
extras["thumbnail"] = thumbnail
|
extras["thumbnail"] = thumbnail
|
||||||
|
|
||||||
|
|
@ -439,7 +443,7 @@ def get_archive_id(url: str) -> dict[str, str | None]:
|
||||||
}
|
}
|
||||||
|
|
||||||
"""
|
"""
|
||||||
idDict: dict[str, None] = {
|
idDict: dict[str, str | None] = {
|
||||||
"id": None,
|
"id": None,
|
||||||
"ie_key": None,
|
"ie_key": None,
|
||||||
"archive_id": None,
|
"archive_id": None,
|
||||||
|
|
|
||||||
|
|
@ -81,6 +81,9 @@ class YTDLP(yt_dlp.YoutubeDL):
|
||||||
except Exception:
|
except Exception:
|
||||||
patched_params = params
|
patched_params = params
|
||||||
|
|
||||||
|
self._ytptube_outtmpl_info: dict[str, Any] | None = None
|
||||||
|
self._ytptube_outtmpl_cache: dict[str, Any] = {}
|
||||||
|
|
||||||
super().__init__(params=patched_params, auto_init=auto_init)
|
super().__init__(params=patched_params, auto_init=auto_init)
|
||||||
|
|
||||||
# Restore param and replace upstream archive set with our proxy
|
# Restore param and replace upstream archive set with our proxy
|
||||||
|
|
@ -90,8 +93,6 @@ class YTDLP(yt_dlp.YoutubeDL):
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
self._ytptube_outtmpl_info: dict[str, Any] | None = None
|
|
||||||
self._ytptube_outtmpl_cache: dict[str, Any] = {}
|
|
||||||
self.archive = _ArchiveProxy(orig_file)
|
self.archive = _ArchiveProxy(orig_file)
|
||||||
|
|
||||||
def _delete_downloaded_files(self, *args, **kwargs) -> None:
|
def _delete_downloaded_files(self, *args, **kwargs) -> None:
|
||||||
|
|
@ -105,14 +106,14 @@ class YTDLP(yt_dlp.YoutubeDL):
|
||||||
self._ytptube_outtmpl_info = None
|
self._ytptube_outtmpl_info = None
|
||||||
self._ytptube_outtmpl_cache = {}
|
self._ytptube_outtmpl_cache = {}
|
||||||
|
|
||||||
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False):
|
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=False, *, _exec=False):
|
||||||
if self._ytptube_outtmpl_info is not info_dict:
|
if self._ytptube_outtmpl_info is not info_dict:
|
||||||
self._ytptube_outtmpl_info = info_dict
|
self._ytptube_outtmpl_info = info_dict
|
||||||
self._ytptube_outtmpl_cache = {}
|
self._ytptube_outtmpl_cache = {}
|
||||||
|
|
||||||
outtmpl, enriched = rewrite_outtmpl(outtmpl, info_dict, cache=self._ytptube_outtmpl_cache)
|
outtmpl, enriched = rewrite_outtmpl(outtmpl, info_dict, cache=self._ytptube_outtmpl_cache)
|
||||||
|
|
||||||
return super().prepare_outtmpl(outtmpl, enriched, sanitize=sanitize)
|
return super().prepare_outtmpl(outtmpl, enriched, sanitize=sanitize, _exec=_exec)
|
||||||
|
|
||||||
def process_info(self, info_dict):
|
def process_info(self, info_dict):
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -50,12 +50,12 @@ class ARGSMerger:
|
||||||
"""
|
"""
|
||||||
return str(self)
|
return str(self)
|
||||||
|
|
||||||
def as_dict(self) -> dict:
|
def as_dict(self) -> list[str]:
|
||||||
"""
|
"""
|
||||||
Get all the options as a dict.
|
Get all the options as a shell argument list.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
dict: The options as a dict
|
list[str]: The options as shell arguments.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
return shlex.split(shlex.join(self.args))
|
return shlex.split(shlex.join(self.args))
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,7 @@ class BackgroundWorker(metaclass=Singleton):
|
||||||
"The queue to hold the tasks."
|
"The queue to hold the tasks."
|
||||||
self.running = True
|
self.running = True
|
||||||
"Whether the background worker is running or not."
|
"Whether the background worker is running or not."
|
||||||
self.thread: threading.Thread = None
|
self.thread: threading.Thread | None = None
|
||||||
"The thread that runs the background worker."
|
"The thread that runs the background worker."
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
@ -49,6 +49,7 @@ class BackgroundWorker(metaclass=Singleton):
|
||||||
try:
|
try:
|
||||||
LOG.debug("Stopping background worker thread.")
|
LOG.debug("Stopping background worker thread.")
|
||||||
self.queue.put((CloseThread, (), {}))
|
self.queue.put((CloseThread, (), {}))
|
||||||
|
if self.thread:
|
||||||
self.thread.join(timeout=5)
|
self.thread.join(timeout=5)
|
||||||
LOG.debug("Background worker thread has been shut down.")
|
LOG.debug("Background worker thread has been shut down.")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
|
|
@ -166,6 +166,12 @@ class DataStore:
|
||||||
def items(self):
|
def items(self):
|
||||||
return self._dict.items()
|
return self._dict.items()
|
||||||
|
|
||||||
|
def __contains__(self, key: str) -> bool:
|
||||||
|
return key in self._dict
|
||||||
|
|
||||||
|
def __len__(self) -> int:
|
||||||
|
return len(self._dict)
|
||||||
|
|
||||||
async def put(self, value: Download, no_notify: bool = False) -> Download:
|
async def put(self, value: Download, no_notify: bool = False) -> Download:
|
||||||
_ = no_notify
|
_ = no_notify
|
||||||
self._dict[value.info._id] = value
|
self._dict[value.info._id] = value
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,7 @@ class Events:
|
||||||
|
|
||||||
ITEM_ADDED: str = "item_added"
|
ITEM_ADDED: str = "item_added"
|
||||||
ITEM_UPDATED: str = "item_updated"
|
ITEM_UPDATED: str = "item_updated"
|
||||||
|
ITEM_PROGRESS: str = "item_progress"
|
||||||
ITEM_COMPLETED: str = "item_completed"
|
ITEM_COMPLETED: str = "item_completed"
|
||||||
ITEM_CANCELLED: str = "item_cancelled"
|
ITEM_CANCELLED: str = "item_cancelled"
|
||||||
ITEM_DELETED: str = "item_deleted"
|
ITEM_DELETED: str = "item_deleted"
|
||||||
|
|
@ -86,6 +87,7 @@ class Events:
|
||||||
Events.LOG_SUCCESS,
|
Events.LOG_SUCCESS,
|
||||||
Events.ITEM_ADDED,
|
Events.ITEM_ADDED,
|
||||||
Events.ITEM_UPDATED,
|
Events.ITEM_UPDATED,
|
||||||
|
Events.ITEM_PROGRESS,
|
||||||
Events.ITEM_CANCELLED,
|
Events.ITEM_CANCELLED,
|
||||||
Events.ITEM_DELETED,
|
Events.ITEM_DELETED,
|
||||||
Events.ITEM_BULK_DELETED,
|
Events.ITEM_BULK_DELETED,
|
||||||
|
|
@ -103,7 +105,7 @@ class Events:
|
||||||
list: The list of debug events.
|
list: The list of debug events.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
return [Events.ITEM_UPDATED]
|
return [Events.ITEM_UPDATED, Events.ITEM_PROGRESS]
|
||||||
|
|
||||||
|
|
||||||
@dataclass(kw_only=True)
|
@dataclass(kw_only=True)
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,17 @@
|
||||||
import base64
|
import base64
|
||||||
import hmac
|
import hmac
|
||||||
import inspect
|
import inspect
|
||||||
from collections.abc import Awaitable
|
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any, cast
|
||||||
|
|
||||||
import anyio
|
import anyio
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
from aiohttp.web import Request, RequestHandler, Response
|
from aiohttp.typedefs import Handler, Middleware
|
||||||
|
from aiohttp.web import Request, Response
|
||||||
from aiohttp.web_log import AccessLogger
|
from aiohttp.web_log import AccessLogger
|
||||||
|
from aiohttp.web_request import BaseRequest
|
||||||
|
from aiohttp.web_response import StreamResponse
|
||||||
|
|
||||||
from app.library.log import get_logger
|
from app.library.log import get_logger
|
||||||
from app.library.Services import Services
|
from app.library.Services import Services
|
||||||
|
|
@ -24,12 +27,12 @@ LOG = get_logger("http")
|
||||||
|
|
||||||
|
|
||||||
class HttpAccessLogger(AccessLogger):
|
class HttpAccessLogger(AccessLogger):
|
||||||
def log(self, request: Request, response: Response, time: float) -> None:
|
def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
|
||||||
try:
|
try:
|
||||||
fmt_info = self._format_line(request, response, time)
|
fmt_info = self._format_line(request, response, time)
|
||||||
|
|
||||||
values: list[object] = []
|
values: list[object] = []
|
||||||
extra: dict[str, object] = {"elapsed_ms": round(time * 1000.0, 2)}
|
extra: dict[str, Any] = {"elapsed_ms": round(time * 1000.0, 2)}
|
||||||
for key, value in fmt_info:
|
for key, value in fmt_info:
|
||||||
values.append(value)
|
values.append(value)
|
||||||
|
|
||||||
|
|
@ -37,7 +40,7 @@ class HttpAccessLogger(AccessLogger):
|
||||||
extra[key] = value
|
extra[key] = value
|
||||||
else:
|
else:
|
||||||
parent, child = key
|
parent, child = key
|
||||||
group = extra.get(parent, {})
|
group: dict[str, Any] = extra.get(parent, {}) # type: ignore[assignment]
|
||||||
if not isinstance(group, dict):
|
if not isinstance(group, dict):
|
||||||
group = {}
|
group = {}
|
||||||
group[child] = value
|
group[child] = value
|
||||||
|
|
@ -177,7 +180,7 @@ class HttpAPI:
|
||||||
registered_options.append(route.path)
|
registered_options.append(route.path)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def basic_auth(username: str, password: str) -> Awaitable:
|
def basic_auth(username: str, password: str) -> Middleware:
|
||||||
"""
|
"""
|
||||||
Middleware to handle basic authentication.
|
Middleware to handle basic authentication.
|
||||||
|
|
||||||
|
|
@ -191,7 +194,7 @@ class HttpAPI:
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@web.middleware
|
@web.middleware
|
||||||
async def auth_handler(request: Request, handler: RequestHandler) -> Response:
|
async def auth_handler(request: Request, handler: Handler) -> StreamResponse:
|
||||||
# if OPTIONS request, skip auth
|
# if OPTIONS request, skip auth
|
||||||
if "OPTIONS" == request.method:
|
if "OPTIONS" == request.method:
|
||||||
return await handler(request)
|
return await handler(request)
|
||||||
|
|
@ -204,7 +207,7 @@ class HttpAPI:
|
||||||
auth_cookie = request.cookies.get("auth")
|
auth_cookie = request.cookies.get("auth")
|
||||||
if auth_cookie is not None:
|
if auth_cookie is not None:
|
||||||
try:
|
try:
|
||||||
data = decrypt_data(auth_cookie, key=Config.get_instance().secret_key)
|
data = decrypt_data(auth_cookie, key=cast("bytes", Config.get_instance().secret_key))
|
||||||
if data is not None:
|
if data is not None:
|
||||||
data = base64.b64encode(data.encode()).decode()
|
data = base64.b64encode(data.encode()).decode()
|
||||||
auth_header = f"Basic {data}"
|
auth_header = f"Basic {data}"
|
||||||
|
|
@ -250,7 +253,7 @@ class HttpAPI:
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
response: Response = await handler(request)
|
response: StreamResponse = await handler(request)
|
||||||
|
|
||||||
contentType: str | None = response.headers.get("content-type", None)
|
contentType: str | None = response.headers.get("content-type", None)
|
||||||
if contentType and not contentType.startswith("text/html"):
|
if contentType and not contentType.startswith("text/html"):
|
||||||
|
|
@ -261,7 +264,7 @@ class HttpAPI:
|
||||||
"auth",
|
"auth",
|
||||||
encrypt_data(
|
encrypt_data(
|
||||||
f"{user_name}:{user_password}",
|
f"{user_name}:{user_password}",
|
||||||
key=Config.get_instance().secret_key,
|
key=cast("bytes", Config.get_instance().secret_key),
|
||||||
),
|
),
|
||||||
max_age=60 * 60 * 24 * 7,
|
max_age=60 * 60 * 24 * 7,
|
||||||
expires=(datetime.now(UTC) + timedelta(days=7)).strftime("%a, %d %b %Y %H:%M:%S GMT"),
|
expires=(datetime.now(UTC) + timedelta(days=7)).strftime("%a, %d %b %Y %H:%M:%S GMT"),
|
||||||
|
|
@ -280,9 +283,9 @@ class HttpAPI:
|
||||||
return auth_handler
|
return auth_handler
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def middle_wares(app: web.Application, base_path: str, download_path: str) -> Awaitable:
|
def middle_wares(app: web.Application, base_path: str, download_path: str) -> Middleware:
|
||||||
@web.middleware
|
@web.middleware
|
||||||
async def middleware_handler(request: Request, handler: RequestHandler) -> Response:
|
async def middleware_handler(request: Request, handler: Handler) -> StreamResponse:
|
||||||
static_path = str(app.router["download_static"].url_for(filename=""))
|
static_path = str(app.router["download_static"].url_for(filename=""))
|
||||||
if request.path.startswith(static_path):
|
if request.path.startswith(static_path):
|
||||||
realFile, status = get_file(
|
realFile, status = get_file(
|
||||||
|
|
@ -301,12 +304,13 @@ class HttpAPI:
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
response: Response = await handler(request)
|
response: StreamResponse = await handler(request)
|
||||||
|
|
||||||
contentType: str = str(response.headers.get("content-type", ""))
|
contentType: str = str(response.headers.get("content-type", ""))
|
||||||
if contentType.startswith("text/html") and getattr(response, "_path", None):
|
if contentType.startswith("text/html") and getattr(response, "_path", None):
|
||||||
rewrite_path: str = base_path.rstrip("/")
|
rewrite_path: str = base_path.rstrip("/")
|
||||||
async with await anyio.open_file(response._path, "rb") as f:
|
response_path = cast("Any", response)._path
|
||||||
|
async with await anyio.open_file(response_path, "rb") as f:
|
||||||
content = await f.read()
|
content = await f.read()
|
||||||
content: str = (
|
content: str = (
|
||||||
content.decode("utf-8")
|
content.decode("utf-8")
|
||||||
|
|
@ -324,13 +328,13 @@ class HttpAPI:
|
||||||
new_response.set_cookie(
|
new_response.set_cookie(
|
||||||
morsel.key,
|
morsel.key,
|
||||||
morsel.value,
|
morsel.value,
|
||||||
expires=morsel["expires"],
|
expires=morsel.get("expires"),
|
||||||
domain=morsel["domain"],
|
domain=morsel.get("domain"),
|
||||||
max_age=morsel["max-age"],
|
max_age=morsel.get("max-age"),
|
||||||
path=morsel["path"],
|
path=morsel.get("path") or "/",
|
||||||
secure=bool(morsel["secure"]),
|
secure=bool(morsel.get("secure")),
|
||||||
httponly=bool(morsel["httponly"]),
|
httponly=bool(morsel.get("httponly")),
|
||||||
samesite=morsel["samesite"] or None,
|
samesite=morsel.get("samesite") or None,
|
||||||
)
|
)
|
||||||
|
|
||||||
return new_response
|
return new_response
|
||||||
|
|
@ -339,7 +343,7 @@ class HttpAPI:
|
||||||
|
|
||||||
return middleware_handler
|
return middleware_handler
|
||||||
|
|
||||||
async def _handle(self, handler: RequestHandler, request: Request) -> Response:
|
async def _handle(self, handler: Handler, request: Request) -> StreamResponse:
|
||||||
"""
|
"""
|
||||||
Call the handler with the request and return the response.
|
Call the handler with the request and return the response.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -104,19 +104,20 @@ class HttpSocket:
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
self._notify.subscribe("frontend", event_handler, f"{__class__.__name__}.emit")
|
self._notify.subscribe("frontend", event_handler, f"{HttpSocket.__name__}.emit")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def ws_event(func): # type: ignore
|
def ws_event(func):
|
||||||
"""
|
"""
|
||||||
Decorator to mark a method as a socket event.
|
Decorator to mark a method as a socket event.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@functools.wraps(func) # type: ignore
|
@functools.wraps(func)
|
||||||
async def wrapper(*args, **kwargs):
|
async def wrapper(*args, **kwargs):
|
||||||
return await func(*args, **kwargs) # type: ignore
|
return await func(*args, **kwargs)
|
||||||
|
|
||||||
wrapper._ws_event = func.__name__ # type: ignore
|
wrapper_with_event: Any = wrapper
|
||||||
|
wrapper_with_event._ws_event = func.__name__
|
||||||
return wrapper
|
return wrapper
|
||||||
|
|
||||||
async def on_shutdown(self, _: web.Application):
|
async def on_shutdown(self, _: web.Application):
|
||||||
|
|
@ -134,9 +135,11 @@ class HttpSocket:
|
||||||
if not (data and data.data):
|
if not (data and data.data):
|
||||||
return
|
return
|
||||||
|
|
||||||
await Services.get_instance().get("queue").add(item=Item.format(data.data))
|
queue = Services.get_instance().get("queue")
|
||||||
|
if queue is not None:
|
||||||
|
await queue.add(item=Item.format(data.data))
|
||||||
|
|
||||||
self._notify.subscribe(Events.ADD_URL, event_handler, f"{__class__.__name__}.add")
|
self._notify.subscribe(Events.ADD_URL, event_handler, f"{HttpSocket.__name__}.add")
|
||||||
|
|
||||||
load_modules(self.rootPath, self.rootPath / "routes" / "socket")
|
load_modules(self.rootPath, self.rootPath / "routes" / "socket")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -93,7 +93,7 @@ class Item:
|
||||||
bool: True if the item has extras, False otherwise.
|
bool: True if the item has extras, False otherwise.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
return self.extras and len(self.extras) > 0
|
return bool(self.extras and len(self.extras) > 0)
|
||||||
|
|
||||||
def add_extras(self, key: str, value: Any) -> None:
|
def add_extras(self, key: str, value: Any) -> None:
|
||||||
"""
|
"""
|
||||||
|
|
@ -117,7 +117,7 @@ class Item:
|
||||||
bool: True if the item has command options for yt, False otherwise.
|
bool: True if the item has command options for yt, False otherwise.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
return self.cli and len(self.cli) > 2
|
return bool(self.cli and len(self.cli) > 2)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _default_preset() -> str:
|
def _default_preset() -> str:
|
||||||
|
|
@ -161,7 +161,7 @@ class Item:
|
||||||
Item: The formatted item.
|
Item: The formatted item.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
url: str = item.get("url")
|
url = item.get("url")
|
||||||
|
|
||||||
if not url or not isinstance(url, str):
|
if not url or not isinstance(url, str):
|
||||||
msg = "url param is required."
|
msg = "url param is required."
|
||||||
|
|
@ -173,7 +173,7 @@ class Item:
|
||||||
if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url):
|
if len(url) >= 11 and re.fullmatch(r"[A-Za-z0-9_-]{11}", url):
|
||||||
url = f"https://www.youtube.com/watch?v={url}"
|
url = f"https://www.youtube.com/watch?v={url}"
|
||||||
|
|
||||||
data: dict[str, str] = {"url": url}
|
data: dict[str, Any] = {"url": url}
|
||||||
|
|
||||||
preset: str | None = item.get("preset")
|
preset: str | None = item.get("preset")
|
||||||
if preset and isinstance(preset, str) and preset != Item._default_preset():
|
if preset and isinstance(preset, str) and preset != Item._default_preset():
|
||||||
|
|
@ -185,24 +185,28 @@ class Item:
|
||||||
|
|
||||||
data["preset"] = preset
|
data["preset"] = preset
|
||||||
|
|
||||||
if item.get("folder") and isinstance(item.get("folder"), str):
|
folder = item.get("folder")
|
||||||
data["folder"] = item.get("folder")
|
if isinstance(folder, str) and folder:
|
||||||
|
data["folder"] = folder
|
||||||
|
|
||||||
if item.get("cookies") and isinstance(item.get("cookies"), str):
|
cookies = item.get("cookies")
|
||||||
data["cookies"] = item.get("cookies")
|
if isinstance(cookies, str) and cookies:
|
||||||
|
data["cookies"] = cookies
|
||||||
|
|
||||||
if item.get("template") and isinstance(item.get("template"), str):
|
template = item.get("template")
|
||||||
data["template"] = item.get("template")
|
if isinstance(template, str) and template:
|
||||||
|
data["template"] = template
|
||||||
|
|
||||||
if "auto_start" in item and isinstance(item.get("auto_start"), bool):
|
if isinstance(item.get("auto_start"), bool):
|
||||||
data["auto_start"] = bool(item.get("auto_start"))
|
data["auto_start"] = item["auto_start"]
|
||||||
|
|
||||||
extras: dict | None = item.get("extras")
|
extras: dict | None = item.get("extras")
|
||||||
if extras and isinstance(extras, dict) and len(extras) > 0:
|
if isinstance(extras, dict) and len(extras) > 0:
|
||||||
data["extras"] = extras
|
data["extras"] = extras
|
||||||
|
|
||||||
if item.get("requeued") and isinstance(item.get("requeued"), bool):
|
requeued = item.get("requeued")
|
||||||
data["requeued"] = item.get("requeued")
|
if isinstance(requeued, bool):
|
||||||
|
data["requeued"] = requeued
|
||||||
|
|
||||||
cli: str | None = item.get("cli")
|
cli: str | None = item.get("cli")
|
||||||
if cli and len(cli) > 2:
|
if cli and len(cli) > 2:
|
||||||
|
|
@ -574,8 +578,10 @@ class ItemDTO:
|
||||||
self.is_archivable = bool(self._archive_file)
|
self.is_archivable = bool(self._archive_file)
|
||||||
if not self.is_archivable:
|
if not self.is_archivable:
|
||||||
self.is_archived = False
|
self.is_archived = False
|
||||||
else:
|
elif self.archive_id and self._archive_file:
|
||||||
self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
|
self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
|
||||||
|
else:
|
||||||
|
self.is_archived = False
|
||||||
|
|
||||||
def get_archive_file(self) -> str | None:
|
def get_archive_file(self) -> str | None:
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ def parse_version(v: str) -> tuple[int, ...]:
|
||||||
|
|
||||||
|
|
||||||
class Packages:
|
class Packages:
|
||||||
def __init__(self, env: str | None, file: str | None, upgrade: bool = False):
|
def __init__(self, env: str | None, file: str | Path | None, upgrade: bool = False):
|
||||||
from_env: list[str] = env.split() if env else []
|
from_env: list[str] = env.split() if env else []
|
||||||
from_file = []
|
from_file = []
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
import asyncio
|
import asyncio
|
||||||
|
from collections.abc import Callable
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from aiocron import Cron
|
from aiocron import Cron
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
|
@ -66,7 +68,7 @@ class Scheduler(metaclass=Singleton):
|
||||||
if data and data.data:
|
if data and data.data:
|
||||||
self.add(**data.data)
|
self.add(**data.data)
|
||||||
|
|
||||||
EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{__class__.__name__}.add")
|
EventBus.get_instance().subscribe(Events.SCHEDULE_ADD, event_handler, f"{type(self).__name__}.add")
|
||||||
|
|
||||||
def get_all(self) -> dict[str, Cron]:
|
def get_all(self) -> dict[str, Cron]:
|
||||||
"""Return the jobs."""
|
"""Return the jobs."""
|
||||||
|
|
@ -99,7 +101,7 @@ class Scheduler(metaclass=Singleton):
|
||||||
return id in self._jobs
|
return id in self._jobs
|
||||||
|
|
||||||
def add(
|
def add(
|
||||||
self, timer: str, func: callable, args: tuple = (), kwargs: dict | None = None, id: str | None = None
|
self, timer: str, func: Callable[..., Any], args: tuple = (), kwargs: dict | None = None, id: str | None = None
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Add a job to the schedule.
|
Add a job to the schedule.
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import inspect
|
import inspect
|
||||||
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints
|
from typing import Annotated, Any, TypeVar, get_args, get_origin, get_type_hints
|
||||||
|
|
||||||
|
|
@ -117,7 +118,7 @@ class Services(metaclass=Singleton):
|
||||||
|
|
||||||
return candidates[0]
|
return candidates[0]
|
||||||
|
|
||||||
def _build_call_args(self, handler: callable, overrides: dict[str, Any]) -> dict[str, Any]:
|
def _build_call_args(self, handler: Callable[..., Any], overrides: dict[str, Any]) -> dict[str, Any]:
|
||||||
sig: inspect.Signature = inspect.signature(handler)
|
sig: inspect.Signature = inspect.signature(handler)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
@ -169,10 +170,10 @@ class Services(metaclass=Singleton):
|
||||||
|
|
||||||
return resolved
|
return resolved
|
||||||
|
|
||||||
async def handle_async(self, handler: callable, **kwargs) -> Any:
|
async def handle_async(self, handler: Callable[..., Any], **kwargs) -> Any:
|
||||||
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
|
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
|
||||||
return await handler(**resolved)
|
return await handler(**resolved)
|
||||||
|
|
||||||
def handle_sync(self, handler: callable, **kwargs) -> Any:
|
def handle_sync(self, handler: Callable[..., Any], **kwargs) -> Any:
|
||||||
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
|
resolved: dict[str, Any] = self._build_call_args(handler, kwargs)
|
||||||
return handler(**resolved)
|
return handler(**resolved)
|
||||||
|
|
|
||||||
|
|
@ -75,7 +75,7 @@ class TerminalSessionManager(metaclass=Singleton):
|
||||||
self._cleanup_job_id = Scheduler.get_instance().add(
|
self._cleanup_job_id = Scheduler.get_instance().add(
|
||||||
timer=CLEANUP_SCHEDULE,
|
timer=CLEANUP_SCHEDULE,
|
||||||
func=self.cleanup,
|
func=self.cleanup,
|
||||||
id=f"{__class__.__name__}.{__class__.cleanup.__name__}",
|
id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
|
||||||
)
|
)
|
||||||
|
|
||||||
async def on_startup(self, _: web.Application) -> None:
|
async def on_startup(self, _: web.Application) -> None:
|
||||||
|
|
|
||||||
|
|
@ -80,7 +80,7 @@ class UpdateChecker(metaclass=Singleton):
|
||||||
self._notify.subscribe(
|
self._notify.subscribe(
|
||||||
event=Events.STARTED,
|
event=Events.STARTED,
|
||||||
callback=event_handler,
|
callback=event_handler,
|
||||||
name=f"{__class__.__name__}.{__class__.attach.__name__}",
|
name=f"{UpdateChecker.__name__}.{type(self).attach.__name__}",
|
||||||
)
|
)
|
||||||
|
|
||||||
self._schedule_check()
|
self._schedule_check()
|
||||||
|
|
@ -108,7 +108,7 @@ class UpdateChecker(metaclass=Singleton):
|
||||||
self._job_id = self._scheduler.add(
|
self._job_id = self._scheduler.add(
|
||||||
timer=timer,
|
timer=timer,
|
||||||
func=lambda: asyncio.create_task(self.check_for_updates()),
|
func=lambda: asyncio.create_task(self.check_for_updates()),
|
||||||
id=f"{__class__.__name__}.{self.check_for_updates.__name__}",
|
id=f"{UpdateChecker.__name__}.{self.check_for_updates.__name__}",
|
||||||
)
|
)
|
||||||
|
|
||||||
async def check_for_updates(self) -> tuple[tuple[str, str | None], tuple[str, str | None]]:
|
async def check_for_updates(self) -> tuple[tuple[str, str | None], tuple[str, str | None]]:
|
||||||
|
|
|
||||||
|
|
@ -40,7 +40,8 @@ TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c")
|
||||||
|
|
||||||
|
|
||||||
class FileLogFormatter(logging.Formatter):
|
class FileLogFormatter(logging.Formatter):
|
||||||
def formatTime(self, record, datefmt=None): # noqa: ARG002, N802
|
def formatTime(self, record, datefmt=None): # noqa: N802
|
||||||
|
_ = datefmt
|
||||||
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
|
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -78,7 +79,8 @@ class JsonLogFormatter(logging.Formatter):
|
||||||
|
|
||||||
return json.dumps(data, ensure_ascii=False, default=str)
|
return json.dumps(data, ensure_ascii=False, default=str)
|
||||||
|
|
||||||
def formatTime(self, record, datefmt=None): # noqa: ARG002, N802
|
def formatTime(self, record, datefmt=None): # noqa: N802
|
||||||
|
_ = datefmt
|
||||||
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
|
return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
@ -210,7 +212,7 @@ class JsonLogFormatter(logging.Formatter):
|
||||||
return source
|
return source
|
||||||
|
|
||||||
|
|
||||||
def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
|
def timed_lru_cache(ttl_seconds: float, max_size: int = 128):
|
||||||
"""
|
"""
|
||||||
Decorator that applies an LRU cache with a time-to-live (TTL) to a function.
|
Decorator that applies an LRU cache with a time-to-live (TTL) to a function.
|
||||||
Supports both synchronous and asynchronous functions.
|
Supports both synchronous and asynchronous functions.
|
||||||
|
|
@ -280,8 +282,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
|
||||||
|
|
||||||
return _CacheInfo(hits=0, misses=len(cache), maxsize=max_size, currsize=len(cache))
|
return _CacheInfo(hits=0, misses=len(cache), maxsize=max_size, currsize=len(cache))
|
||||||
|
|
||||||
async_wrapper.cache_clear = cache_clear
|
async_wrapper_cache: Any = async_wrapper
|
||||||
async_wrapper.cache_info = cache_info
|
async_wrapper_cache.cache_clear = cache_clear
|
||||||
|
async_wrapper_cache.cache_info = cache_info
|
||||||
return async_wrapper
|
return async_wrapper
|
||||||
|
|
||||||
# For sync functions, use the original implementation
|
# For sync functions, use the original implementation
|
||||||
|
|
@ -309,8 +312,9 @@ def timed_lru_cache(ttl_seconds: int, max_size: int = 128):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
# expose cache_clear, cache_info
|
# expose cache_clear, cache_info
|
||||||
sync_wrapper.cache_clear = cached.cache_clear
|
sync_wrapper_cache: Any = sync_wrapper
|
||||||
sync_wrapper.cache_info = cached.cache_info
|
sync_wrapper_cache.cache_clear = cached.cache_clear
|
||||||
|
sync_wrapper_cache.cache_info = cached.cache_info
|
||||||
return sync_wrapper
|
return sync_wrapper
|
||||||
|
|
||||||
return decorator
|
return decorator
|
||||||
|
|
@ -381,7 +385,12 @@ def _is_safe_key(key: Any) -> bool:
|
||||||
|
|
||||||
|
|
||||||
def merge_dict(
|
def merge_dict(
|
||||||
source: dict, destination: dict, max_depth: int = 50, max_list_size: int = 10000, _depth: int = 0, _seen: set = None
|
source: dict,
|
||||||
|
destination: dict,
|
||||||
|
max_depth: int = 50,
|
||||||
|
max_list_size: int = 10000,
|
||||||
|
_depth: int = 0,
|
||||||
|
_seen: set | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""
|
"""
|
||||||
Merge data from source into destination.
|
Merge data from source into destination.
|
||||||
|
|
@ -472,7 +481,7 @@ def merge_dict(
|
||||||
return destination_copy
|
return destination_copy
|
||||||
|
|
||||||
|
|
||||||
def check_id(file: Path) -> bool | str:
|
def check_id(file: Path) -> bool | Path:
|
||||||
"""
|
"""
|
||||||
Check if we are able to get an id from the file name.
|
Check if we are able to get an id from the file name.
|
||||||
if so check if any video file with the same id exists.
|
if so check if any video file with the same id exists.
|
||||||
|
|
@ -491,6 +500,8 @@ def check_id(file: Path) -> bool | str:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
id: str | None = match.groupdict().get("id")
|
id: str | None = match.groupdict().get("id")
|
||||||
|
if id is None:
|
||||||
|
return False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if not file.parent.exists():
|
if not file.parent.exists():
|
||||||
|
|
@ -545,9 +556,9 @@ def validate_url(url: str, allow_internal: bool = False) -> bool:
|
||||||
from yarl import URL
|
from yarl import URL
|
||||||
|
|
||||||
parsed_url = URL(url)
|
parsed_url = URL(url)
|
||||||
except ValueError:
|
except ValueError as exc:
|
||||||
msg = "Invalid URL."
|
msg = "Invalid URL."
|
||||||
raise ValueError(msg) # noqa: B904
|
raise ValueError(msg) from exc
|
||||||
|
|
||||||
# Check allowed schemes
|
# Check allowed schemes
|
||||||
if parsed_url.scheme not in ["http", "https"]:
|
if parsed_url.scheme not in ["http", "https"]:
|
||||||
|
|
@ -596,7 +607,7 @@ def validate_uuid(uuid_str: str, version: int = 4) -> bool:
|
||||||
|
|
||||||
|
|
||||||
@timed_lru_cache(ttl_seconds=60, max_size=256)
|
@timed_lru_cache(ttl_seconds=60, max_size=256)
|
||||||
def get_file_sidecar(file: Path | None = None) -> dict[dict]:
|
def get_file_sidecar(file: Path | None = None) -> dict[str, list[dict[str, Any]]]:
|
||||||
"""
|
"""
|
||||||
Get sidecar files for the given file.
|
Get sidecar files for the given file.
|
||||||
|
|
||||||
|
|
@ -925,7 +936,7 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
|
||||||
download_path = Path(download_path)
|
download_path = Path(download_path)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
realFile: str = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
|
realFile: Path = Path(calc_download_path(base_path=download_path, folder=str(file), create_path=False))
|
||||||
if realFile.exists():
|
if realFile.exists():
|
||||||
return (realFile, 200)
|
return (realFile, 200)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
@ -936,11 +947,11 @@ def get_file(download_path: str | Path, file: str | Path) -> tuple[Path, int]:
|
||||||
)
|
)
|
||||||
return (Path(file), 404)
|
return (Path(file), 404)
|
||||||
|
|
||||||
possibleFile: bool | str = check_id(file=realFile)
|
possibleFile: bool | Path = check_id(file=realFile)
|
||||||
if not possibleFile:
|
if not isinstance(possibleFile, Path):
|
||||||
return (realFile, 404)
|
return (realFile, 404)
|
||||||
|
|
||||||
return (Path(possibleFile), 302)
|
return (possibleFile, 302)
|
||||||
|
|
||||||
|
|
||||||
def encrypt_data(data: str, key: bytes) -> str:
|
def encrypt_data(data: str, key: bytes) -> str:
|
||||||
|
|
@ -962,7 +973,7 @@ def encrypt_data(data: str, key: bytes) -> str:
|
||||||
return base64.urlsafe_b64encode(iv + ciphertext + tag).decode()
|
return base64.urlsafe_b64encode(iv + ciphertext + tag).decode()
|
||||||
|
|
||||||
|
|
||||||
def decrypt_data(data: str, key: bytes) -> str:
|
def decrypt_data(data: str, key: bytes) -> str | None:
|
||||||
"""
|
"""
|
||||||
Decrypts AES-GCM encrypted data
|
Decrypts AES-GCM encrypted data
|
||||||
|
|
||||||
|
|
@ -975,8 +986,8 @@ def decrypt_data(data: str, key: bytes) -> str:
|
||||||
|
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
data = base64.urlsafe_b64decode(data)
|
raw: bytes = base64.urlsafe_b64decode(data)
|
||||||
iv, ciphertext, tag = data[:12], data[12:-16], data[-16:]
|
iv, ciphertext, tag = raw[:12], raw[12:-16], raw[-16:]
|
||||||
cipher = AES.new(key, AES.MODE_GCM, nonce=iv)
|
cipher = AES.new(key, AES.MODE_GCM, nonce=iv)
|
||||||
plaintext = cipher.decrypt_and_verify(ciphertext, tag)
|
plaintext = cipher.decrypt_and_verify(ciphertext, tag)
|
||||||
return plaintext.decode()
|
return plaintext.decode()
|
||||||
|
|
@ -987,7 +998,7 @@ def decrypt_data(data: str, key: bytes) -> str:
|
||||||
def get(
|
def get(
|
||||||
data: dict | list,
|
data: dict | list,
|
||||||
path: str | list | None = None,
|
path: str | list | None = None,
|
||||||
default: any = None,
|
default: Any = None,
|
||||||
separator=".",
|
separator=".",
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
|
|
@ -1111,7 +1122,7 @@ def get_files(
|
||||||
extra={"requested_dir": dir, "base_path": str(base_path), "exception_type": type(e).__name__},
|
extra={"requested_dir": dir, "base_path": str(base_path), "exception_type": type(e).__name__},
|
||||||
exc_info=True,
|
exc_info=True,
|
||||||
)
|
)
|
||||||
return []
|
return [], 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
dir_path.relative_to(base_path)
|
dir_path.relative_to(base_path)
|
||||||
|
|
@ -1122,7 +1133,7 @@ def get_files(
|
||||||
base_path,
|
base_path,
|
||||||
extra={"path": str(dir_path), "base_path": str(base_path)},
|
extra={"path": str(dir_path), "base_path": str(base_path)},
|
||||||
)
|
)
|
||||||
return []
|
return [], 0
|
||||||
|
|
||||||
if not str(dir_path).startswith(str(base_path)):
|
if not str(dir_path).startswith(str(base_path)):
|
||||||
LOG.warning(
|
LOG.warning(
|
||||||
|
|
@ -1131,7 +1142,7 @@ def get_files(
|
||||||
base_path,
|
base_path,
|
||||||
extra={"path": str(dir_path), "base_path": str(base_path)},
|
extra={"path": str(dir_path), "base_path": str(base_path)},
|
||||||
)
|
)
|
||||||
return []
|
return [], 0
|
||||||
|
|
||||||
if not dir_path.is_dir():
|
if not dir_path.is_dir():
|
||||||
LOG.warning(
|
LOG.warning(
|
||||||
|
|
@ -1139,7 +1150,7 @@ def get_files(
|
||||||
dir_path,
|
dir_path,
|
||||||
extra={"path": str(dir_path), "base_path": str(base_path)},
|
extra={"path": str(dir_path), "base_path": str(base_path)},
|
||||||
)
|
)
|
||||||
return []
|
return [], 0
|
||||||
|
|
||||||
contents: list = []
|
contents: list = []
|
||||||
for file in dir_path.iterdir():
|
for file in dir_path.iterdir():
|
||||||
|
|
@ -1298,7 +1309,7 @@ def load_cookies(file: str | Path) -> tuple[bool, MozillaCookieJar]:
|
||||||
try:
|
try:
|
||||||
from http.cookiejar import MozillaCookieJar
|
from http.cookiejar import MozillaCookieJar
|
||||||
|
|
||||||
cookies = MozillaCookieJar(str(file), None, None)
|
cookies = MozillaCookieJar(str(file), delayload=False, policy=None)
|
||||||
cookies.load()
|
cookies.load()
|
||||||
|
|
||||||
return (True, cookies)
|
return (True, cookies)
|
||||||
|
|
@ -1477,35 +1488,6 @@ def str_to_dt(time_str: str, now=None) -> datetime:
|
||||||
return dt
|
return dt
|
||||||
|
|
||||||
|
|
||||||
def list_folders(path: Path, base: Path, depth_limit: int) -> list[str]:
|
|
||||||
"""
|
|
||||||
List all folders relative to a base path, up to a specified depth limit.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
path (Path): The path to start listing folders from.
|
|
||||||
base (Path): The base path to which the folders should be relative.
|
|
||||||
depth_limit (int): The maximum depth to traverse from the base path.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
list[str]: A list of folder paths relative to the base path, up to the specified
|
|
||||||
|
|
||||||
"""
|
|
||||||
if "/" == str(path):
|
|
||||||
return []
|
|
||||||
|
|
||||||
rel_depth: int = len(path.relative_to(base).parts)
|
|
||||||
if rel_depth > depth_limit:
|
|
||||||
return []
|
|
||||||
|
|
||||||
folders: list[str] = []
|
|
||||||
for entry in path.iterdir():
|
|
||||||
if entry.is_dir():
|
|
||||||
folders.append(str(entry.relative_to(base)))
|
|
||||||
folders.extend(list_folders(entry, base, depth_limit))
|
|
||||||
|
|
||||||
return folders
|
|
||||||
|
|
||||||
|
|
||||||
def get_channel_images(thumbnails: list[dict]) -> dict:
|
def get_channel_images(thumbnails: list[dict]) -> dict:
|
||||||
"""
|
"""
|
||||||
Extract channel images from a list of thumbnail dictionaries.
|
Extract channel images from a list of thumbnail dictionaries.
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,20 @@
|
||||||
|
import sys
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
def _dict_value(data: dict[Any, Any], key: Any) -> tuple[bool, Any]:
|
||||||
|
if key not in data or data[key] is None:
|
||||||
|
return False, None
|
||||||
|
return True, data[key]
|
||||||
|
|
||||||
|
|
||||||
|
def _dict_delete(data: dict[Any, Any], key: Any) -> bool:
|
||||||
|
if key not in data:
|
||||||
|
return False
|
||||||
|
del data[key]
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def get_value(value: Any) -> Any:
|
def get_value(value: Any) -> Any:
|
||||||
"""
|
"""
|
||||||
Return the result of calling `value` if it is callable, else return `value`.
|
Return the result of calling `value` if it is callable, else return `value`.
|
||||||
|
|
@ -91,14 +105,19 @@ def ag(
|
||||||
return val
|
return val
|
||||||
return get_value(default)
|
return get_value(default)
|
||||||
key = path
|
key = path
|
||||||
if isinstance(array, dict) and key in array and array[key] is not None:
|
if isinstance(array, dict):
|
||||||
return array[key]
|
found, value = _dict_value(array, key)
|
||||||
|
if found:
|
||||||
|
return value
|
||||||
if not isinstance(key, str) or separator not in key:
|
if not isinstance(key, str) or separator not in key:
|
||||||
return get_value(default)
|
return get_value(default)
|
||||||
current = array
|
current = array
|
||||||
for segment in key.split(separator):
|
for segment in key.split(separator):
|
||||||
if isinstance(current, dict) and segment in current and current[segment] is not None:
|
if isinstance(current, dict):
|
||||||
current = current[segment]
|
found, value = _dict_value(current, segment)
|
||||||
|
if not found:
|
||||||
|
return get_value(default)
|
||||||
|
current = value
|
||||||
elif isinstance(current, list):
|
elif isinstance(current, list):
|
||||||
try:
|
try:
|
||||||
idx = int(segment)
|
idx = int(segment)
|
||||||
|
|
@ -150,7 +169,8 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
|
||||||
except TypeError:
|
except TypeError:
|
||||||
return False
|
return False
|
||||||
if isinstance(data, dict):
|
if isinstance(data, dict):
|
||||||
if path in data and data[path] is not None:
|
found, _ = _dict_value(data, path)
|
||||||
|
if found:
|
||||||
return True
|
return True
|
||||||
elif isinstance(data, list) and isinstance(path, int):
|
elif isinstance(data, list) and isinstance(path, int):
|
||||||
return 0 <= path < len(data) and data[path] is not None
|
return 0 <= path < len(data) and data[path] is not None
|
||||||
|
|
@ -158,8 +178,11 @@ def ag_exists(data: dict[Any, Any] | list[Any] | object, path: str | int, separa
|
||||||
segments = path_str.split(separator)
|
segments = path_str.split(separator)
|
||||||
current = data
|
current = data
|
||||||
for seg in segments:
|
for seg in segments:
|
||||||
if isinstance(current, dict) and seg in current and current[seg] is not None:
|
if isinstance(current, dict):
|
||||||
current = current[seg]
|
found, value = _dict_value(current, seg)
|
||||||
|
if not found:
|
||||||
|
return False
|
||||||
|
current = value
|
||||||
elif isinstance(current, list):
|
elif isinstance(current, list):
|
||||||
try:
|
try:
|
||||||
idx = int(seg)
|
idx = int(seg)
|
||||||
|
|
@ -189,17 +212,16 @@ def ag_delete(
|
||||||
The modified structure.
|
The modified structure.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
if isinstance(path, list | tuple):
|
|
||||||
for p in path:
|
|
||||||
ag_delete(data, p, separator)
|
|
||||||
return data
|
|
||||||
if not isinstance(data, dict | list):
|
if not isinstance(data, dict | list):
|
||||||
try:
|
try:
|
||||||
data = vars(data)
|
data = vars(data)
|
||||||
except TypeError:
|
except TypeError:
|
||||||
return data # type: ignore
|
return data # type: ignore
|
||||||
if isinstance(data, dict) and path in data:
|
if isinstance(path, list | tuple):
|
||||||
del data[path]
|
for p in path:
|
||||||
|
ag_delete(data, p, separator)
|
||||||
|
return data
|
||||||
|
if isinstance(data, dict) and _dict_delete(data, path):
|
||||||
return data
|
return data
|
||||||
if isinstance(data, list) and isinstance(path, int):
|
if isinstance(data, list) and isinstance(path, int):
|
||||||
if 0 <= path < len(data):
|
if 0 <= path < len(data):
|
||||||
|
|
@ -210,8 +232,12 @@ def ag_delete(
|
||||||
last = segments.pop()
|
last = segments.pop()
|
||||||
current = data
|
current = data
|
||||||
for seg in segments:
|
for seg in segments:
|
||||||
if isinstance(current, dict) and seg in current:
|
if isinstance(current, dict):
|
||||||
current = current[seg]
|
found, value = _dict_value(current, seg)
|
||||||
|
if not found:
|
||||||
|
current = None
|
||||||
|
break
|
||||||
|
current = value
|
||||||
elif isinstance(current, list):
|
elif isinstance(current, list):
|
||||||
try:
|
try:
|
||||||
idx = int(seg)
|
idx = int(seg)
|
||||||
|
|
@ -228,8 +254,8 @@ def ag_delete(
|
||||||
break
|
break
|
||||||
if current is None:
|
if current is None:
|
||||||
return data
|
return data
|
||||||
if isinstance(current, dict) and last in current:
|
if isinstance(current, dict):
|
||||||
del current[last]
|
_dict_delete(current, last)
|
||||||
elif isinstance(current, list):
|
elif isinstance(current, list):
|
||||||
try:
|
try:
|
||||||
idx = int(last)
|
idx = int(last)
|
||||||
|
|
@ -246,10 +272,12 @@ def run_tests():
|
||||||
assert ag_set(d, "a.c.d", 42) == {"a": {"b": 1, "c": {"d": 42}}}
|
assert ag_set(d, "a.c.d", 42) == {"a": {"b": 1, "c": {"d": 42}}}
|
||||||
|
|
||||||
def test_ag_set_overwrite_error():
|
def test_ag_set_overwrite_error():
|
||||||
|
error = ""
|
||||||
try:
|
try:
|
||||||
ag_set({"a": 1}, "a.b", 2)
|
ag_set({"a": 1}, "a.b", 2)
|
||||||
except RuntimeError as e:
|
except RuntimeError as e:
|
||||||
assert "Cannot set value at path" in str(e) # noqa: PT017
|
error = str(e)
|
||||||
|
assert "Cannot set value at path" in error
|
||||||
|
|
||||||
def test_ag_get_value_callable_and_value():
|
def test_ag_get_value_callable_and_value():
|
||||||
assert get_value(5) == 5
|
assert get_value(5) == 5
|
||||||
|
|
@ -320,9 +348,9 @@ def run_tests():
|
||||||
]:
|
]:
|
||||||
try:
|
try:
|
||||||
test()
|
test()
|
||||||
print(f"PASS: {test.__name__}") # noqa: T201
|
sys.stdout.write(f"PASS: {test.__name__}\n")
|
||||||
except AssertionError:
|
except AssertionError:
|
||||||
print(f"FAIL {test.__name__}") # noqa: T201
|
sys.stdout.write(f"FAIL {test.__name__}\n")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@ class Cache(metaclass=ThreadSafe):
|
||||||
Scheduler.get_instance().add(
|
Scheduler.get_instance().add(
|
||||||
timer="* * * * *",
|
timer="* * * * *",
|
||||||
func=self.cleanup,
|
func=self.cleanup,
|
||||||
id=f"{__class__.__name__}.{__class__.cleanup.__name__}",
|
id=f"{type(self).__name__}.{type(self).cleanup.__name__}",
|
||||||
)
|
)
|
||||||
|
|
||||||
def set(self, key: str, value: Any, ttl: float | None = None) -> None:
|
def set(self, key: str, value: Any, ttl: float | None = None) -> None:
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ from __future__ import annotations
|
||||||
import http.cookiejar
|
import http.cookiejar
|
||||||
from abc import ABC
|
from abc import ABC
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from typing import Any, ClassVar
|
from typing import Any, ClassVar, cast
|
||||||
|
|
||||||
from yt_dlp.networking.common import (
|
from yt_dlp.networking.common import (
|
||||||
_REQUEST_HANDLERS,
|
_REQUEST_HANDLERS,
|
||||||
|
|
@ -223,11 +223,13 @@ class CFSolverRH(RequestHandler, ABC):
|
||||||
try:
|
try:
|
||||||
response: Response = director.send(request)
|
response: Response = director.send(request)
|
||||||
except HTTPError as error:
|
except HTTPError as error:
|
||||||
if error.response and is_cf_challenge(getattr(error.response, "status", None), error.response.headers):
|
if error.response and is_cf_challenge(
|
||||||
|
getattr(error.response, "status", None), cast("Any", error.response.headers)
|
||||||
|
):
|
||||||
return self._retry_with_clearance(request, error.response, director)
|
return self._retry_with_clearance(request, error.response, director)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
if is_cf_challenge(getattr(response, "status", None), response.headers):
|
if is_cf_challenge(getattr(response, "status", None), cast("Any", response.headers)):
|
||||||
return self._retry_with_clearance(request, response, director)
|
return self._retry_with_clearance(request, response, director)
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
|
||||||
|
|
@ -1,16 +1,18 @@
|
||||||
# flake8: noqa: S310
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from typing import Any
|
from typing import TYPE_CHECKING, Any
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from app.library.log import get_logger
|
from app.library.log import get_logger
|
||||||
|
|
||||||
from .cache import Cache
|
from .cache import Cache
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Mapping
|
||||||
|
|
||||||
CACHE: Cache = Cache()
|
CACHE: Cache = Cache()
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
|
|
||||||
|
|
@ -57,7 +59,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
|
||||||
if user_agent:
|
if user_agent:
|
||||||
payload.setdefault("headers", {})["User-Agent"] = user_agent
|
payload.setdefault("headers", {})["User-Agent"] = user_agent
|
||||||
|
|
||||||
req = urllib.request.Request(
|
req = urllib.request.Request( # noqa: S310
|
||||||
endpoint,
|
endpoint,
|
||||||
data=json.dumps(payload).encode("utf-8"),
|
data=json.dumps(payload).encode("utf-8"),
|
||||||
headers={"Content-Type": "application/json"},
|
headers={"Content-Type": "application/json"},
|
||||||
|
|
@ -68,7 +70,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
|
||||||
"Solving Cloudflare challenge for '%s' via FlareSolverr.", host, extra={"host": host, "endpoint": endpoint}
|
"Solving Cloudflare challenge for '%s' via FlareSolverr.", host, extra={"host": host, "endpoint": endpoint}
|
||||||
)
|
)
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp:
|
with urllib.request.urlopen(req, timeout=float(config.flaresolverr_client_timeout)) as resp: # noqa: S310
|
||||||
result = json.loads(resp.read().decode("utf-8"))
|
result = json.loads(resp.read().decode("utf-8"))
|
||||||
|
|
||||||
if "ok" != result.get("status"):
|
if "ok" != result.get("status"):
|
||||||
|
|
@ -98,7 +100,7 @@ def solver(url: str, cookies: list[dict[str, Any]], user_agent: str | None) -> d
|
||||||
return CACHE.get(host)
|
return CACHE.get(host)
|
||||||
|
|
||||||
|
|
||||||
def is_cf_challenge(status: int | None, headers: dict[str, Any] | None) -> bool:
|
def is_cf_challenge(status: int | None, headers: Mapping[str, Any] | None) -> bool:
|
||||||
"""
|
"""
|
||||||
Determine whether a response indicates a Cloudflare challenge.
|
Determine whether a response indicates a Cloudflare challenge.
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import logging
|
||||||
import multiprocessing
|
import multiprocessing
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from logging.handlers import TimedRotatingFileHandler
|
from logging.handlers import TimedRotatingFileHandler
|
||||||
|
|
@ -30,7 +31,7 @@ APP_THIRD_PARTY_LOG_LEVELS: tuple[tuple[str, int], ...] = (
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from subprocess import CompletedProcess
|
from subprocess import CompletedProcess
|
||||||
|
|
||||||
SUPPORTED_CODECS: tuple[str] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264")
|
SUPPORTED_CODECS: tuple[str, ...] = ("h264_qsv", "h264_nvenc", "h264_amf", "h264_videotoolbox", "h264_vaapi", "libx264")
|
||||||
"Supported encoder names in order of preference."
|
"Supported encoder names in order of preference."
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -47,9 +48,6 @@ class Config(metaclass=Singleton):
|
||||||
download_path: str = "."
|
download_path: str = "."
|
||||||
"""The path to the download directory."""
|
"""The path to the download directory."""
|
||||||
|
|
||||||
download_path_depth: int = 2
|
|
||||||
"""How many subdirectories to show in auto complete."""
|
|
||||||
|
|
||||||
download_info_expires: int = 10800
|
download_info_expires: int = 10800
|
||||||
"""How long (in seconds) the download info is valid before it needs to be re-extracted."""
|
"""How long (in seconds) the download info is valid before it needs to be re-extracted."""
|
||||||
|
|
||||||
|
|
@ -182,7 +180,7 @@ class Config(metaclass=Singleton):
|
||||||
file_logging: bool = True
|
file_logging: bool = True
|
||||||
"Enable file logging."
|
"Enable file logging."
|
||||||
|
|
||||||
secret_key: str
|
secret_key: str | bytes
|
||||||
"The secret key to use for the application."
|
"The secret key to use for the application."
|
||||||
|
|
||||||
tasks_handler_timer: str = "15 */1 * * *"
|
tasks_handler_timer: str = "15 */1 * * *"
|
||||||
|
|
@ -230,6 +228,9 @@ class Config(metaclass=Singleton):
|
||||||
default_pagination: int = 50
|
default_pagination: int = 50
|
||||||
"""The default number of items per page for pagination."""
|
"""The default number of items per page for pagination."""
|
||||||
|
|
||||||
|
queue_display_limit: int = 100
|
||||||
|
"""Maximum number of queued downloads returned to the UI. 0 means unlimited."""
|
||||||
|
|
||||||
task_handler_random_delay: float = 60.0
|
task_handler_random_delay: float = 60.0
|
||||||
"""The maximum random delay in seconds before starting a task handler."""
|
"""The maximum random delay in seconds before starting a task handler."""
|
||||||
|
|
||||||
|
|
@ -282,10 +283,10 @@ class Config(metaclass=Singleton):
|
||||||
"max_workers_per_extractor",
|
"max_workers_per_extractor",
|
||||||
"extract_info_timeout",
|
"extract_info_timeout",
|
||||||
"debugpy_port",
|
"debugpy_port",
|
||||||
"download_path_depth",
|
|
||||||
"download_info_expires",
|
"download_info_expires",
|
||||||
"auto_clear_history_days",
|
"auto_clear_history_days",
|
||||||
"default_pagination",
|
"default_pagination",
|
||||||
|
"queue_display_limit",
|
||||||
"extract_info_concurrency",
|
"extract_info_concurrency",
|
||||||
"thumb_concurrency",
|
"thumb_concurrency",
|
||||||
"flaresolverr_max_timeout",
|
"flaresolverr_max_timeout",
|
||||||
|
|
@ -346,6 +347,7 @@ class Config(metaclass=Singleton):
|
||||||
"app_build_date",
|
"app_build_date",
|
||||||
"app_branch",
|
"app_branch",
|
||||||
"default_pagination",
|
"default_pagination",
|
||||||
|
"queue_display_limit",
|
||||||
"check_for_updates",
|
"check_for_updates",
|
||||||
"new_version",
|
"new_version",
|
||||||
"yt_new_version",
|
"yt_new_version",
|
||||||
|
|
@ -376,7 +378,7 @@ class Config(metaclass=Singleton):
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
|
|
||||||
self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or str(Path(baseDefaultPath) / "var" / "config")
|
self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or str(Path(baseDefaultPath) / "var" / "config")
|
||||||
envFile: str = Path(self.config_path) / ".env"
|
envFile: Path = Path(self.config_path) / ".env"
|
||||||
|
|
||||||
if envFile.exists():
|
if envFile.exists():
|
||||||
LOG.info("Loading environment variables from '%s'.", envFile)
|
LOG.info("Loading environment variables from '%s'.", envFile)
|
||||||
|
|
@ -387,7 +389,7 @@ class Config(metaclass=Singleton):
|
||||||
self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or str(
|
self.download_path = os.environ.get("YTP_DOWNLOAD_PATH", None) or str(
|
||||||
Path(baseDefaultPath) / "var" / "downloads"
|
Path(baseDefaultPath) / "var" / "downloads"
|
||||||
)
|
)
|
||||||
self.app_path = Path(__file__).parent.parent.absolute()
|
self.app_path = str(Path(__file__).parent.parent.absolute())
|
||||||
|
|
||||||
for k, v in self._get_attributes().items():
|
for k, v in self._get_attributes().items():
|
||||||
if k.startswith("_") or k in self._manual_vars:
|
if k.startswith("_") or k in self._manual_vars:
|
||||||
|
|
@ -500,7 +502,7 @@ class Config(metaclass=Singleton):
|
||||||
handler.setFormatter(formatter)
|
handler.setFormatter(formatter)
|
||||||
logging.getLogger().addHandler(handler)
|
logging.getLogger().addHandler(handler)
|
||||||
|
|
||||||
key_file: str = Path(self.config_path) / "secret.key"
|
key_file: Path = Path(self.config_path) / "secret.key"
|
||||||
|
|
||||||
if key_file.exists() and key_file.stat().st_size > 2:
|
if key_file.exists() and key_file.stat().st_size > 2:
|
||||||
with open(key_file, "rb") as f:
|
with open(key_file, "rb") as f:
|
||||||
|
|
@ -510,7 +512,7 @@ class Config(metaclass=Singleton):
|
||||||
with open(key_file, "wb") as f:
|
with open(key_file, "wb") as f:
|
||||||
f.write(self.secret_key)
|
f.write(self.secret_key)
|
||||||
|
|
||||||
self.started = time.time()
|
self.started = int(time.time())
|
||||||
|
|
||||||
for _tool, _level in APP_THIRD_PARTY_LOG_LEVELS:
|
for _tool, _level in APP_THIRD_PARTY_LOG_LEVELS:
|
||||||
logging.getLogger(_tool).setLevel(_level)
|
logging.getLogger(_tool).setLevel(_level)
|
||||||
|
|
@ -544,7 +546,7 @@ class Config(metaclass=Singleton):
|
||||||
|
|
||||||
def _get_attributes(self) -> dict:
|
def _get_attributes(self) -> dict:
|
||||||
attrs: dict = {}
|
attrs: dict = {}
|
||||||
vClass: str = self.__class__
|
vClass: type = self.__class__
|
||||||
|
|
||||||
for attribute in vClass.__dict__:
|
for attribute in vClass.__dict__:
|
||||||
if attribute.startswith("_"):
|
if attribute.startswith("_"):
|
||||||
|
|
@ -600,7 +602,7 @@ class Config(metaclass=Singleton):
|
||||||
dict[str, str]: The replacer variables.
|
dict[str, str]: The replacer variables.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
keys: tuple[str] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file")
|
keys: tuple[str, ...] = ("os_sep", "download_path", "temp_path", "config_path", "archive_file")
|
||||||
return {k: getattr(self, k) for k in keys}
|
return {k: getattr(self, k) for k in keys}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
@ -618,15 +620,20 @@ class Config(metaclass=Singleton):
|
||||||
This is used to set the version to the latest git tag.
|
This is used to set the version to the latest git tag.
|
||||||
"""
|
"""
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
git_path: str = Path(__file__).parent / ".." / ".." / ".git"
|
git_path: Path = Path(__file__).parent / ".." / ".." / ".git"
|
||||||
if not git_path.exists():
|
if not git_path.exists():
|
||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
git = shutil.which("git")
|
||||||
|
if not git:
|
||||||
|
LOG.warning("Git executable was not found.")
|
||||||
|
return
|
||||||
|
|
||||||
branch_result: CompletedProcess[str] = subprocess.run(
|
branch_result: CompletedProcess[str] = subprocess.run(
|
||||||
["git", "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"], # noqa: S607
|
[git, "-c", f"safe.directory={git_path.parent!s}", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||||
cwd=os.path.dirname(git_path),
|
cwd=os.path.dirname(git_path),
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
|
|
@ -644,7 +651,7 @@ class Config(metaclass=Singleton):
|
||||||
return
|
return
|
||||||
|
|
||||||
commit_result: CompletedProcess[str] = subprocess.run(
|
commit_result: CompletedProcess[str] = subprocess.run(
|
||||||
["git", "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"], # noqa: S607
|
[git, "-c", f"safe.directory={git_path.parent!s}", "log", "-1", "--format=%ct_%H"],
|
||||||
cwd=os.path.dirname(git_path),
|
cwd=os.path.dirname(git_path),
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,14 @@ if TYPE_CHECKING:
|
||||||
class Download:
|
class Download:
|
||||||
update_task: asyncio.Task | None = None
|
update_task: asyncio.Task | None = None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _raise_no_formats(msg: str) -> None:
|
||||||
|
raise ValueError(msg)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _raise_interrupted() -> None:
|
||||||
|
raise SystemExit(130)
|
||||||
|
|
||||||
def __init__(self, info: ItemDTO, info_dict: dict | None = None, logs: list[str] | None = None):
|
def __init__(self, info: ItemDTO, info_dict: dict | None = None, logs: list[str] | None = None):
|
||||||
"""
|
"""
|
||||||
Initialize download task.
|
Initialize download task.
|
||||||
|
|
@ -51,7 +59,7 @@ class Download:
|
||||||
"""
|
"""
|
||||||
config: Config = Config.get_instance()
|
config: Config = Config.get_instance()
|
||||||
|
|
||||||
self.download_dir: str = info.download_dir
|
self.download_dir: str = info.download_dir or ""
|
||||||
self.temp_dir: str | None = info.temp_dir
|
self.temp_dir: str | None = info.temp_dir
|
||||||
self.template: str | None = info.template
|
self.template: str | None = info.template
|
||||||
self.template_chapter: str | None = info.template_chapter
|
self.template_chapter: str | None = info.template_chapter
|
||||||
|
|
@ -98,6 +106,16 @@ class Download:
|
||||||
params: dict[str, Any] = {}
|
params: dict[str, Any] = {}
|
||||||
download_skipped = False
|
download_skipped = False
|
||||||
|
|
||||||
|
hook_handlers = self._hook_handlers
|
||||||
|
if hook_handlers is None:
|
||||||
|
msg = "_hook_handlers must be initialized before _download(). Call start() first."
|
||||||
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
|
status_queue = self.status_queue
|
||||||
|
if status_queue is None:
|
||||||
|
msg = "status_queue must be initialized before _download(). Call start() first."
|
||||||
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
params = (
|
params = (
|
||||||
self.info.get_ytdlp_opts()
|
self.info.get_ytdlp_opts()
|
||||||
|
|
@ -125,9 +143,9 @@ class Download:
|
||||||
|
|
||||||
params.update(
|
params.update(
|
||||||
{
|
{
|
||||||
"progress_hooks": [self._hook_handlers.progress_hook],
|
"progress_hooks": [hook_handlers.progress_hook],
|
||||||
"postprocessor_hooks": [self._hook_handlers.postprocessor_hook],
|
"postprocessor_hooks": [hook_handlers.postprocessor_hook],
|
||||||
"post_hooks": [self._hook_handlers.post_hook],
|
"post_hooks": [hook_handlers.post_hook],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -137,7 +155,10 @@ class Download:
|
||||||
|
|
||||||
if self.info.cookies:
|
if self.info.cookies:
|
||||||
try:
|
try:
|
||||||
cookie_file = Path(self._temp_manager.temp_path or self.temp_dir) / f"cookie_{self.info._id}.txt"
|
cookie_file = (
|
||||||
|
Path(self._temp_manager.temp_path or self.temp_dir or self.download_dir)
|
||||||
|
/ f"cookie_{self.info._id}.txt"
|
||||||
|
)
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
f"Creating cookie file for '{self.info.title}' at '{cookie_file}'.",
|
f"Creating cookie file for '{self.info.title}' at '{cookie_file}'.",
|
||||||
extra={
|
extra={
|
||||||
|
|
@ -173,7 +194,9 @@ class Download:
|
||||||
f"Info dict for '{self.info.url}' marked for re-extraction, ignoring pre-extracted info."
|
f"Info dict for '{self.info.url}' marked for re-extraction, ignoring pre-extracted info."
|
||||||
)
|
)
|
||||||
self.info_dict = None
|
self.info_dict = None
|
||||||
elif self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and self.info.get_preset().default:
|
elif self.info_dict.get("extractor_key") in GENERIC_EXTRACTORS and (
|
||||||
|
(preset := self.info.get_preset()) is not None and preset.default
|
||||||
|
):
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
"Removing download archive option for generic extractor on '%s'.",
|
"Removing download archive option for generic extractor on '%s'.",
|
||||||
self.info.url,
|
self.info.url,
|
||||||
|
|
@ -267,7 +290,7 @@ class Download:
|
||||||
if filtered_logs := extract_ytdlp_logs(self.logs):
|
if filtered_logs := extract_ytdlp_logs(self.logs):
|
||||||
msg += " " + ", ".join(filtered_logs)
|
msg += " " + ", ".join(filtered_logs)
|
||||||
|
|
||||||
raise ValueError(msg) # noqa: TRY301
|
self._raise_no_formats(msg)
|
||||||
|
|
||||||
self.logger.info(
|
self.logger.info(
|
||||||
f"Downloading '{self.info.title}' from '{self.info.url}' to '{self.download_dir}'.",
|
f"Downloading '{self.info.title}' from '{self.info.url}' to '{self.download_dir}'.",
|
||||||
|
|
@ -327,11 +350,11 @@ class Download:
|
||||||
def mark_cancelled(*_) -> None:
|
def mark_cancelled(*_) -> None:
|
||||||
cls._interrupted = True
|
cls._interrupted = True
|
||||||
cls.to_screen("[info] Interrupt received, exiting cleanly...")
|
cls.to_screen("[info] Interrupt received, exiting cleanly...")
|
||||||
raise SystemExit(130) # noqa: TRY301
|
self._raise_interrupted()
|
||||||
|
|
||||||
signal.signal(signal.SIGUSR1, mark_cancelled)
|
signal.signal(signal.SIGUSR1, mark_cancelled)
|
||||||
|
|
||||||
self.status_queue.put({"id": self.id, "status": "downloading", "download_skipped": download_skipped})
|
status_queue.put({"id": self.id, "status": "downloading", "download_skipped": download_skipped})
|
||||||
|
|
||||||
if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
|
if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
|
|
@ -386,7 +409,7 @@ class Download:
|
||||||
)
|
)
|
||||||
ret = cls.download(url_list=[self.info.url])
|
ret = cls.download(url_list=[self.info.url])
|
||||||
|
|
||||||
self.status_queue.put(
|
status_queue.put(
|
||||||
{
|
{
|
||||||
"id": self.id,
|
"id": self.id,
|
||||||
"status": "finished" if 0 == ret else "error",
|
"status": "finished" if 0 == ret else "error",
|
||||||
|
|
@ -407,7 +430,7 @@ class Download:
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
self.status_queue.put(
|
status_queue.put(
|
||||||
{
|
{
|
||||||
"id": self.id,
|
"id": self.id,
|
||||||
"status": "skip",
|
"status": "skip",
|
||||||
|
|
@ -432,7 +455,7 @@ class Download:
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
self.status_queue.put(
|
status_queue.put(
|
||||||
{
|
{
|
||||||
"id": self.id,
|
"id": self.id,
|
||||||
"status": "error",
|
"status": "error",
|
||||||
|
|
@ -442,7 +465,7 @@ class Download:
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
self.status_queue.put(Terminator())
|
status_queue.put(Terminator())
|
||||||
if cookie_file and cookie_file.exists():
|
if cookie_file and cookie_file.exists():
|
||||||
try:
|
try:
|
||||||
cookie_file.unlink()
|
cookie_file.unlink()
|
||||||
|
|
@ -499,24 +522,24 @@ class Download:
|
||||||
Process exit code or None
|
Process exit code or None
|
||||||
|
|
||||||
"""
|
"""
|
||||||
self.status_queue = Config.get_manager().Queue()
|
status_queue: Any = Config.get_manager().Queue()
|
||||||
|
self.status_queue = status_queue
|
||||||
|
|
||||||
if temp_path := self._temp_manager.create_temp_path():
|
temp_path = self._temp_manager.create_temp_path()
|
||||||
self.info.temp_path = str(temp_path)
|
|
||||||
|
|
||||||
self._status_tracker = StatusTracker(
|
self._status_tracker = StatusTracker(
|
||||||
info=self.info,
|
info=self.info,
|
||||||
download_id=self.id,
|
download_id=self.id,
|
||||||
download_dir=self.download_dir,
|
download_dir=self.download_dir,
|
||||||
temp_path=temp_path,
|
temp_path=temp_path,
|
||||||
status_queue=self.status_queue,
|
status_queue=status_queue,
|
||||||
logger=self.logger,
|
logger=self.logger,
|
||||||
debug=self.debug,
|
debug=self.debug,
|
||||||
)
|
)
|
||||||
|
|
||||||
self._hook_handlers = HookHandlers(
|
self._hook_handlers = HookHandlers(
|
||||||
download_id=self.id,
|
download_id=self.id,
|
||||||
status_queue=self.status_queue,
|
status_queue=status_queue,
|
||||||
logger=self.logger,
|
logger=self.logger,
|
||||||
debug=self.debug,
|
debug=self.debug,
|
||||||
)
|
)
|
||||||
|
|
@ -529,7 +552,8 @@ class Download:
|
||||||
EventBus.get_instance().emit(Events.ITEM_UPDATED, data=self.info)
|
EventBus.get_instance().emit(Events.ITEM_UPDATED, data=self.info)
|
||||||
asyncio.create_task(self._status_tracker.progress_update(), name=f"update-{self.id}")
|
asyncio.create_task(self._status_tracker.progress_update(), name=f"update-{self.id}")
|
||||||
|
|
||||||
ret = await asyncio.get_running_loop().run_in_executor(None, self._process_manager.proc.join)
|
proc = self._process_manager.proc
|
||||||
|
ret = await asyncio.get_running_loop().run_in_executor(None, proc.join) if proc else None
|
||||||
|
|
||||||
if self._status_tracker.final_update:
|
if self._status_tracker.final_update:
|
||||||
return ret
|
return ret
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ import time
|
||||||
import uuid
|
import uuid
|
||||||
from numbers import Number
|
from numbers import Number
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import yt_dlp.utils
|
import yt_dlp.utils
|
||||||
|
|
||||||
|
|
@ -67,7 +67,7 @@ async def add_item(
|
||||||
already=None,
|
already=None,
|
||||||
logs: list | None = None,
|
logs: list | None = None,
|
||||||
yt_params: dict | None = None,
|
yt_params: dict | None = None,
|
||||||
) -> dict[str, str]:
|
) -> dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Route an entry to the appropriate processor based on type.
|
Route an entry to the appropriate processor based on type.
|
||||||
|
|
||||||
|
|
@ -102,7 +102,7 @@ async def add_item(
|
||||||
|
|
||||||
async def add(
|
async def add(
|
||||||
queue: "DownloadQueue", item: "Item", already: set | None = None, entry: dict | None = None
|
queue: "DownloadQueue", item: "Item", already: set | None = None, entry: dict | None = None
|
||||||
) -> dict[str, str]:
|
) -> dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Add an item to the download queue.
|
Add an item to the download queue.
|
||||||
|
|
||||||
|
|
@ -204,7 +204,7 @@ async def add(
|
||||||
)
|
)
|
||||||
|
|
||||||
if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"):
|
if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"):
|
||||||
dlInfo.info.msg += f" Found in archive '{archive_file}'."
|
dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{archive_file}'."
|
||||||
|
|
||||||
await queue.done.put(dlInfo)
|
await queue.done.put(dlInfo)
|
||||||
|
|
||||||
|
|
@ -306,7 +306,10 @@ async def add(
|
||||||
new_archive_id: str | None = None
|
new_archive_id: str | None = None
|
||||||
|
|
||||||
if entry.get("extractor_key") and entry.get("id"):
|
if entry.get("extractor_key") and entry.get("id"):
|
||||||
new_archive_id: str = f"{entry.get('extractor_key').lower()} {entry.get('id')}"
|
_extractor_key = entry.get("extractor_key")
|
||||||
|
_entry_id = entry.get("id")
|
||||||
|
if isinstance(_extractor_key, str) and _entry_id is not None:
|
||||||
|
new_archive_id = f"{_extractor_key.lower()} {_entry_id!s}"
|
||||||
if new_archive_id != archive_id:
|
if new_archive_id != archive_id:
|
||||||
extra_ids.append(new_archive_id)
|
extra_ids.append(new_archive_id)
|
||||||
|
|
||||||
|
|
@ -336,7 +339,7 @@ async def add(
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
dlInfo.info.msg += f" Found in archive '{_archive_file}'."
|
dlInfo.info.msg = (dlInfo.info.msg or "") + f" Found in archive '{_archive_file}'."
|
||||||
await queue.done.put(dlInfo)
|
await queue.done.put(dlInfo)
|
||||||
|
|
||||||
queue._notify.emit(
|
queue._notify.emit(
|
||||||
|
|
@ -374,7 +377,7 @@ async def add(
|
||||||
|
|
||||||
if condition.extras.get("ignore_download", False):
|
if condition.extras.get("ignore_download", False):
|
||||||
extra_msg: str = ""
|
extra_msg: str = ""
|
||||||
if _archive_file and not condition.extras.get("no_archive", False):
|
if _archive_file and not condition.extras.get("no_archive", False) and archive_id:
|
||||||
archive_add(_archive_file, [archive_id])
|
archive_add(_archive_file, [archive_id])
|
||||||
extra_msg = f" and added to archive file '{_archive_file}'"
|
extra_msg = f" and added to archive file '{_archive_file}'"
|
||||||
|
|
||||||
|
|
@ -385,7 +388,7 @@ async def add(
|
||||||
if not store_type:
|
if not store_type:
|
||||||
dlInfo = Download(
|
dlInfo = Download(
|
||||||
info=ItemDTO(
|
info=ItemDTO(
|
||||||
id=entry.get("id"),
|
id=str(entry.get("id") or item.url),
|
||||||
title=_name,
|
title=_name,
|
||||||
url=item.url,
|
url=item.url,
|
||||||
preset=item.preset,
|
preset=item.preset,
|
||||||
|
|
@ -401,6 +404,9 @@ async def add(
|
||||||
|
|
||||||
LOG.info(log_message)
|
LOG.info(log_message)
|
||||||
queue._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message)
|
queue._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message)
|
||||||
|
if dlInfo is None:
|
||||||
|
msg = "Download info is not available."
|
||||||
|
raise RuntimeError(msg)
|
||||||
queue._notify.emit(
|
queue._notify.emit(
|
||||||
Events.ITEM_MOVED,
|
Events.ITEM_MOVED,
|
||||||
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
|
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
|
||||||
|
|
|
||||||
|
|
@ -121,8 +121,8 @@ async def process_playlist(
|
||||||
|
|
||||||
max_downloads: int = -1
|
max_downloads: int = -1
|
||||||
ytdlp_opts: dict[str, Any] = item.get_ytdlp_opts().get_all()
|
ytdlp_opts: dict[str, Any] = item.get_ytdlp_opts().get_all()
|
||||||
if ytdlp_opts.get("max_downloads") and isinstance(ytdlp_opts.get("max_downloads"), int):
|
if isinstance(ytdlp_opts.get("max_downloads"), int):
|
||||||
max_downloads: int = ytdlp_opts.get("max_downloads")
|
max_downloads = ytdlp_opts["max_downloads"]
|
||||||
|
|
||||||
results: list[dict[str, str]] = []
|
results: list[dict[str, str]] = []
|
||||||
for i, etr in enumerate(entries, start=1):
|
for i, etr in enumerate(entries, start=1):
|
||||||
|
|
|
||||||
|
|
@ -240,7 +240,7 @@ class PoolManager:
|
||||||
await self.queue.done.put(entry)
|
await self.queue.done.put(entry)
|
||||||
self._notify.emit(
|
self._notify.emit(
|
||||||
Events.ITEM_MOVED,
|
Events.ITEM_MOVED,
|
||||||
data={"to": "history", "preset": entry.info.preset, "item": entry.info},
|
data={"from": "queue", "to": "history", "preset": entry.info.preset, "item": entry.info},
|
||||||
title=nTitle,
|
title=nTitle,
|
||||||
message=nMessage,
|
message=nMessage,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -99,15 +99,19 @@ class ProcessManager:
|
||||||
if not self.running():
|
if not self.running():
|
||||||
return False
|
return False
|
||||||
|
|
||||||
procId: int | None = self.proc.ident
|
proc = self.proc
|
||||||
|
if proc is None:
|
||||||
|
return False
|
||||||
|
|
||||||
|
procId: int | None = proc.ident
|
||||||
try:
|
try:
|
||||||
self.logger.info(
|
self.logger.info(
|
||||||
"Stopping download process PID=%s.",
|
"Stopping download process PID=%s.",
|
||||||
self.proc.pid,
|
proc.pid,
|
||||||
extra={
|
extra={
|
||||||
"download": {
|
"download": {
|
||||||
"download_id": self.download_id,
|
"download_id": self.download_id,
|
||||||
"process_id": self.proc.pid,
|
"process_id": proc.pid,
|
||||||
"process_ident": procId,
|
"process_ident": procId,
|
||||||
"is_live": self.is_live,
|
"is_live": self.is_live,
|
||||||
}
|
}
|
||||||
|
|
@ -117,25 +121,25 @@ class ProcessManager:
|
||||||
if self.is_live:
|
if self.is_live:
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
"Requesting graceful live cancellation for PID=%s.",
|
"Requesting graceful live cancellation for PID=%s.",
|
||||||
self.proc.pid,
|
proc.pid,
|
||||||
extra={
|
extra={
|
||||||
"download": {
|
"download": {
|
||||||
"download_id": self.download_id,
|
"download_id": self.download_id,
|
||||||
"process_id": self.proc.pid,
|
"process_id": proc.pid,
|
||||||
"is_live": self.is_live,
|
"is_live": self.is_live,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
self.cancel_event.set()
|
self.cancel_event.set()
|
||||||
|
|
||||||
if wait_for_process_with_timeout(self.proc, 10):
|
if wait_for_process_with_timeout(proc, 10):
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
"Download process PID=%s stopped gracefully.",
|
"Download process PID=%s stopped gracefully.",
|
||||||
self.proc.pid,
|
proc.pid,
|
||||||
extra={
|
extra={
|
||||||
"download": {
|
"download": {
|
||||||
"download_id": self.download_id,
|
"download_id": self.download_id,
|
||||||
"process_id": self.proc.pid,
|
"process_id": proc.pid,
|
||||||
"is_live": self.is_live,
|
"is_live": self.is_live,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
@ -143,43 +147,43 @@ class ProcessManager:
|
||||||
return True
|
return True
|
||||||
self.logger.warning(
|
self.logger.warning(
|
||||||
"Download process PID=%s did not respond to live cancellation; forcing termination.",
|
"Download process PID=%s did not respond to live cancellation; forcing termination.",
|
||||||
self.proc.pid,
|
proc.pid,
|
||||||
extra={
|
extra={
|
||||||
"download": {
|
"download": {
|
||||||
"download_id": self.download_id,
|
"download_id": self.download_id,
|
||||||
"process_id": self.proc.pid,
|
"process_id": proc.pid,
|
||||||
"is_live": self.is_live,
|
"is_live": self.is_live,
|
||||||
"force": True,
|
"force": True,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
elif self.proc.pid and "posix" == os.name:
|
elif proc.pid and "posix" == os.name:
|
||||||
signal_name = "SIGUSR1"
|
signal_name = "SIGUSR1"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
"Sending %s signal to download process PID=%s.",
|
"Sending %s signal to download process PID=%s.",
|
||||||
signal_name,
|
signal_name,
|
||||||
self.proc.pid,
|
proc.pid,
|
||||||
extra={
|
extra={
|
||||||
"download": {
|
"download": {
|
||||||
"download_id": self.download_id,
|
"download_id": self.download_id,
|
||||||
"process_id": self.proc.pid,
|
"process_id": proc.pid,
|
||||||
"signal": signal_name,
|
"signal": signal_name,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
os.kill(self.proc.pid, signal.SIGUSR1)
|
os.kill(proc.pid, signal.SIGUSR1)
|
||||||
|
|
||||||
if wait_for_process_with_timeout(self.proc, 5):
|
if wait_for_process_with_timeout(proc, 5):
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
"Download process PID=%s stopped gracefully.",
|
"Download process PID=%s stopped gracefully.",
|
||||||
self.proc.pid,
|
proc.pid,
|
||||||
extra={
|
extra={
|
||||||
"download": {
|
"download": {
|
||||||
"download_id": self.download_id,
|
"download_id": self.download_id,
|
||||||
"process_id": self.proc.pid,
|
"process_id": proc.pid,
|
||||||
"signal": signal_name,
|
"signal": signal_name,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
@ -187,12 +191,12 @@ class ProcessManager:
|
||||||
return True
|
return True
|
||||||
self.logger.warning(
|
self.logger.warning(
|
||||||
"Download process PID=%s did not respond to %s; forcing termination.",
|
"Download process PID=%s did not respond to %s; forcing termination.",
|
||||||
self.proc.pid,
|
proc.pid,
|
||||||
signal_name,
|
signal_name,
|
||||||
extra={
|
extra={
|
||||||
"download": {
|
"download": {
|
||||||
"download_id": self.download_id,
|
"download_id": self.download_id,
|
||||||
"process_id": self.proc.pid,
|
"process_id": proc.pid,
|
||||||
"signal": signal_name,
|
"signal": signal_name,
|
||||||
"force": True,
|
"force": True,
|
||||||
}
|
}
|
||||||
|
|
@ -203,51 +207,49 @@ class ProcessManager:
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
"Failed to send %s signal to download process PID=%s. %s",
|
"Failed to send %s signal to download process PID=%s. %s",
|
||||||
signal_name,
|
signal_name,
|
||||||
self.proc.pid,
|
proc.pid,
|
||||||
e,
|
e,
|
||||||
extra={
|
extra={
|
||||||
"download": {
|
"download": {
|
||||||
"download_id": self.download_id,
|
"download_id": self.download_id,
|
||||||
"process_id": self.proc.pid,
|
"process_id": proc.pid,
|
||||||
"signal": signal_name,
|
"signal": signal_name,
|
||||||
"exception_type": type(e).__name__,
|
"exception_type": type(e).__name__,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.proc.is_alive():
|
if proc.is_alive():
|
||||||
self.logger.info(
|
self.logger.info(
|
||||||
"Terminating download process PID=%s.",
|
"Terminating download process PID=%s.",
|
||||||
self.proc.pid,
|
proc.pid,
|
||||||
extra={"download": {"download_id": self.download_id, "process_id": self.proc.pid, "force": True}},
|
extra={"download": {"download_id": self.download_id, "process_id": proc.pid, "force": True}},
|
||||||
)
|
)
|
||||||
self.proc.terminate()
|
proc.terminate()
|
||||||
|
|
||||||
if not wait_for_process_with_timeout(self.proc, 1 if self.is_live else 2):
|
if not wait_for_process_with_timeout(proc, 1 if self.is_live else 2):
|
||||||
self.logger.warning(
|
self.logger.warning(
|
||||||
"Download process PID=%s did not terminate; killing forcefully.",
|
"Download process PID=%s did not terminate; killing forcefully.",
|
||||||
self.proc.pid,
|
proc.pid,
|
||||||
extra={
|
extra={"download": {"download_id": self.download_id, "process_id": proc.pid, "force": True}},
|
||||||
"download": {"download_id": self.download_id, "process_id": self.proc.pid, "force": True}
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
self.proc.kill()
|
proc.kill()
|
||||||
wait_for_process_with_timeout(self.proc, 1)
|
wait_for_process_with_timeout(proc, 1)
|
||||||
|
|
||||||
self.logger.info(
|
self.logger.info(
|
||||||
"Download process PID=%s stopped.",
|
"Download process PID=%s stopped.",
|
||||||
self.proc.pid,
|
proc.pid,
|
||||||
extra={"download": {"download_id": self.download_id, "process_id": self.proc.pid}},
|
extra={"download": {"download_id": self.download_id, "process_id": proc.pid}},
|
||||||
)
|
)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.exception(
|
self.logger.exception(
|
||||||
f"Failed to stop download process PID={self.proc.pid}, ident={procId}.",
|
f"Failed to stop download process PID={proc.pid if proc else None}, ident={procId}.",
|
||||||
extra={
|
extra={
|
||||||
"download": {
|
"download": {
|
||||||
"download_id": self.download_id,
|
"download_id": self.download_id,
|
||||||
"process_id": self.proc.pid,
|
"process_id": proc.pid if proc else None,
|
||||||
"process_ident": procId,
|
"process_ident": procId,
|
||||||
"is_live": self.is_live,
|
"is_live": self.is_live,
|
||||||
"exception_type": type(e).__name__,
|
"exception_type": type(e).__name__,
|
||||||
|
|
@ -271,11 +273,12 @@ class ProcessManager:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
self.cancel_in_progress = True
|
self.cancel_in_progress = True
|
||||||
procId: int | None = self.proc.ident
|
proc = self.proc
|
||||||
|
procId: int | None = proc.ident if proc else None
|
||||||
|
|
||||||
if not procId:
|
if not procId:
|
||||||
if self.proc:
|
if proc:
|
||||||
self.proc.close()
|
proc.close()
|
||||||
self.proc = None
|
self.proc = None
|
||||||
self.logger.warning("Attempted to close download process, but it is not running.")
|
self.logger.warning("Attempted to close download process, but it is not running.")
|
||||||
return False
|
return False
|
||||||
|
|
@ -293,13 +296,14 @@ class ProcessManager:
|
||||||
|
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
|
|
||||||
if self.proc.is_alive():
|
current = self.proc
|
||||||
|
if current is not None and current.is_alive():
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
"Waiting for download process PID='%s' to close.",
|
"Waiting for download process PID='%s' to close.",
|
||||||
procId,
|
procId,
|
||||||
extra={"download": {"download_id": self.download_id, "process_ident": procId}},
|
extra={"download": {"download_id": self.download_id, "process_ident": procId}},
|
||||||
)
|
)
|
||||||
await loop.run_in_executor(None, self.proc.join)
|
await loop.run_in_executor(None, current.join)
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
"Download process PID='%s' closed.",
|
"Download process PID='%s' closed.",
|
||||||
procId,
|
procId,
|
||||||
|
|
|
||||||
|
|
@ -53,7 +53,7 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
async def event_handler(_, __):
|
async def event_handler(_, __):
|
||||||
await self.initialize()
|
await self.initialize()
|
||||||
|
|
||||||
self._notify.subscribe(Events.STARTED, event_handler, f"{__class__.__name__}.{__class__.initialize.__name__}")
|
self._notify.subscribe(Events.STARTED, event_handler, f"{DownloadQueue.__name__}.initialize")
|
||||||
|
|
||||||
Scheduler.get_instance().add(
|
Scheduler.get_instance().add(
|
||||||
timer="* * * * *",
|
timer="* * * * *",
|
||||||
|
|
@ -320,7 +320,7 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
await self.done.put(item)
|
await self.done.put(item)
|
||||||
self._notify.emit(
|
self._notify.emit(
|
||||||
Events.ITEM_MOVED,
|
Events.ITEM_MOVED,
|
||||||
data={"to": "history", "preset": item.info.preset, "item": item.info},
|
data={"from": "queue", "to": "history", "preset": item.info.preset, "item": item.info},
|
||||||
title="Download Cancelled",
|
title="Download Cancelled",
|
||||||
message=f"Download '{item.info.title}' has been cancelled.",
|
message=f"Download '{item.info.title}' has been cancelled.",
|
||||||
)
|
)
|
||||||
|
|
@ -500,16 +500,15 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
if not ids:
|
if not ids:
|
||||||
return {"deleted": 0}
|
return {"deleted": 0}
|
||||||
|
|
||||||
items = await self.done.get_many_by_ids(ids)
|
items: list[tuple[str, Download]] = await self.done.get_many_by_ids(ids)
|
||||||
if not items:
|
if not items:
|
||||||
return {"deleted": 0}
|
return {"deleted": 0}
|
||||||
|
|
||||||
if self.config.remove_files is not True:
|
remove_file = False if self.config.remove_files is not True else remove_file
|
||||||
remove_file = False
|
|
||||||
|
|
||||||
removed_files = 0
|
removed_files = 0
|
||||||
deleted_ids: list[str] = []
|
deleted_ids: list[str] = []
|
||||||
deleted_titles: list[str] = []
|
item_summaries: list[dict] = []
|
||||||
|
|
||||||
for item_id, item in items:
|
for item_id, item in items:
|
||||||
filename: str = ""
|
filename: str = ""
|
||||||
|
|
@ -524,6 +523,8 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
"title": item.info.title,
|
"title": item.info.title,
|
||||||
"status": item.info.status,
|
"status": item.info.status,
|
||||||
"remove_file": remove_file,
|
"remove_file": remove_file,
|
||||||
|
"preset": item.info.preset,
|
||||||
|
"path": str(p) if (p := item.info.get_file()) else None,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
@ -555,7 +556,9 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
"download_id": item_id,
|
"download_id": item_id,
|
||||||
"item_id": item.info.id,
|
"item_id": item.info.id,
|
||||||
"title": item.info.title,
|
"title": item.info.title,
|
||||||
|
"preset": item.info.preset,
|
||||||
"filename": file_ref.name,
|
"filename": file_ref.name,
|
||||||
|
"path": str(file_ref),
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
@ -571,6 +574,8 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
"item_id": item.info.id,
|
"item_id": item.info.id,
|
||||||
"title": item.info.title,
|
"title": item.info.title,
|
||||||
"filename": rf.name,
|
"filename": rf.name,
|
||||||
|
"preset": item.info.preset,
|
||||||
|
"path": str(rf),
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
@ -587,6 +592,8 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
"item_id": item.info.id,
|
"item_id": item.info.id,
|
||||||
"title": item.info.title,
|
"title": item.info.title,
|
||||||
"filename": filename,
|
"filename": filename,
|
||||||
|
"preset": item.info.preset,
|
||||||
|
"path": str(rf),
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
@ -601,6 +608,7 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
"item_id": item.info.id,
|
"item_id": item.info.id,
|
||||||
"title": item.info.title,
|
"title": item.info.title,
|
||||||
"filename": filename,
|
"filename": filename,
|
||||||
|
"preset": item.info.preset,
|
||||||
"remove_file": remove_file,
|
"remove_file": remove_file,
|
||||||
"exception_type": type(e).__name__,
|
"exception_type": type(e).__name__,
|
||||||
}
|
}
|
||||||
|
|
@ -608,63 +616,80 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
)
|
)
|
||||||
|
|
||||||
deleted_ids.append(item_id)
|
deleted_ids.append(item_id)
|
||||||
deleted_titles.append(item.info.title or item.info.id or item_id)
|
item_summaries.append(
|
||||||
|
{
|
||||||
|
"id": item_id,
|
||||||
|
"title": item.info.title,
|
||||||
|
"url": item.info.url,
|
||||||
|
"preset": item.info.preset,
|
||||||
|
"path": str(p) if (p := item.info.get_file()) else None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
deleted_count = await self.done.bulk_delete(deleted_ids)
|
deleted_count: int = await self.done.bulk_delete(deleted_ids)
|
||||||
if deleted_count < 1:
|
if deleted_count < 1:
|
||||||
return {"deleted": 0}
|
return {"deleted": 0}
|
||||||
|
|
||||||
title = "History Removed" if removed_files > 0 else "History Cleared"
|
message: list[str] = [f"Cleared {deleted_count} item{'s' if deleted_count != 1 else ''} from history."]
|
||||||
message = f"Removed {deleted_count} item{'s' if deleted_count != 1 else ''} from history."
|
|
||||||
if removed_files > 0:
|
if removed_files > 0:
|
||||||
message += f" Also removed {removed_files} local file{'s' if removed_files != 1 else ''}."
|
message.append(f"Also removed {removed_files} local file{'s' if removed_files != 1 else ''}.")
|
||||||
|
|
||||||
self._notify.emit(
|
self._notify.emit(
|
||||||
Events.ITEM_BULK_DELETED,
|
Events.ITEM_BULK_DELETED,
|
||||||
data={"ids": deleted_ids, "count": deleted_count},
|
data={"count": deleted_count, "removed_files": removed_files, "items": item_summaries},
|
||||||
title=title,
|
title="History Cleared",
|
||||||
message=message,
|
message=" ".join(message),
|
||||||
)
|
)
|
||||||
|
|
||||||
summary = ", ".join(deleted_titles[:5])
|
|
||||||
if deleted_count > 5:
|
|
||||||
summary += ", ..."
|
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"Cleared %s history item(s), including %s.",
|
"Cleared %d history item(s).",
|
||||||
deleted_count,
|
deleted_count,
|
||||||
summary,
|
extra={"deleted_count": deleted_count, "removed_files": removed_files, "items": item_summaries},
|
||||||
extra={"deleted_count": deleted_count, "removed_files": removed_files, "titles": deleted_titles[:5]},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return {"deleted": deleted_count}
|
return {"deleted": deleted_count}
|
||||||
|
|
||||||
async def clear_by_status(self, status_filter: str, remove_file: bool = False) -> dict[str, int | str]:
|
async def clear_by_status(self, status_filter: str, remove_file: bool = False) -> dict[str, int | str]:
|
||||||
if self.config.remove_files is not True:
|
if not (items := await self.done.get_many_by_status(status_filter)):
|
||||||
remove_file = False
|
|
||||||
|
|
||||||
if not remove_file:
|
|
||||||
deleted_count = await self.done.bulk_delete_by_status(status_filter)
|
|
||||||
if deleted_count < 1:
|
|
||||||
return {"deleted": 0}
|
return {"deleted": 0}
|
||||||
|
|
||||||
|
remove_file = False if self.config.remove_files is not True else remove_file
|
||||||
|
|
||||||
|
if remove_file:
|
||||||
|
return await self.clear_bulk([item_id for item_id, _ in items], remove_file=remove_file)
|
||||||
|
|
||||||
|
deleted_count: int = await self.done.bulk_delete_by_status(status_filter)
|
||||||
|
item_summaries: list[dict[str, str | None]] = [
|
||||||
|
{
|
||||||
|
"id": item_id,
|
||||||
|
"title": dto.info.title,
|
||||||
|
"url": dto.info.url,
|
||||||
|
"preset": dto.info.preset,
|
||||||
|
"path": str(p) if (p := dto.info.get_file()) else None,
|
||||||
|
}
|
||||||
|
for item_id, dto in items
|
||||||
|
]
|
||||||
|
|
||||||
self._notify.emit(
|
self._notify.emit(
|
||||||
Events.ITEM_BULK_DELETED,
|
Events.ITEM_BULK_DELETED,
|
||||||
data={"count": deleted_count, "status": status_filter},
|
data={"count": deleted_count, "status": status_filter, "items": item_summaries},
|
||||||
title="History Cleared",
|
title="History Cleared",
|
||||||
message=f"Cleared {deleted_count} item{'s' if deleted_count != 1 else ''} from history.",
|
message=f"Cleared {deleted_count} item(s) from history.",
|
||||||
)
|
)
|
||||||
|
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"Cleared %s history item(s) with status '%s'.",
|
"Cleared %d history item(s) with status '%s'.",
|
||||||
deleted_count,
|
deleted_count,
|
||||||
status_filter,
|
status_filter,
|
||||||
extra={"deleted_count": deleted_count, "status_filter": status_filter},
|
extra={
|
||||||
|
"deleted_count": deleted_count,
|
||||||
|
"status_filter": status_filter,
|
||||||
|
"items": item_summaries,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
return {"deleted": deleted_count}
|
return {"deleted": deleted_count}
|
||||||
|
|
||||||
items = await self.done.get_many_by_status(status_filter)
|
async def get(self, mode: str = "all") -> dict[str, dict[str, ItemDTO]]:
|
||||||
return await self.clear_bulk([item_id for item_id, _ in items], remove_file=remove_file)
|
|
||||||
|
|
||||||
async def get(self, mode: str = "all") -> dict[str, list[dict[str, ItemDTO]]]:
|
|
||||||
"""
|
"""
|
||||||
Get the download queue and the download history.
|
Get the download queue and the download history.
|
||||||
|
|
||||||
|
|
@ -700,6 +725,32 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
|
|
||||||
return items
|
return items
|
||||||
|
|
||||||
|
def live_queue(self, limit: int = 0) -> dict[str, int | dict[str, ItemDTO]]:
|
||||||
|
"""Return a display-limited live queue snapshot with total metadata."""
|
||||||
|
limit = max(0, int(limit or 0))
|
||||||
|
items: dict[str, ItemDTO] = {}
|
||||||
|
active = self.pool.get_active_downloads()
|
||||||
|
|
||||||
|
for key, download in active.items():
|
||||||
|
if key in self.queue:
|
||||||
|
items[key] = download.info
|
||||||
|
|
||||||
|
for key, download in self.queue.items():
|
||||||
|
if limit > 0 and len(items) >= limit:
|
||||||
|
break
|
||||||
|
|
||||||
|
if key in items:
|
||||||
|
continue
|
||||||
|
|
||||||
|
items[key] = active[key].info if key in active else download.info
|
||||||
|
|
||||||
|
return {
|
||||||
|
"queue": items,
|
||||||
|
"queue_count": len(self.queue),
|
||||||
|
"queue_loaded": len(items),
|
||||||
|
"queue_limit": limit,
|
||||||
|
}
|
||||||
|
|
||||||
async def get_item(self, **kwargs) -> tuple["StoreType", Download] | tuple[None, None]:
|
async def get_item(self, **kwargs) -> tuple["StoreType", Download] | tuple[None, None]:
|
||||||
"""
|
"""
|
||||||
Get a specific item from the download queue or history.
|
Get a specific item from the download queue or history.
|
||||||
|
|
@ -708,7 +759,7 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
**kwargs: The key-value pair to search for. Supported keys are 'id', 'url'.
|
**kwargs: The key-value pair to search for. Supported keys are 'id', 'url'.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
(StoreType, Download) | None: The requested item if found, otherwise None.
|
tuple[StoreType, Download] | tuple[None, None]: The requested item if found, otherwise None.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
from app.library.DataStore import StoreType
|
from app.library.DataStore import StoreType
|
||||||
|
|
|
||||||
|
|
@ -55,7 +55,37 @@ class StatusTracker:
|
||||||
self.final_update = False
|
self.final_update = False
|
||||||
self._terminator_sent: bool = False
|
self._terminator_sent: bool = False
|
||||||
self._candidate_filepath: Path | None = None
|
self._candidate_filepath: Path | None = None
|
||||||
self.update_task: asyncio.Task | None = None
|
self.update_task: asyncio.Future[Any] | None = None
|
||||||
|
self._last_progress_time: float = 0.0
|
||||||
|
self._pending_progress: bool = False
|
||||||
|
self._progress_interval: float = 0.5
|
||||||
|
|
||||||
|
def _progress_payload(self) -> dict:
|
||||||
|
return {
|
||||||
|
"_id": self.info._id,
|
||||||
|
"status": self.info.status,
|
||||||
|
"percent": self.info.percent,
|
||||||
|
"speed": self.info.speed,
|
||||||
|
"eta": self.info.eta,
|
||||||
|
"downloaded_bytes": self.info.downloaded_bytes,
|
||||||
|
"total_bytes": self.info.total_bytes,
|
||||||
|
"msg": self.info.msg,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _emit_progress(self) -> None:
|
||||||
|
now = time.monotonic()
|
||||||
|
if (now - self._last_progress_time) >= self._progress_interval:
|
||||||
|
self._notify.emit(Events.ITEM_PROGRESS, data=self._progress_payload())
|
||||||
|
self._last_progress_time = now
|
||||||
|
self._pending_progress = False
|
||||||
|
else:
|
||||||
|
self._pending_progress = True
|
||||||
|
|
||||||
|
def _flush_progress(self) -> None:
|
||||||
|
if self._pending_progress:
|
||||||
|
self._notify.emit(Events.ITEM_PROGRESS, data=self._progress_payload())
|
||||||
|
self._pending_progress = False
|
||||||
|
self._last_progress_time = time.monotonic()
|
||||||
|
|
||||||
async def _finalize_file(self, filepath: Path) -> None:
|
async def _finalize_file(self, filepath: Path) -> None:
|
||||||
"""
|
"""
|
||||||
|
|
@ -184,6 +214,7 @@ class StatusTracker:
|
||||||
|
|
||||||
self.tmpfilename = status.get("tmpfilename")
|
self.tmpfilename = status.get("tmpfilename")
|
||||||
|
|
||||||
|
old_status = self.info.status
|
||||||
self.info.status = status.get("status", self.info.status)
|
self.info.status = status.get("status", self.info.status)
|
||||||
if "download_skipped" in status:
|
if "download_skipped" in status:
|
||||||
self.info.download_skipped = bool(status.get("download_skipped"))
|
self.info.download_skipped = bool(status.get("download_skipped"))
|
||||||
|
|
@ -225,7 +256,7 @@ class StatusTracker:
|
||||||
self.info.percent = status["downloaded_bytes"] / total * 100
|
self.info.percent = status["downloaded_bytes"] / total * 100
|
||||||
except ZeroDivisionError:
|
except ZeroDivisionError:
|
||||||
self.info.percent = 0
|
self.info.percent = 0
|
||||||
self.info.total_bytes = total
|
self.info.total_bytes = int(total)
|
||||||
|
|
||||||
self.info.speed = status.get("speed")
|
self.info.speed = status.get("speed")
|
||||||
self.info.eta = status.get("eta")
|
self.info.eta = status.get("eta")
|
||||||
|
|
@ -235,7 +266,11 @@ class StatusTracker:
|
||||||
await self._finalize_file(Path(final_name))
|
await self._finalize_file(Path(final_name))
|
||||||
self.info.status = "finished"
|
self.info.status = "finished"
|
||||||
|
|
||||||
|
if self.info.status != old_status or self.final_update:
|
||||||
|
self._flush_progress()
|
||||||
self._notify.emit(Events.ITEM_UPDATED, data=self.info)
|
self._notify.emit(Events.ITEM_UPDATED, data=self.info)
|
||||||
|
else:
|
||||||
|
self._emit_progress()
|
||||||
|
|
||||||
async def progress_update(self) -> None:
|
async def progress_update(self) -> None:
|
||||||
"""
|
"""
|
||||||
|
|
@ -245,12 +280,15 @@ class StatusTracker:
|
||||||
"""
|
"""
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
self.update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get)
|
update_task = asyncio.get_running_loop().run_in_executor(None, self.status_queue.get)
|
||||||
status = await self.update_task
|
self.update_task = asyncio.ensure_future(update_task)
|
||||||
|
status = await update_task
|
||||||
if status is None or isinstance(status, Terminator):
|
if status is None or isinstance(status, Terminator):
|
||||||
|
self._flush_progress()
|
||||||
return
|
return
|
||||||
await self.process_status_update(status)
|
await self.process_status_update(status)
|
||||||
except (asyncio.CancelledError, OSError, FileNotFoundError, EOFError, BrokenPipeError, ConnectionError):
|
except (asyncio.CancelledError, OSError, FileNotFoundError, EOFError, BrokenPipeError, ConnectionError):
|
||||||
|
self._flush_progress()
|
||||||
return
|
return
|
||||||
|
|
||||||
async def drain_queue(self, max_iterations: int = 50) -> None:
|
async def drain_queue(self, max_iterations: int = 50) -> None:
|
||||||
|
|
@ -295,6 +333,8 @@ class StatusTracker:
|
||||||
except (queue.Empty, BrokenPipeError, ConnectionRefusedError, EOFError, OSError):
|
except (queue.Empty, BrokenPipeError, ConnectionRefusedError, EOFError, OSError):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
self._flush_progress()
|
||||||
|
|
||||||
def cancel_update_task(self) -> None:
|
def cancel_update_task(self) -> None:
|
||||||
"""Cancel the progress update task if it's running."""
|
"""Cancel the progress update task if it's running."""
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import functools
|
import functools
|
||||||
|
import importlib.util
|
||||||
import threading
|
import threading
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Literal, cast, overload
|
from typing import Any, Literal, cast, overload
|
||||||
|
|
@ -98,12 +99,7 @@ def _normalize_proxy(proxy: str | dict[str, str] | None) -> str | None:
|
||||||
|
|
||||||
@functools.lru_cache(maxsize=1)
|
@functools.lru_cache(maxsize=1)
|
||||||
def _curl_available() -> bool:
|
def _curl_available() -> bool:
|
||||||
try:
|
return importlib.util.find_spec("httpx_curl_cffi") is not None
|
||||||
import httpx_curl_cffi # noqa: F401
|
|
||||||
|
|
||||||
return True
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_curl_transport(use_curl: bool = True) -> bool:
|
def resolve_curl_transport(use_curl: bool = True) -> bool:
|
||||||
|
|
@ -121,7 +117,7 @@ def _build_async_curl_transport(
|
||||||
from httpx_curl_cffi import AsyncCurlTransport
|
from httpx_curl_cffi import AsyncCurlTransport
|
||||||
|
|
||||||
return AsyncCurlTransport(
|
return AsyncCurlTransport(
|
||||||
impersonate=curl_impersonate,
|
impersonate=cast("Any", curl_impersonate),
|
||||||
default_headers=curl_default_headers,
|
default_headers=curl_default_headers,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -308,10 +304,7 @@ def get_async_client(
|
||||||
curl_default_headers=curl_default_headers,
|
curl_default_headers=curl_default_headers,
|
||||||
)
|
)
|
||||||
client = httpx.AsyncClient(
|
client = httpx.AsyncClient(
|
||||||
transport=cast(
|
transport=_get_transport(enable_cf, is_async=True, transport=transport),
|
||||||
"httpx.AsyncBaseTransport",
|
|
||||||
_get_transport(enable_cf, is_async=True, transport=transport),
|
|
||||||
),
|
|
||||||
proxy=cast("Any", proxy),
|
proxy=cast("Any", proxy),
|
||||||
)
|
)
|
||||||
Globals.SHARED_ASYNC_CLIENTS[key] = client
|
Globals.SHARED_ASYNC_CLIENTS[key] = client
|
||||||
|
|
@ -330,10 +323,7 @@ def get_sync_client(
|
||||||
return Globals.SHARED_SYNC_CLIENTS[key]
|
return Globals.SHARED_SYNC_CLIENTS[key]
|
||||||
|
|
||||||
client = httpx.Client(
|
client = httpx.Client(
|
||||||
transport=cast(
|
transport=_get_transport(enable_cf, is_async=False, transport=None),
|
||||||
"httpx.BaseTransport",
|
|
||||||
_get_transport(enable_cf, is_async=False, transport=None),
|
|
||||||
),
|
|
||||||
proxy=cast("Any", proxy),
|
proxy=cast("Any", proxy),
|
||||||
)
|
)
|
||||||
Globals.SHARED_SYNC_CLIENTS[key] = client
|
Globals.SHARED_SYNC_CLIENTS[key] = client
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,3 @@
|
||||||
# flake8: noqa: S608
|
|
||||||
"""
|
"""
|
||||||
This module is a fork of the Caribou library
|
This module is a fork of the Caribou library
|
||||||
(https://pypi.org/project/caribou/) modified to work for our specific use case.
|
(https://pypi.org/project/caribou/) modified to work for our specific use case.
|
||||||
|
|
@ -9,10 +8,11 @@ import datetime
|
||||||
import glob
|
import glob
|
||||||
import os.path
|
import os.path
|
||||||
import traceback
|
import traceback
|
||||||
from collections.abc import AsyncIterator, Iterable, Sequence
|
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
|
||||||
from importlib.machinery import ModuleSpec, SourceFileLoader
|
from importlib.machinery import ModuleSpec, SourceFileLoader
|
||||||
from importlib.util import module_from_spec, spec_from_loader
|
from importlib.util import module_from_spec, spec_from_loader
|
||||||
from typing import TYPE_CHECKING
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
from sqlalchemy.ext.asyncio import AsyncConnection
|
from sqlalchemy.ext.asyncio import AsyncConnection
|
||||||
|
|
@ -44,7 +44,9 @@ class InvalidNameError(Error):
|
||||||
|
|
||||||
|
|
||||||
@contextlib.asynccontextmanager
|
@contextlib.asynccontextmanager
|
||||||
async def execute(conn: AsyncConnection, sql: str, params: Sequence[object] | None = None) -> AsyncIterator:
|
async def execute(
|
||||||
|
conn: AsyncConnection, sql: str, params: Mapping[str, Any] | Sequence[Mapping[str, Any]] | None = None
|
||||||
|
) -> AsyncIterator:
|
||||||
params = {} if params is None else params
|
params = {} if params is None else params
|
||||||
result = await conn.execute(text(sql), params)
|
result = await conn.execute(text(sql), params)
|
||||||
try:
|
try:
|
||||||
|
|
@ -126,12 +128,12 @@ class Database:
|
||||||
|
|
||||||
self.version_table: str = version_table
|
self.version_table: str = version_table
|
||||||
|
|
||||||
self._owns_connection = bool(isinstance(db_url, str))
|
self._owns_connection = isinstance(db_url, str)
|
||||||
if self._owns_connection:
|
if isinstance(db_url, str):
|
||||||
self.conn: AsyncConnection | None = None
|
self.conn: AsyncConnection | None = None
|
||||||
self.db_url: str = db_url
|
self.db_url: str = db_url
|
||||||
else:
|
else:
|
||||||
self.conn: AsyncConnection = db_url
|
self.conn = db_url
|
||||||
|
|
||||||
async def __aenter__(self):
|
async def __aenter__(self):
|
||||||
if self._owns_connection and self.conn is None:
|
if self._owns_connection and self.conn is None:
|
||||||
|
|
@ -183,7 +185,8 @@ class Database:
|
||||||
async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None:
|
async def downgrade(self, migrations: list[Migration], target_version: str | int) -> None:
|
||||||
assert self.conn
|
assert self.conn
|
||||||
if target_version not in (0, "0"):
|
if target_version not in (0, "0"):
|
||||||
_assert_migration_exists(migrations, target_version)
|
_assert_migration_exists(migrations, str(target_version))
|
||||||
|
target = str(target_version)
|
||||||
|
|
||||||
migrations.sort(key=lambda x: x.get_version(), reverse=True)
|
migrations.sort(key=lambda x: x.get_version(), reverse=True)
|
||||||
database_version: str | None = await self.get_version()
|
database_version: str | None = await self.get_version()
|
||||||
|
|
@ -192,7 +195,7 @@ class Database:
|
||||||
current_version: str = migration.get_version()
|
current_version: str = migration.get_version()
|
||||||
if database_version is not None and current_version > database_version:
|
if database_version is not None and current_version > database_version:
|
||||||
continue
|
continue
|
||||||
if current_version <= target_version:
|
if current_version <= target:
|
||||||
break
|
break
|
||||||
await migration.downgrade(self.conn)
|
await migration.downgrade(self.conn)
|
||||||
next_version: str | int = 0
|
next_version: str | int = 0
|
||||||
|
|
@ -210,14 +213,14 @@ class Database:
|
||||||
assert self.conn
|
assert self.conn
|
||||||
if not await self.is_version_controlled():
|
if not await self.is_version_controlled():
|
||||||
return None
|
return None
|
||||||
sql: str = f"SELECT version FROM {self.version_table}"
|
sql: str = f"SELECT version FROM {self.version_table}" # noqa: S608
|
||||||
async with execute(self.conn, sql) as result:
|
async with execute(self.conn, sql) as result:
|
||||||
rows = result.fetchall()
|
rows = result.fetchall()
|
||||||
return rows[0][0] if rows else "0"
|
return rows[0][0] if rows else "0"
|
||||||
|
|
||||||
async def update_version(self, version: str) -> None:
|
async def update_version(self, version: str) -> None:
|
||||||
assert self.conn
|
assert self.conn
|
||||||
sql: str = f"UPDATE {self.version_table} SET version = :version"
|
sql: str = f"UPDATE {self.version_table} SET version = :version" # noqa: S608
|
||||||
async with transaction(self.conn):
|
async with transaction(self.conn):
|
||||||
await self.conn.execute(text(sql), {"version": version})
|
await self.conn.execute(text(sql), {"version": version})
|
||||||
|
|
||||||
|
|
@ -226,7 +229,7 @@ class Database:
|
||||||
sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """
|
sql: str = f"""CREATE TABLE IF NOT EXISTS {self.version_table} ( version TEXT ) """
|
||||||
async with transaction(self.conn):
|
async with transaction(self.conn):
|
||||||
await self.conn.execute(text(sql))
|
await self.conn.execute(text(sql))
|
||||||
await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"})
|
await self.conn.execute(text(f"INSERT INTO {self.version_table} VALUES (:version)"), {"version": "0"}) # noqa: S608
|
||||||
|
|
||||||
def __repr__(self) -> str:
|
def __repr__(self) -> str:
|
||||||
return f'Database("{self.db_url if self._owns_connection else "external_connection"}")'
|
return f'Database("{self.db_url if self._owns_connection else "external_connection"}")'
|
||||||
|
|
@ -238,7 +241,7 @@ def _assert_migration_exists(migrations: Iterable[Migration], version: str | int
|
||||||
raise Error(msg)
|
raise Error(msg)
|
||||||
|
|
||||||
|
|
||||||
def load_migrations(directory: str) -> list[Migration]:
|
def load_migrations(directory: str | Path) -> list[Migration]:
|
||||||
"""Return the migrations contained in the given directory."""
|
"""Return the migrations contained in the given directory."""
|
||||||
directory = str(directory)
|
directory = str(directory)
|
||||||
if not os.path.exists(directory) or not os.path.isdir(directory):
|
if not os.path.exists(directory) or not os.path.isdir(directory):
|
||||||
|
|
@ -247,7 +250,7 @@ def load_migrations(directory: str) -> list[Migration]:
|
||||||
return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))]
|
return [Migration(f) for f in glob.glob(os.path.join(directory, "*.py"))]
|
||||||
|
|
||||||
|
|
||||||
async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: str | None = None) -> None:
|
async def upgrade(db_url: AsyncConnection | str, migration_dir: str | Path, version: str | None = None) -> None:
|
||||||
"""
|
"""
|
||||||
Upgrade the given database with the migrations contained in the
|
Upgrade the given database with the migrations contained in the
|
||||||
migrations directory. If a version is not specified, upgrade
|
migrations directory. If a version is not specified, upgrade
|
||||||
|
|
@ -260,7 +263,7 @@ async def upgrade(db_url: AsyncConnection | str, migration_dir: str, version: st
|
||||||
await db.upgrade(load_migrations(migration_dir), version)
|
await db.upgrade(load_migrations(migration_dir), version)
|
||||||
|
|
||||||
|
|
||||||
async def downgrade(db_url: str | AsyncConnection, migration_dir: str, version: str) -> None:
|
async def downgrade(db_url: str | AsyncConnection, migration_dir: str | Path, version: str) -> None:
|
||||||
"""
|
"""
|
||||||
Downgrade the database to the given version with the migrations
|
Downgrade the database to the given version with the migrations
|
||||||
contained in the given migration directory.
|
contained in the given migration directory.
|
||||||
|
|
|
||||||
|
|
@ -2,14 +2,16 @@ import asyncio
|
||||||
import contextlib
|
import contextlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable, Mapping
|
||||||
from dataclasses import fields
|
from dataclasses import fields
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from email.utils import formatdate
|
from email.utils import formatdate
|
||||||
|
from typing import Any
|
||||||
from urllib.parse import quote_plus
|
from urllib.parse import quote_plus
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.engine.row import RowMapping
|
||||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||||
from sqlalchemy.ext.asyncio.engine import AsyncConnection
|
from sqlalchemy.ext.asyncio.engine import AsyncConnection
|
||||||
|
|
||||||
|
|
@ -36,13 +38,13 @@ def _memory_db_url(db_path: str) -> str:
|
||||||
return f"sqlite+aiosqlite:///file:{quoted_name}?mode=memory&cache=shared&uri=true"
|
return f"sqlite+aiosqlite:///file:{quoted_name}?mode=memory&cache=shared&uri=true"
|
||||||
|
|
||||||
|
|
||||||
def _decode_row(row: dict) -> ItemDTO:
|
def _decode_row(row: Mapping[str, Any] | RowMapping) -> ItemDTO:
|
||||||
row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S") # noqa: DTZ007
|
row_date = datetime.strptime(row["created_at"], "%Y-%m-%d %H:%M:%S").replace(tzinfo=UTC)
|
||||||
data = json.loads(row["data"])
|
data = json.loads(row["data"])
|
||||||
data.pop("_id", None)
|
data.pop("_id", None)
|
||||||
item = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
|
item = init_class(ItemDTO, data, ITEM_DTO_FIELDS)
|
||||||
item._id = row["id"]
|
item._id = row["id"]
|
||||||
item.datetime = formatdate(row_date.replace(tzinfo=UTC).timestamp())
|
item.datetime = formatdate(row_date.timestamp())
|
||||||
return item
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -149,8 +151,8 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
return self._sessionmaker
|
return self._sessionmaker
|
||||||
|
|
||||||
async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]:
|
async def fetch_saved(self, type_value: str) -> list[tuple[str, ItemDTO]]:
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
result = await self._conn.execute(
|
result = await conn.execute(
|
||||||
text(
|
text(
|
||||||
'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value ORDER BY "created_at" ASC'
|
'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value ORDER BY "created_at" ASC'
|
||||||
),
|
),
|
||||||
|
|
@ -163,12 +165,12 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
async def exists(self, type_value: str, key: str | None = None, url: str | None = None) -> bool:
|
async def exists(self, type_value: str, key: str | None = None, url: str | None = None) -> bool:
|
||||||
return await self.get(type_value, key=key, url=url) is not None
|
return await self.get(type_value, key=key, url=url) is not None
|
||||||
|
|
||||||
async def get(self, type_value: str, key: str | None = None, url: str | None = None) -> bool:
|
async def get(self, type_value: str, key: str | None = None, url: str | None = None) -> ItemDTO | None:
|
||||||
if not key and not url:
|
if not key and not url:
|
||||||
msg = "key or url must be provided."
|
msg = "key or url must be provided."
|
||||||
raise KeyError(msg)
|
raise KeyError(msg)
|
||||||
|
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
|
|
||||||
clauses: list[str] = []
|
clauses: list[str] = []
|
||||||
params: dict[str, str] = {"type_value": type_value}
|
params: dict[str, str] = {"type_value": type_value}
|
||||||
|
|
@ -186,7 +188,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) LIMIT 1' # noqa: S608
|
f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) LIMIT 1' # noqa: S608
|
||||||
)
|
)
|
||||||
|
|
||||||
result = await self._conn.execute(text(query), params)
|
result = await conn.execute(text(query), params)
|
||||||
row = result.mappings().first()
|
row = result.mappings().first()
|
||||||
|
|
||||||
if not row:
|
if not row:
|
||||||
|
|
@ -195,8 +197,8 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
return _decode_row(row)
|
return _decode_row(row)
|
||||||
|
|
||||||
async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None:
|
async def get_by_id(self, type_value: str, id: str) -> ItemDTO | None:
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
result = await self._conn.execute(
|
result = await conn.execute(
|
||||||
text('SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" = :id'),
|
text('SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" = :id'),
|
||||||
{"type_value": type_value, "id": id},
|
{"type_value": type_value, "id": id},
|
||||||
)
|
)
|
||||||
|
|
@ -212,24 +214,24 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
if not ids_list:
|
if not ids_list:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
placeholders = ", ".join(f":id_{index}" for index in range(len(ids_list)))
|
placeholders = ", ".join(f":id_{index}" for index in range(len(ids_list)))
|
||||||
params: dict[str, str] = {"type_value": type_value}
|
params: dict[str, str] = {"type_value": type_value}
|
||||||
params.update({f"id_{index}": item_id for index, item_id in enumerate(ids_list)})
|
params.update({f"id_{index}": item_id for index, item_id in enumerate(ids_list)})
|
||||||
|
|
||||||
result = await self._conn.execute(
|
result = await conn.execute(
|
||||||
text(
|
text(
|
||||||
f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})' # noqa: S608
|
f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})' # noqa: S608
|
||||||
),
|
),
|
||||||
params,
|
params,
|
||||||
)
|
)
|
||||||
rows = result.mappings().all()
|
rows = list(result.mappings().all())
|
||||||
order = {item_id: index for index, item_id in enumerate(ids_list)}
|
order = {item_id: index for index, item_id in enumerate(ids_list)}
|
||||||
rows.sort(key=lambda row: order.get(row["id"], len(ids_list)))
|
rows.sort(key=lambda row: order.get(row["id"], len(ids_list)))
|
||||||
return [(row["id"], _decode_row(row)) for row in rows]
|
return [(row["id"], _decode_row(row)) for row in rows]
|
||||||
|
|
||||||
async def get_many_by_status(self, type_value: str, status_filter: str) -> list[tuple[str, ItemDTO]]:
|
async def get_many_by_status(self, type_value: str, status_filter: str) -> list[tuple[str, ItemDTO]]:
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
params: dict[str, str] = {"type_value": type_value}
|
params: dict[str, str] = {"type_value": type_value}
|
||||||
where_clauses = ['"type" = :type_value']
|
where_clauses = ['"type" = :type_value']
|
||||||
|
|
||||||
|
|
@ -239,7 +241,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
params.update(status_params)
|
params.update(status_params)
|
||||||
|
|
||||||
query = f'SELECT "id", "data", "created_at" FROM "history" WHERE {" AND ".join(where_clauses)} ORDER BY "created_at" DESC' # noqa: S608
|
query = f'SELECT "id", "data", "created_at" FROM "history" WHERE {" AND ".join(where_clauses)} ORDER BY "created_at" DESC' # noqa: S608
|
||||||
result = await self._conn.execute(text(query), params)
|
result = await conn.execute(text(query), params)
|
||||||
rows = result.mappings().all()
|
rows = result.mappings().all()
|
||||||
return [(row["id"], _decode_row(row)) for row in rows]
|
return [(row["id"], _decode_row(row)) for row in rows]
|
||||||
|
|
||||||
|
|
@ -254,7 +256,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
if not kwargs:
|
if not kwargs:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
|
|
||||||
clauses: list[str] = []
|
clauses: list[str] = []
|
||||||
params: dict[str, str | float | int] = {"type_value": type_value}
|
params: dict[str, str | float | int] = {"type_value": type_value}
|
||||||
|
|
@ -321,7 +323,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
where_clause: str = " OR ".join(f"({clause})" for clause in clauses)
|
where_clause: str = " OR ".join(f"({clause})" for clause in clauses)
|
||||||
query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) ORDER BY "created_at" ASC LIMIT 1' # noqa: S608
|
query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = :type_value AND ({where_clause}) ORDER BY "created_at" ASC LIMIT 1' # noqa: S608
|
||||||
|
|
||||||
result = await self._conn.execute(text(query), params)
|
result = await conn.execute(text(query), params)
|
||||||
row = result.mappings().first()
|
row = result.mappings().first()
|
||||||
|
|
||||||
if not row:
|
if not row:
|
||||||
|
|
@ -332,7 +334,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
return item if any(matches_condition(k, v, item.__dict__) for k, v in kwargs.items()) else None
|
return item if any(matches_condition(k, v, item.__dict__) for k, v in kwargs.items()) else None
|
||||||
|
|
||||||
async def count(self, type_value: str, status_filter: str | None = None) -> int:
|
async def count(self, type_value: str, status_filter: str | None = None) -> int:
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
where_clauses: list[str] = ['"type" = :type_value']
|
where_clauses: list[str] = ['"type" = :type_value']
|
||||||
params: dict[str, str] = {"type_value": type_value}
|
params: dict[str, str] = {"type_value": type_value}
|
||||||
|
|
||||||
|
|
@ -344,7 +346,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
where_clause: str = " AND ".join(where_clauses)
|
where_clause: str = " AND ".join(where_clauses)
|
||||||
count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
|
count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
|
||||||
|
|
||||||
result = await self._conn.execute(text(count_query), params)
|
result = await conn.execute(text(count_query), params)
|
||||||
row = result.mappings().first()
|
row = result.mappings().first()
|
||||||
|
|
||||||
return row["count"] if row else 0
|
return row["count"] if row else 0
|
||||||
|
|
@ -357,7 +359,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
order: str,
|
order: str,
|
||||||
status_filter: str | None = None,
|
status_filter: str | None = None,
|
||||||
) -> tuple[list[tuple[str, ItemDTO]], int, int, int]:
|
) -> tuple[list[tuple[str, ItemDTO]], int, int, int]:
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
where_clauses: list[str] = ['"type" = :type_value']
|
where_clauses: list[str] = ['"type" = :type_value']
|
||||||
params: dict[str, str | int] = {"type_value": type_value}
|
params: dict[str, str | int] = {"type_value": type_value}
|
||||||
|
|
||||||
|
|
@ -369,7 +371,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
where_clause: str = " AND ".join(where_clauses)
|
where_clause: str = " AND ".join(where_clauses)
|
||||||
count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
|
count_query: str = f'SELECT COUNT(*) as count FROM "history" WHERE {where_clause}' # noqa: S608
|
||||||
|
|
||||||
result = await self._conn.execute(text(count_query), params)
|
result = await conn.execute(text(count_query), params)
|
||||||
count_row = result.mappings().first()
|
count_row = result.mappings().first()
|
||||||
|
|
||||||
total_items = count_row["count"] if count_row else 0
|
total_items = count_row["count"] if count_row else 0
|
||||||
|
|
@ -384,7 +386,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
|
|
||||||
query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT :limit OFFSET :offset' # noqa: S608
|
query: str = f'SELECT "id", "data", "created_at" FROM "history" WHERE {where_clause} ORDER BY "created_at" {order} LIMIT :limit OFFSET :offset' # noqa: S608
|
||||||
|
|
||||||
result = await self._conn.execute(text(query), params)
|
result = await conn.execute(text(query), params)
|
||||||
rows = result.mappings().all()
|
rows = result.mappings().all()
|
||||||
|
|
||||||
items = [(row["id"], _decode_row(row)) for row in rows]
|
items = [(row["id"], _decode_row(row)) for row in rows]
|
||||||
|
|
@ -396,15 +398,15 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
await self._upsert_now(type_value, item)
|
await self._upsert_now(type_value, item)
|
||||||
|
|
||||||
async def delete(self, type_value: str, key: str) -> None:
|
async def delete(self, type_value: str, key: str) -> None:
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
await self._conn.execute(
|
await conn.execute(
|
||||||
text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'),
|
text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'),
|
||||||
{"type_value": type_value, "key": key},
|
{"type_value": type_value, "key": key},
|
||||||
)
|
)
|
||||||
await self._conn.commit()
|
await conn.commit()
|
||||||
|
|
||||||
async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int:
|
async def bulk_delete(self, type_value: str, keys: Iterable[str]) -> int:
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
keys_list: list[str] = list(keys)
|
keys_list: list[str] = list(keys)
|
||||||
if not keys_list:
|
if not keys_list:
|
||||||
return 0
|
return 0
|
||||||
|
|
@ -412,15 +414,15 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
params: dict[str, str] = {"type_value": type_value}
|
params: dict[str, str] = {"type_value": type_value}
|
||||||
params.update({f"key_{i}": key for i, key in enumerate(keys_list)})
|
params.update({f"key_{i}": key for i, key in enumerate(keys_list)})
|
||||||
|
|
||||||
result = await self._conn.execute(
|
result = await conn.execute(
|
||||||
text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608
|
text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608
|
||||||
params,
|
params,
|
||||||
)
|
)
|
||||||
await self._conn.commit()
|
await conn.commit()
|
||||||
return result.rowcount if result else 0
|
return result.rowcount if result else 0
|
||||||
|
|
||||||
async def bulk_delete_by_status(self, type_value: str, status_filter: str) -> int:
|
async def bulk_delete_by_status(self, type_value: str, status_filter: str) -> int:
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
params: dict[str, str] = {"type_value": type_value}
|
params: dict[str, str] = {"type_value": type_value}
|
||||||
where_clauses = ['"type" = :type_value']
|
where_clauses = ['"type" = :type_value']
|
||||||
|
|
||||||
|
|
@ -429,11 +431,11 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
where_clauses.append(status_clause)
|
where_clauses.append(status_clause)
|
||||||
params.update(status_params)
|
params.update(status_params)
|
||||||
|
|
||||||
result = await self._conn.execute(
|
result = await conn.execute(
|
||||||
text(f'DELETE FROM "history" WHERE {" AND ".join(where_clauses)}'), # noqa: S608
|
text(f'DELETE FROM "history" WHERE {" AND ".join(where_clauses)}'), # noqa: S608
|
||||||
params,
|
params,
|
||||||
)
|
)
|
||||||
await self._conn.commit()
|
await conn.commit()
|
||||||
return result.rowcount if result else 0
|
return result.rowcount if result else 0
|
||||||
|
|
||||||
async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None:
|
async def enqueue_upsert(self, type_value: str, item: ItemDTO) -> None:
|
||||||
|
|
@ -484,6 +486,9 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
|
|
||||||
async def _enqueue(self, op: _Op) -> None:
|
async def _enqueue(self, op: _Op) -> None:
|
||||||
self._ensure_worker()
|
self._ensure_worker()
|
||||||
|
if self._queue is None:
|
||||||
|
msg = "Queue was not initialized by _ensure_worker."
|
||||||
|
raise RuntimeError(msg)
|
||||||
await self._queue.put(op)
|
await self._queue.put(op)
|
||||||
|
|
||||||
def _ensure_worker(self) -> None:
|
def _ensure_worker(self) -> None:
|
||||||
|
|
@ -492,10 +497,14 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
self._task = asyncio.create_task(self._writer(), name="sqlite-store-writer")
|
self._task = asyncio.create_task(self._writer(), name="sqlite-store-writer")
|
||||||
|
|
||||||
async def _writer(self) -> None:
|
async def _writer(self) -> None:
|
||||||
|
queue = self._queue
|
||||||
|
if queue is None:
|
||||||
|
msg = "Writer started before queue was initialized."
|
||||||
|
raise RuntimeError(msg)
|
||||||
while True:
|
while True:
|
||||||
op = await self._queue.get()
|
op = await queue.get()
|
||||||
if isinstance(op.op, Terminator):
|
if isinstance(op.op, Terminator):
|
||||||
self._queue.task_done()
|
queue.task_done()
|
||||||
break
|
break
|
||||||
try:
|
try:
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
|
|
@ -507,34 +516,34 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
extra={"operation": op.op, "type_value": op.type_value, "exception_type": type(ex).__name__},
|
extra={"operation": op.op, "type_value": op.type_value, "exception_type": type(ex).__name__},
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
self._queue.task_done()
|
queue.task_done()
|
||||||
await asyncio.sleep(self._flush_interval)
|
await asyncio.sleep(self._flush_interval)
|
||||||
|
|
||||||
async def _apply(self, op: _Op) -> None:
|
async def _apply(self, op: _Op) -> None:
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
if op.op == "upsert" and op.item:
|
if op.op == "upsert" and op.item:
|
||||||
await self._upsert_now_conn(self._conn, op.type_value, op.item)
|
await self._upsert_now_conn(conn, op.type_value, op.item)
|
||||||
await self._conn.commit()
|
await conn.commit()
|
||||||
elif op.op == "delete" and op.key:
|
elif op.op == "delete" and op.key:
|
||||||
await self._conn.execute(
|
await conn.execute(
|
||||||
text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'),
|
text('DELETE FROM "history" WHERE "type" = :type_value AND "id" = :key'),
|
||||||
{"type_value": op.type_value, "key": op.key},
|
{"type_value": op.type_value, "key": op.key},
|
||||||
)
|
)
|
||||||
await self._conn.commit()
|
await conn.commit()
|
||||||
elif op.op == "bulk_delete" and op.keys:
|
elif op.op == "bulk_delete" and op.keys:
|
||||||
placeholders = ",".join(f":key_{i}" for i in range(len(op.keys)))
|
placeholders = ",".join(f":key_{i}" for i in range(len(op.keys)))
|
||||||
params: dict[str, str] = {"type_value": op.type_value}
|
params: dict[str, str] = {"type_value": op.type_value}
|
||||||
params.update({f"key_{i}": key for i, key in enumerate(op.keys)})
|
params.update({f"key_{i}": key for i, key in enumerate(op.keys)})
|
||||||
await self._conn.execute(
|
await conn.execute(
|
||||||
text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608
|
text(f'DELETE FROM "history" WHERE "type" = :type_value AND "id" IN ({placeholders})'), # noqa: S608
|
||||||
params,
|
params,
|
||||||
)
|
)
|
||||||
await self._conn.commit()
|
await conn.commit()
|
||||||
|
|
||||||
async def _upsert_now(self, type_value: str, item: ItemDTO) -> None:
|
async def _upsert_now(self, type_value: str, item: ItemDTO) -> None:
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
await self._upsert_now_conn(self._conn, type_value, item)
|
await self._upsert_now_conn(conn, type_value, item)
|
||||||
await self._conn.commit()
|
await conn.commit()
|
||||||
|
|
||||||
async def _upsert_now_conn(self, conn, type_value: str, item: ItemDTO) -> None:
|
async def _upsert_now_conn(self, conn, type_value: str, item: ItemDTO) -> None:
|
||||||
"""Helper to upsert using an existing connection."""
|
"""Helper to upsert using an existing connection."""
|
||||||
|
|
@ -560,7 +569,7 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
|
|
||||||
async def execute_raw(self, query: str, params: dict | tuple | None = None) -> None:
|
async def execute_raw(self, query: str, params: dict | tuple | None = None) -> None:
|
||||||
"""Execute raw SQL query (for testing purposes)."""
|
"""Execute raw SQL query (for testing purposes)."""
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
if isinstance(params, tuple):
|
if isinstance(params, tuple):
|
||||||
# Convert positional params to dict for SQLAlchemy
|
# Convert positional params to dict for SQLAlchemy
|
||||||
# Assuming queries use ? placeholders, we need to count them
|
# Assuming queries use ? placeholders, we need to count them
|
||||||
|
|
@ -575,16 +584,16 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
# Replace ? with :p0, :p1, etc.
|
# Replace ? with :p0, :p1, etc.
|
||||||
for i in range(len(params)):
|
for i in range(len(params)):
|
||||||
query = query.replace("?", f":p{i}", 1)
|
query = query.replace("?", f":p{i}", 1)
|
||||||
await self._conn.execute(text(query), param_dict)
|
await conn.execute(text(query), param_dict)
|
||||||
elif isinstance(params, dict):
|
elif isinstance(params, dict):
|
||||||
await self._conn.execute(text(query), params)
|
await conn.execute(text(query), params)
|
||||||
else:
|
else:
|
||||||
await self._conn.execute(text(query))
|
await conn.execute(text(query))
|
||||||
await self._conn.commit()
|
await conn.commit()
|
||||||
|
|
||||||
async def fetch_raw(self, query: str, params: dict | tuple | None = None):
|
async def fetch_raw(self, query: str, params: dict | tuple | None = None):
|
||||||
"""Fetch results from raw SQL query (for testing purposes)."""
|
"""Fetch results from raw SQL query (for testing purposes)."""
|
||||||
await self.get_connection()
|
conn = await self.get_connection()
|
||||||
if isinstance(params, tuple):
|
if isinstance(params, tuple):
|
||||||
# Convert positional params to dict for SQLAlchemy
|
# Convert positional params to dict for SQLAlchemy
|
||||||
placeholders = query.count("?")
|
placeholders = query.count("?")
|
||||||
|
|
@ -596,11 +605,11 @@ class SqliteStore(metaclass=ThreadSafe):
|
||||||
param_dict = {f"p{i}": params[i] for i in range(len(params))}
|
param_dict = {f"p{i}": params[i] for i in range(len(params))}
|
||||||
for i in range(len(params)):
|
for i in range(len(params)):
|
||||||
query = query.replace("?", f":p{i}", 1)
|
query = query.replace("?", f":p{i}", 1)
|
||||||
result = await self._conn.execute(text(query), param_dict)
|
result = await conn.execute(text(query), param_dict)
|
||||||
elif isinstance(params, dict):
|
elif isinstance(params, dict):
|
||||||
result = await self._conn.execute(text(query), params)
|
result = await conn.execute(text(query), params)
|
||||||
else:
|
else:
|
||||||
result = await self._conn.execute(text(query))
|
result = await conn.execute(text(query))
|
||||||
return result.mappings().all()
|
return result.mappings().all()
|
||||||
|
|
||||||
async def get_connection(self) -> AsyncConnection:
|
async def get_connection(self) -> AsyncConnection:
|
||||||
|
|
|
||||||
23
app/local.py
23
app/local.py
|
|
@ -1,14 +1,14 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# flake8: noqa: S310 T201
|
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import http.client
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
import socket
|
import socket
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import urllib.request
|
|
||||||
import webbrowser
|
import webbrowser
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
from multiprocessing import freeze_support
|
from multiprocessing import freeze_support
|
||||||
|
|
@ -26,7 +26,7 @@ APP_ROOT = str((pathlib.Path(__file__).parent / "..").resolve())
|
||||||
if APP_ROOT not in sys.path:
|
if APP_ROOT not in sys.path:
|
||||||
sys.path.insert(0, APP_ROOT)
|
sys.path.insert(0, APP_ROOT)
|
||||||
|
|
||||||
import platformdirs # type: ignore
|
import platformdirs
|
||||||
|
|
||||||
|
|
||||||
def set_env():
|
def set_env():
|
||||||
|
|
@ -55,18 +55,27 @@ def open_browser_when_ready(url: str, timeout: float = 5.0) -> None:
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
def wait_then_open():
|
def wait_then_open():
|
||||||
|
parsed = urlsplit(url)
|
||||||
|
path = parsed.path or "/"
|
||||||
deadline = time.time() + timeout
|
deadline = time.time() + timeout
|
||||||
while time.time() < deadline:
|
while time.time() < deadline:
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(url, timeout=1) as response:
|
conn = http.client.HTTPConnection(parsed.hostname or "127.0.0.1", parsed.port or 80, timeout=1)
|
||||||
if 200 == response.status:
|
try:
|
||||||
|
conn.request("GET", path)
|
||||||
|
response = conn.getresponse()
|
||||||
|
if response.status == 200:
|
||||||
os.environ.pop("LD_LIBRARY_PATH", None)
|
os.environ.pop("LD_LIBRARY_PATH", None)
|
||||||
webbrowser.open_new(url)
|
webbrowser.open_new(url)
|
||||||
return
|
return
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
except Exception:
|
except Exception:
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
|
|
||||||
print(f"Failed to open browser automatically within {timeout} seconds. Please open {url} manually.")
|
sys.stderr.write(
|
||||||
|
f"Failed to open browser automatically within {timeout} seconds. Please open {url} manually.\n"
|
||||||
|
)
|
||||||
|
|
||||||
threading.Thread(target=wait_then_open, daemon=True).start()
|
threading.Thread(target=wait_then_open, daemon=True).start()
|
||||||
|
|
||||||
|
|
@ -115,7 +124,7 @@ def main():
|
||||||
|
|
||||||
set_env()
|
set_env()
|
||||||
|
|
||||||
env_file: pathlib.Path = pathlib.Path(os.getenv("YTP_CONFIG_PATH")) / ".env"
|
env_file: pathlib.Path = pathlib.Path(os.getenv("YTP_CONFIG_PATH", "")) / ".env"
|
||||||
|
|
||||||
port = None
|
port = None
|
||||||
if env_file.exists():
|
if env_file.exists():
|
||||||
|
|
|
||||||
22
app/main.py
22
app/main.py
|
|
@ -199,18 +199,16 @@ class Main:
|
||||||
)
|
)
|
||||||
HTTP_LOGGER_CLASS = HttpAccessLogger
|
HTTP_LOGGER_CLASS = HttpAccessLogger
|
||||||
|
|
||||||
run_args = {
|
web.run_app(
|
||||||
"host": host,
|
self._app,
|
||||||
"port": port,
|
host=host,
|
||||||
"loop": asyncio.get_event_loop(),
|
port=port,
|
||||||
"access_log": HTTP_LOGGER,
|
loop=asyncio.get_event_loop(),
|
||||||
"print": started,
|
access_log=HTTP_LOGGER,
|
||||||
"handle_signals": cb is None,
|
print=started,
|
||||||
}
|
handle_signals=cb is None,
|
||||||
if HTTP_LOGGER_CLASS is not None:
|
access_log_class=HTTP_LOGGER_CLASS if HTTP_LOGGER_CLASS is not None else web.AccessLogger,
|
||||||
run_args["access_log_class"] = HTTP_LOGGER_CLASS
|
)
|
||||||
|
|
||||||
web.run_app(self._app, **run_args)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -144,7 +144,7 @@ async def serve_static_file(request: Request, config: Config) -> StreamResponse:
|
||||||
else:
|
else:
|
||||||
return web.json_response({"error": "File not found.", "file": path}, status=web.HTTPNotFound.status_code)
|
return web.json_response({"error": "File not found.", "file": path}, status=web.HTTPNotFound.status_code)
|
||||||
|
|
||||||
return web.FileResponse(
|
response = web.FileResponse(
|
||||||
path=file_path,
|
path=file_path,
|
||||||
headers={
|
headers={
|
||||||
"Pragma": "public",
|
"Pragma": "public",
|
||||||
|
|
@ -154,6 +154,16 @@ async def serve_static_file(request: Request, config: Config) -> StreamResponse:
|
||||||
status=web.HTTPOk.status_code,
|
status=web.HTTPOk.status_code,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if STATIC_STATE.index_file is not None and file_path == STATIC_STATE.index_file:
|
||||||
|
response.set_cookie(
|
||||||
|
name="simple_mode",
|
||||||
|
value="true" if config.simple_mode else "false",
|
||||||
|
path=config.base_path if "/" != config.base_path else "/",
|
||||||
|
samesite="Lax",
|
||||||
|
)
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
def setup_static_routes(root_path: Path, config: Config) -> None:
|
def setup_static_routes(root_path: Path, config: Config) -> None:
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,7 @@ async def get_ffprobe(request: Request, config: Config, encoder: Encoder, app: w
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
file: str = request.match_info.get("file")
|
file: str | None = request.match_info.get("file")
|
||||||
if not file:
|
if not file:
|
||||||
return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code)
|
return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code)
|
||||||
|
|
||||||
|
|
@ -76,7 +76,7 @@ async def get_file_info(request: Request, config: Config, encoder: Encoder, app:
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
file: str = request.match_info.get("file")
|
file: str | None = request.match_info.get("file")
|
||||||
if not file:
|
if not file:
|
||||||
return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code)
|
return web.json_response(data={"error": "file is required."}, status=web.HTTPBadRequest.status_code)
|
||||||
|
|
||||||
|
|
@ -137,7 +137,7 @@ async def file_browser(request: Request, config: Config, encoder: Encoder) -> Re
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
req_path: str = request.match_info.get("path")
|
req_path: str | None = request.match_info.get("path")
|
||||||
req_path: str = "/" if not req_path else unquote_plus(req_path)
|
req_path: str = "/" if not req_path else unquote_plus(req_path)
|
||||||
|
|
||||||
raw_req: str = (req_path or "").strip()
|
raw_req: str = (req_path or "").strip()
|
||||||
|
|
@ -461,7 +461,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
|
||||||
record(path, ok=False, error=str(e), action=action, extra={"item": params})
|
record(path, ok=False, error=str(e), action=action, extra={"item": params})
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
extra_info: dict[str, Path] = {"new_path": renamed}
|
extra_info: dict[str, Any] = {"new_path": renamed}
|
||||||
if sidecar_renamed:
|
if sidecar_renamed:
|
||||||
extra_info["sidecar_count"] = len(sidecar_renamed)
|
extra_info["sidecar_count"] = len(sidecar_renamed)
|
||||||
record(path, ok=True, action=action, extra=extra_info)
|
record(path, ok=True, action=action, extra=extra_info)
|
||||||
|
|
@ -591,7 +591,7 @@ async def path_actions(request: Request, config: Config, queue: DownloadQueue, n
|
||||||
record(path, ok=False, error=str(e), action=action, extra={"item": params})
|
record(path, ok=False, error=str(e), action=action, extra={"item": params})
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
extra_info: dict[str, Path] = {"new_path": moved}
|
extra_info: dict[str, Any] = {"new_path": moved}
|
||||||
if sidecar_moved:
|
if sidecar_moved:
|
||||||
extra_info["sidecar_count"] = len(sidecar_moved)
|
extra_info["sidecar_count"] = len(sidecar_moved)
|
||||||
|
|
||||||
|
|
@ -617,7 +617,7 @@ async def prepare_zip_file(request: Request, config: Config, cache: Cache):
|
||||||
if not json or not isinstance(json, list):
|
if not json or not isinstance(json, list):
|
||||||
return web.json_response({"error": "Invalid parameters."}, status=400)
|
return web.json_response({"error": "Invalid parameters."}, status=400)
|
||||||
|
|
||||||
files: list[str] = []
|
files: list[Path] = []
|
||||||
for f in json:
|
for f in json:
|
||||||
if not isinstance(f, str):
|
if not isinstance(f, str):
|
||||||
continue
|
continue
|
||||||
|
|
@ -627,12 +627,10 @@ async def prepare_zip_file(request: Request, config: Config, cache: Cache):
|
||||||
continue
|
continue
|
||||||
files.append(ref)
|
files.append(ref)
|
||||||
|
|
||||||
sc: list[dict] = get_file_sidecar(ref)
|
sc: dict[str, list[dict[str, Any]]] = get_file_sidecar(ref)
|
||||||
if sc:
|
if sc:
|
||||||
for side in sc:
|
for value in sc.values():
|
||||||
for scf in sc[side]:
|
files.extend(scf["file"] for scf in value if isinstance(scf, dict) and "file" in scf)
|
||||||
if isinstance(scf, dict) and "file" in scf:
|
|
||||||
files.append(scf["file"]) # noqa: PERF401
|
|
||||||
|
|
||||||
if not files:
|
if not files:
|
||||||
return web.json_response({"error": "No valid files."}, status=400)
|
return web.json_response({"error": "No valid files."}, status=400)
|
||||||
|
|
|
||||||
|
|
@ -95,7 +95,7 @@ async def get_doc(request: Request, config: Config, cache: Cache) -> Response:
|
||||||
|
|
||||||
cache.set(cache_key, dct, ttl=3600)
|
cache.set(cache_key, dct, ttl=3600)
|
||||||
|
|
||||||
return web.Response(**dct)
|
return web.Response(body=dct["body"], headers=dct["headers"])
|
||||||
except Exception:
|
except Exception:
|
||||||
LOG.exception(
|
LOG.exception(
|
||||||
"Failed to request doc '%s' from '%s'.",
|
"Failed to request doc '%s' from '%s'.",
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
from aiohttp.web import Request, Response
|
from aiohttp.web import Request, Response, StreamResponse
|
||||||
|
|
||||||
from app.library.config import Config
|
from app.library.config import Config
|
||||||
from app.library.log import get_logger
|
from app.library.log import get_logger
|
||||||
|
|
@ -10,7 +10,7 @@ LOG = get_logger()
|
||||||
|
|
||||||
|
|
||||||
@route(["GET", "HEAD"], "/api/download/{filename:.+}", "download_static")
|
@route(["GET", "HEAD"], "/api/download/{filename:.+}", "download_static")
|
||||||
async def download_file(request: Request, config: Config, app: web.Application) -> Response:
|
async def download_file(request: Request, config: Config, app: web.Application) -> StreamResponse:
|
||||||
"""
|
"""
|
||||||
Serve download files.
|
Serve download files.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,10 @@
|
||||||
import asyncio
|
import asyncio
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import Any
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
from aiohttp.web import Request, Response
|
from aiohttp.web import Request, Response
|
||||||
|
from aiohttp.web_response import StreamResponse
|
||||||
|
|
||||||
from app.features.presets.schemas import Preset
|
from app.features.presets.schemas import Preset
|
||||||
from app.features.presets.service import Presets
|
from app.features.presets.service import Presets
|
||||||
|
|
@ -20,10 +21,6 @@ from app.library.log import get_logger
|
||||||
from app.library.router import route
|
from app.library.router import route
|
||||||
from app.library.Utils import calc_download_path, get_file_sidecar, rename_file
|
from app.library.Utils import calc_download_path, get_file_sidecar, rename_file
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from library.downloads import Download
|
|
||||||
|
|
||||||
|
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -123,23 +120,29 @@ async def items_list(request: Request, queue: DownloadQueue, encoder: Encoder, c
|
||||||
|
|
||||||
|
|
||||||
@route("GET", "api/history/live", "items_live")
|
@route("GET", "api/history/live", "items_live")
|
||||||
async def items_live(queue: DownloadQueue, encoder: Encoder) -> Response:
|
async def items_live(request: Request, queue: DownloadQueue, encoder: Encoder, config: Config) -> Response:
|
||||||
"""
|
"""
|
||||||
Get live queue data
|
Get live queue data
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
request (Request): The request object.
|
||||||
queue (DownloadQueue): The download queue instance.
|
queue (DownloadQueue): The download queue instance.
|
||||||
encoder (Encoder): The encoder instance.
|
encoder (Encoder): The encoder instance.
|
||||||
|
config (Config): The configuration instance.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Response: The response object with live queue items.
|
Response: The response object with live queue items.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
try:
|
||||||
|
limit = int(request.query.get("limit", config.queue_display_limit))
|
||||||
|
except ValueError:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
data={
|
data={"error": "limit must be a valid integer."}, status=web.HTTPBadRequest.status_code
|
||||||
"queue": (await queue.get("queue"))["queue"],
|
)
|
||||||
"history_count": await queue.done.get_total_count(),
|
|
||||||
},
|
return web.json_response(
|
||||||
|
data={**queue.live_queue(limit), "history_count": await queue.done.get_total_count()},
|
||||||
status=web.HTTPOk.status_code,
|
status=web.HTTPOk.status_code,
|
||||||
dumps=encoder.encode,
|
dumps=encoder.encode,
|
||||||
)
|
)
|
||||||
|
|
@ -274,7 +277,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) ->
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
id: str = request.match_info.get("id")
|
id: str | None = request.match_info.get("id")
|
||||||
if not id:
|
if not id:
|
||||||
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
|
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
|
||||||
|
|
||||||
|
|
@ -308,7 +311,7 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder) ->
|
||||||
|
|
||||||
|
|
||||||
@route(["GET", "HEAD"], r"api/history/{id}/thumbnail", "history.item.thumbnail")
|
@route(["GET", "HEAD"], r"api/history/{id}/thumbnail", "history.item.thumbnail")
|
||||||
async def item_thumbnail(request: Request, queue: DownloadQueue, config: Config) -> Response:
|
async def item_thumbnail(request: Request, queue: DownloadQueue, config: Config) -> StreamResponse:
|
||||||
if not (id := request.match_info.get("id")):
|
if not (id := request.match_info.get("id")):
|
||||||
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
|
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
|
||||||
|
|
||||||
|
|
@ -452,7 +455,7 @@ async def item_update(request: Request, queue: DownloadQueue, encoder: Encoder,
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
id: str = request.match_info.get("id")
|
id: str | None = request.match_info.get("id")
|
||||||
if not id:
|
if not id:
|
||||||
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
|
return web.json_response(data={"error": "id is required."}, status=web.HTTPBadRequest.status_code)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -116,8 +116,9 @@ async def get_background(request: Request, config: Config, cache: Cache) -> Resp
|
||||||
safe_backend: str = _safe_url(backend)
|
safe_backend: str = _safe_url(backend)
|
||||||
await cache.aset(key=CACHE_KEY_BING, value=backend, ttl=3600 * 24)
|
await cache.aset(key=CACHE_KEY_BING, value=backend, ttl=3600 * 24)
|
||||||
else:
|
else:
|
||||||
backend = await cache.aget(CACHE_KEY_BING)
|
cached_backend = await cache.aget(CACHE_KEY_BING)
|
||||||
safe_backend = _safe_url(backend if isinstance(backend, str) else None)
|
backend = cached_backend if isinstance(cached_backend, str) else ""
|
||||||
|
safe_backend = _safe_url(backend)
|
||||||
|
|
||||||
if not isinstance(backend, str) or not backend:
|
if not isinstance(backend, str) or not backend:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
from collections.abc import Awaitable, Callable
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
|
@ -110,7 +111,7 @@ async def _read_logfile(file: Path, offset: int = 0, limit: int = 50) -> dict:
|
||||||
return {"logs": [], "next_offset": None, "end_is_reached": True}
|
return {"logs": [], "next_offset": None, "end_is_reached": True}
|
||||||
|
|
||||||
|
|
||||||
async def _tail_log(file: Path, emitter: callable, sleep_time: float = 0.5):
|
async def _tail_log(file: Path, emitter: Callable[[dict], Awaitable[None]], sleep_time: float = 0.5):
|
||||||
"""
|
"""
|
||||||
Live tail.
|
Live tail.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import asyncio
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
from aiohttp.web import Request, Response
|
from aiohttp.web import Request, Response
|
||||||
|
|
@ -20,17 +21,14 @@ from app.library.log import get_logger
|
||||||
from app.library.router import route
|
from app.library.router import route
|
||||||
from app.library.TerminalSessionManager import TerminalSessionConflictError, TerminalSessionManager
|
from app.library.TerminalSessionManager import TerminalSessionConflictError, TerminalSessionManager
|
||||||
from app.library.UpdateChecker import UpdateChecker
|
from app.library.UpdateChecker import UpdateChecker
|
||||||
from app.library.Utils import list_folders
|
|
||||||
|
|
||||||
LOG = get_logger()
|
LOG = get_logger()
|
||||||
DIAGNOSTICS_CACHE_KEY = "system:diagnostics"
|
|
||||||
DIAGNOSTICS_CACHE_TTL = 5.0
|
|
||||||
|
|
||||||
|
|
||||||
@route("GET", "api/system/configuration", "system.configuration")
|
@route("GET", "api/system/configuration", "system.configuration")
|
||||||
async def system_config(queue: DownloadQueue, config: Config, encoder: Encoder) -> Response:
|
async def system_config(queue: DownloadQueue, config: Config, encoder: Encoder) -> Response:
|
||||||
"""
|
"""
|
||||||
Pause non-active downloads.
|
Get the system configuration.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
queue (DownloadQueue): The download queue instance.
|
queue (DownloadQueue): The download queue instance.
|
||||||
|
|
@ -47,19 +45,61 @@ async def system_config(queue: DownloadQueue, config: Config, encoder: Encoder)
|
||||||
"presets": Presets.get_instance().get_all(),
|
"presets": Presets.get_instance().get_all(),
|
||||||
"dl_fields": await DLFields.get_instance().get_all_serialized(),
|
"dl_fields": await DLFields.get_instance().get_all_serialized(),
|
||||||
"paused": queue.is_paused(),
|
"paused": queue.is_paused(),
|
||||||
"folders": list_folders(
|
|
||||||
path=Path(config.download_path),
|
|
||||||
base=Path(config.download_path),
|
|
||||||
depth_limit=config.download_path_depth - 1,
|
|
||||||
),
|
|
||||||
"history_count": await queue.done.get_total_count(),
|
"history_count": await queue.done.get_total_count(),
|
||||||
"queue": (await queue.get("queue"))["queue"],
|
|
||||||
},
|
},
|
||||||
status=web.HTTPOk.status_code,
|
status=web.HTTPOk.status_code,
|
||||||
dumps=encoder.encode,
|
dumps=encoder.encode,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@route("GET", "api/system/folders", "system.folders")
|
||||||
|
async def system_folders(request: Request, config: Config, encoder: Encoder, cache: Cache) -> Response:
|
||||||
|
"""
|
||||||
|
List child directories for a given relative path.
|
||||||
|
|
||||||
|
Query params:
|
||||||
|
path: Relative path within the download directory (default: root).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Response: The response object.
|
||||||
|
|
||||||
|
"""
|
||||||
|
raw_path: str = request.query.get("path", "").strip().lstrip("/")
|
||||||
|
base: Path = Path(config.download_path).resolve()
|
||||||
|
target: Path = (base / raw_path).resolve()
|
||||||
|
|
||||||
|
if not target.is_relative_to(base):
|
||||||
|
return web.json_response(
|
||||||
|
data={"path": raw_path, "folders": []},
|
||||||
|
status=web.HTTPOk.status_code,
|
||||||
|
dumps=encoder.encode,
|
||||||
|
)
|
||||||
|
|
||||||
|
cache_key = f"folders:{target!s}"
|
||||||
|
if (cached := cache.get(cache_key)) is not None:
|
||||||
|
return web.json_response(data=cached, status=web.HTTPOk.status_code, dumps=encoder.encode)
|
||||||
|
|
||||||
|
folders: list[str] = []
|
||||||
|
if target.is_dir():
|
||||||
|
try:
|
||||||
|
folders.extend(
|
||||||
|
entry.name
|
||||||
|
for entry in sorted(target.iterdir())
|
||||||
|
if entry.is_dir() and entry.resolve().is_relative_to(base)
|
||||||
|
)
|
||||||
|
except PermissionError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
resolved_rel: str = str(target.relative_to(base)) if target != base else ""
|
||||||
|
if resolved_rel == ".":
|
||||||
|
resolved_rel = ""
|
||||||
|
|
||||||
|
data: dict[str, str | list[str]] = {"path": resolved_rel, "folders": folders}
|
||||||
|
cache.set(cache_key, data, ttl=30.0)
|
||||||
|
|
||||||
|
return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode)
|
||||||
|
|
||||||
|
|
||||||
@route("POST", "api/system/pause", "system.pause")
|
@route("POST", "api/system/pause", "system.pause")
|
||||||
async def downloads_pause(queue: DownloadQueue, encoder: Encoder, notify: EventBus) -> Response:
|
async def downloads_pause(queue: DownloadQueue, encoder: Encoder, notify: EventBus) -> Response:
|
||||||
"""
|
"""
|
||||||
|
|
@ -247,7 +287,7 @@ async def system_diagnostics(
|
||||||
LOG.exception("Failed to collect system diagnostics.")
|
LOG.exception("Failed to collect system diagnostics.")
|
||||||
data = diagnostics_error_report(config)
|
data = diagnostics_error_report(config)
|
||||||
else:
|
else:
|
||||||
cache.set(cache_key, data, ttl=60)
|
cache.set(cache_key, data, ttl=60.0)
|
||||||
|
|
||||||
return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode)
|
return web.json_response(data=data, status=web.HTTPOk.status_code, dumps=encoder.encode)
|
||||||
|
|
||||||
|
|
@ -450,7 +490,7 @@ async def system_limits(queue: DownloadQueue, config: Config, encoder: Encoder)
|
||||||
|
|
||||||
for download in [*active_non_live, *queued_non_live]:
|
for download in [*active_non_live, *queued_non_live]:
|
||||||
extractor = (download.info.get_extractor() or "unknown").lower()
|
extractor = (download.info.get_extractor() or "unknown").lower()
|
||||||
entry = per_extractor.setdefault(
|
entry: dict[str, Any] = per_extractor.setdefault(
|
||||||
extractor,
|
extractor,
|
||||||
{
|
{
|
||||||
"name": extractor,
|
"name": extractor,
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine
|
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
|
||||||
|
|
||||||
APP_ROOT = str((Path(__file__).parent / ".." / "..").resolve())
|
APP_ROOT = str((Path(__file__).parent / ".." / "..").resolve())
|
||||||
if APP_ROOT not in sys.path:
|
if APP_ROOT not in sys.path:
|
||||||
|
|
@ -69,7 +69,7 @@ def parse_args() -> argparse.Namespace:
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
async def connect_db(db_file: str) -> AsyncConnection:
|
async def connect_db(db_file: Path) -> tuple[AsyncConnection, AsyncEngine]:
|
||||||
"""Create async SQLAlchemy connection."""
|
"""Create async SQLAlchemy connection."""
|
||||||
engine = create_async_engine(f"sqlite+aiosqlite:///{db_file}")
|
engine = create_async_engine(f"sqlite+aiosqlite:///{db_file}")
|
||||||
conn = await engine.connect()
|
conn = await engine.connect()
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
from typing import Any
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
@ -91,7 +92,7 @@ class TestAgSet:
|
||||||
|
|
||||||
def test_ag_set_error_on_non_dict_final(self):
|
def test_ag_set_error_on_non_dict_final(self):
|
||||||
"""Test error when final target is not a dict."""
|
"""Test error when final target is not a dict."""
|
||||||
data = "not_a_dict"
|
data: Any = "not_a_dict"
|
||||||
with pytest.raises(RuntimeError, match="Cannot set value at path 'key'"):
|
with pytest.raises(RuntimeError, match="Cannot set value at path 'key'"):
|
||||||
ag_set(data, "key", "value")
|
ag_set(data, "key", "value")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import http.cookiejar
|
import http.cookiejar
|
||||||
|
from typing import Any
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
@ -12,10 +13,10 @@ try:
|
||||||
YTDLP_AVAILABLE = True
|
YTDLP_AVAILABLE = True
|
||||||
except ImportError:
|
except ImportError:
|
||||||
YTDLP_AVAILABLE = False
|
YTDLP_AVAILABLE = False
|
||||||
Request = Mock
|
Request: Any = Mock
|
||||||
Response = Mock
|
Response: Any = Mock
|
||||||
RequestDirector = Mock
|
RequestDirector: Any = Mock
|
||||||
HTTPError = Exception
|
HTTPError: Any = Exception
|
||||||
|
|
||||||
pytestmark = pytest.mark.skipif(not YTDLP_AVAILABLE, reason="yt-dlp not available")
|
pytestmark = pytest.mark.skipif(not YTDLP_AVAILABLE, reason="yt-dlp not available")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
from typing import Any
|
||||||
from unittest.mock import AsyncMock, Mock, patch
|
from unittest.mock import AsyncMock, Mock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
@ -24,7 +25,7 @@ class TestConditionIgnoreMatching:
|
||||||
second = SimpleNamespace(id=2, name="Fallback", enabled=True, filter="duration > 0", priority=10)
|
second = SimpleNamespace(id=2, name="Fallback", enabled=True, filter="duration > 0", priority=10)
|
||||||
|
|
||||||
service = Conditions.get_instance()
|
service = Conditions.get_instance()
|
||||||
service._repo = Mock(list=AsyncMock(return_value=[first, second]))
|
service._repo = Mock(all=AsyncMock(return_value=[first, second]))
|
||||||
|
|
||||||
matched = await service.match(info={"duration": 60}, ignore_conditions=["Primary"])
|
matched = await service.match(info={"duration": 60}, ignore_conditions=["Primary"])
|
||||||
|
|
||||||
|
|
@ -36,7 +37,7 @@ class TestConditionIgnoreMatching:
|
||||||
second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10)
|
second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10)
|
||||||
|
|
||||||
service = Conditions.get_instance()
|
service = Conditions.get_instance()
|
||||||
service._repo = Mock(list=AsyncMock(return_value=[first, second]))
|
service._repo = Mock(all=AsyncMock(return_value=[first, second]))
|
||||||
|
|
||||||
matched = await service.match(info={"duration": 60}, ignore_conditions=["123"])
|
matched = await service.match(info={"duration": 60}, ignore_conditions=["123"])
|
||||||
|
|
||||||
|
|
@ -48,7 +49,7 @@ class TestConditionIgnoreMatching:
|
||||||
second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10)
|
second = SimpleNamespace(id=124, name="Fallback", enabled=True, filter="duration > 0", priority=10)
|
||||||
|
|
||||||
service = Conditions.get_instance()
|
service = Conditions.get_instance()
|
||||||
service._repo = Mock(list=AsyncMock(return_value=[first, second]))
|
service._repo = Mock(all=AsyncMock(return_value=[first, second]))
|
||||||
|
|
||||||
matched = await service.match(info={"duration": 60}, ignore_conditions=[123])
|
matched = await service.match(info={"duration": 60}, ignore_conditions=[123])
|
||||||
|
|
||||||
|
|
@ -59,7 +60,7 @@ class TestConditionIgnoreMatching:
|
||||||
first = SimpleNamespace(id=1, name="Primary", enabled=True, filter="duration > 0", priority=20)
|
first = SimpleNamespace(id=1, name="Primary", enabled=True, filter="duration > 0", priority=20)
|
||||||
|
|
||||||
service = Conditions.get_instance()
|
service = Conditions.get_instance()
|
||||||
service._repo = Mock(list=AsyncMock(return_value=[first]))
|
service._repo = Mock(all=AsyncMock(return_value=[first]))
|
||||||
|
|
||||||
matched = await service.match(info={"duration": 60}, ignore_conditions=["*"])
|
matched = await service.match(info={"duration": 60}, ignore_conditions=["*"])
|
||||||
|
|
||||||
|
|
@ -78,10 +79,11 @@ class TestConditionIgnorePropagation:
|
||||||
preset="default",
|
preset="default",
|
||||||
extras={"ignore_conditions": [" 123 ", "Primary", "", " * "]},
|
extras={"ignore_conditions": [" 123 ", "Primary", "", " * "]},
|
||||||
)
|
)
|
||||||
item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
|
mock_item: Any = item
|
||||||
item.get_archive_id = Mock(return_value=None)
|
mock_item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
|
||||||
item.get_archive_file = Mock(return_value=None)
|
mock_item.get_archive_id = Mock(return_value=None)
|
||||||
item.is_archived = Mock(return_value=False)
|
mock_item.get_archive_file = Mock(return_value=None)
|
||||||
|
mock_item.is_archived = Mock(return_value=False)
|
||||||
|
|
||||||
matcher = Mock(match=AsyncMock(return_value=None))
|
matcher = Mock(match=AsyncMock(return_value=None))
|
||||||
entry = {"id": "video-1", "duration": 60}
|
entry = {"id": "video-1", "duration": 60}
|
||||||
|
|
@ -113,10 +115,11 @@ class TestConditionIgnorePropagation:
|
||||||
preset="default",
|
preset="default",
|
||||||
extras={"ignore_conditions": [123, "Primary", True, " * "]},
|
extras={"ignore_conditions": [123, "Primary", True, " * "]},
|
||||||
)
|
)
|
||||||
item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
|
mock_item: Any = item
|
||||||
item.get_archive_id = Mock(return_value=None)
|
mock_item.get_ytdlp_opts = Mock(return_value=Mock(get_all=Mock(return_value={})))
|
||||||
item.get_archive_file = Mock(return_value=None)
|
mock_item.get_archive_id = Mock(return_value=None)
|
||||||
item.is_archived = Mock(return_value=False)
|
mock_item.get_archive_file = Mock(return_value=None)
|
||||||
|
mock_item.is_archived = Mock(return_value=False)
|
||||||
|
|
||||||
matcher = Mock(match=AsyncMock(return_value=None))
|
matcher = Mock(match=AsyncMock(return_value=None))
|
||||||
entry = {"id": "video-1", "duration": 60}
|
entry = {"id": "video-1", "duration": 60}
|
||||||
|
|
@ -135,7 +138,7 @@ class TestConditionIgnorePropagation:
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_playlist_keeps_parent_ignore(self) -> None:
|
async def test_playlist_keeps_parent_ignore(self) -> None:
|
||||||
captured: dict[str, object] = {}
|
captured: dict[str, Any] = {}
|
||||||
|
|
||||||
class FakeItem:
|
class FakeItem:
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
|
|
@ -158,7 +161,8 @@ class TestConditionIgnorePropagation:
|
||||||
}
|
}
|
||||||
|
|
||||||
with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")):
|
with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")):
|
||||||
result = await process_playlist(queue=queue, entry=entry, item=FakeItem())
|
fake: Any = FakeItem()
|
||||||
|
result = await process_playlist(queue=queue, entry=entry, item=fake)
|
||||||
|
|
||||||
assert result == {"status": "ok"}
|
assert result == {"status": "ok"}
|
||||||
assert captured["url"] == "https://example.com/v/1"
|
assert captured["url"] == "https://example.com/v/1"
|
||||||
|
|
@ -209,7 +213,8 @@ class TestConditionIgnorePropagation:
|
||||||
}
|
}
|
||||||
|
|
||||||
with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")):
|
with patch("app.library.downloads.playlist_processor.ytdlp_reject", return_value=(True, "")):
|
||||||
result = await process_playlist(queue=queue, entry=entry, item=FakeItem())
|
fake: Any = FakeItem()
|
||||||
|
result = await process_playlist(queue=queue, entry=entry, item=fake)
|
||||||
|
|
||||||
assert result == {"status": "ok"}
|
assert result == {"status": "ok"}
|
||||||
assert queue.add.await_count == 2
|
assert queue.add.await_count == 2
|
||||||
|
|
|
||||||
|
|
@ -4,10 +4,13 @@ from collections import OrderedDict
|
||||||
from dataclasses import asdict
|
from dataclasses import asdict
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from email.utils import formatdate
|
from email.utils import formatdate
|
||||||
|
from typing import Any
|
||||||
from unittest.mock import AsyncMock, Mock
|
from unittest.mock import AsyncMock, Mock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.library.DataStore import DataStore, StoreType
|
from app.library.DataStore import DataStore, StoreType
|
||||||
|
from app.library.downloads import Download
|
||||||
from app.library.ItemDTO import ItemDTO
|
from app.library.ItemDTO import ItemDTO
|
||||||
from app.library.operations import Operation
|
from app.library.operations import Operation
|
||||||
from app.library.sqlite_store import SqliteStore
|
from app.library.sqlite_store import SqliteStore
|
||||||
|
|
@ -62,17 +65,18 @@ async def make_db(data: int = 0) -> SqliteStore:
|
||||||
return ins
|
return ins
|
||||||
|
|
||||||
|
|
||||||
class StubDownload:
|
class StubDownload(Download):
|
||||||
def __init__(self, info: ItemDTO, started: bool = False, cancelled: bool = False):
|
def __init__(self, info: ItemDTO, started: bool = False, cancelled: bool = False) -> None:
|
||||||
self.info = info
|
self.info = info
|
||||||
self._started: bool = started
|
self.id = info._id
|
||||||
self._cancelled: bool = cancelled
|
self._stub_started: bool = started
|
||||||
|
self._stub_cancelled: bool = cancelled
|
||||||
|
|
||||||
def started(self) -> bool:
|
def started(self) -> bool:
|
||||||
return self._started
|
return self._stub_started
|
||||||
|
|
||||||
def is_cancelled(self) -> bool:
|
def is_cancelled(self) -> bool:
|
||||||
return self._cancelled
|
return self._stub_cancelled
|
||||||
|
|
||||||
|
|
||||||
async def make_store_async(store_type: StoreType) -> DataStore:
|
async def make_store_async(store_type: StoreType) -> DataStore:
|
||||||
|
|
@ -357,7 +361,8 @@ class TestDataStore:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.info = None
|
self.info = None
|
||||||
|
|
||||||
store._dict["broken"] = BrokenDownload()
|
broken: Any = BrokenDownload()
|
||||||
|
store._dict["broken"] = broken
|
||||||
|
|
||||||
# Should still find the valid item
|
# Should still find the valid item
|
||||||
result = await store.get_item(title="Video 1")
|
result = await store.get_item(title="Video 1")
|
||||||
|
|
@ -960,10 +965,10 @@ class TestDataStoreOperations:
|
||||||
|
|
||||||
item1 = make_item(id="vid1", title="Video 1")
|
item1 = make_item(id="vid1", title="Video 1")
|
||||||
item1._id = "id1"
|
item1._id = "id1"
|
||||||
item1.filesize = 1000
|
setattr(item1, "filesize", 1000)
|
||||||
item2 = make_item(id="vid2", title="Video 2")
|
item2 = make_item(id="vid2", title="Video 2")
|
||||||
item2._id = "id2"
|
item2._id = "id2"
|
||||||
item2.filesize = 2000
|
setattr(item2, "filesize", 2000)
|
||||||
|
|
||||||
await store.put(StubDownload(info=item1))
|
await store.put(StubDownload(info=item1))
|
||||||
await store.put(StubDownload(info=item2))
|
await store.put(StubDownload(info=item2))
|
||||||
|
|
@ -986,10 +991,10 @@ class TestDataStoreOperations:
|
||||||
|
|
||||||
item1 = make_item(id="vid1", title="Video 1")
|
item1 = make_item(id="vid1", title="Video 1")
|
||||||
item1._id = "id1"
|
item1._id = "id1"
|
||||||
item1.filesize = 1000
|
setattr(item1, "filesize", 1000)
|
||||||
item2 = make_item(id="vid2", title="Video 2")
|
item2 = make_item(id="vid2", title="Video 2")
|
||||||
item2._id = "id2"
|
item2._id = "id2"
|
||||||
item2.filesize = 2000
|
setattr(item2, "filesize", 2000)
|
||||||
|
|
||||||
await store.put(StubDownload(info=item1))
|
await store.put(StubDownload(info=item1))
|
||||||
await store.put(StubDownload(info=item2))
|
await store.put(StubDownload(info=item2))
|
||||||
|
|
@ -1007,7 +1012,7 @@ class TestDataStoreOperations:
|
||||||
|
|
||||||
item1 = make_item(id="vid1", title="Video 1")
|
item1 = make_item(id="vid1", title="Video 1")
|
||||||
item1._id = "id1"
|
item1._id = "id1"
|
||||||
item1.filesize = 1000
|
setattr(item1, "filesize", 1000)
|
||||||
|
|
||||||
await store.put(StubDownload(info=item1))
|
await store.put(StubDownload(info=item1))
|
||||||
|
|
||||||
|
|
@ -1033,7 +1038,7 @@ class TestDataStoreOperations:
|
||||||
|
|
||||||
item1 = make_item(id="vid1", title="Video 1")
|
item1 = make_item(id="vid1", title="Video 1")
|
||||||
item1._id = "id1"
|
item1._id = "id1"
|
||||||
item1.filesize = 1000
|
setattr(item1, "filesize", 1000)
|
||||||
|
|
||||||
await store.put(StubDownload(info=item1))
|
await store.put(StubDownload(info=item1))
|
||||||
|
|
||||||
|
|
@ -1084,7 +1089,7 @@ class TestDataStoreOperations:
|
||||||
|
|
||||||
item1 = make_item(id="vid1", title="Video 1")
|
item1 = make_item(id="vid1", title="Video 1")
|
||||||
item1._id = "id1"
|
item1._id = "id1"
|
||||||
item1.description = None
|
setattr(item1, "description", None)
|
||||||
|
|
||||||
await store.put(StubDownload(info=item1))
|
await store.put(StubDownload(info=item1))
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -307,7 +307,8 @@ class TestDownloadFlow:
|
||||||
postprocessor_hook=Mock(),
|
postprocessor_hook=Mock(),
|
||||||
post_hook=Mock(),
|
post_hook=Mock(),
|
||||||
)
|
)
|
||||||
download.info.get_ytdlp_opts = Mock(
|
download_any: Any = download.info
|
||||||
|
download_any.get_ytdlp_opts = Mock(
|
||||||
return_value=Mock(
|
return_value=Mock(
|
||||||
add=Mock(
|
add=Mock(
|
||||||
return_value=Mock(
|
return_value=Mock(
|
||||||
|
|
@ -379,7 +380,8 @@ class TestDownloadFlow:
|
||||||
postprocessor_hook=Mock(),
|
postprocessor_hook=Mock(),
|
||||||
post_hook=Mock(),
|
post_hook=Mock(),
|
||||||
)
|
)
|
||||||
download.info.get_ytdlp_opts = Mock(
|
download_any2: Any = download.info
|
||||||
|
download_any2.get_ytdlp_opts = Mock(
|
||||||
return_value=Mock(add=Mock(return_value=Mock(get_all=Mock(return_value={}))))
|
return_value=Mock(add=Mock(return_value=Mock(get_all=Mock(return_value={}))))
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -481,7 +483,8 @@ class TestDownloadFlow:
|
||||||
)
|
)
|
||||||
queue.put(Terminator())
|
queue.put(Terminator())
|
||||||
|
|
||||||
download._download = fake_download
|
download_mock: Any = download
|
||||||
|
download_mock._download = fake_download
|
||||||
|
|
||||||
class InlineProcess:
|
class InlineProcess:
|
||||||
def __init__(self, target):
|
def __init__(self, target):
|
||||||
|
|
@ -578,7 +581,8 @@ class TestDownloadFlow:
|
||||||
queue.put({"id": download.id, "status": "finished", "final_name": str(final_file)})
|
queue.put({"id": download.id, "status": "finished", "final_name": str(final_file)})
|
||||||
queue.put(Terminator())
|
queue.put(Terminator())
|
||||||
|
|
||||||
download._download = fake_download
|
download_mock: Any = download
|
||||||
|
download_mock._download = fake_download
|
||||||
|
|
||||||
class InlineProcess:
|
class InlineProcess:
|
||||||
def __init__(self, target):
|
def __init__(self, target):
|
||||||
|
|
@ -1028,7 +1032,7 @@ class TestStatusTracker:
|
||||||
"debug": False,
|
"debug": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
def test_init_sets_attributes(self, mock_config: dict) -> None:
|
def test_init_sets_attributes(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
assert st.id == "test-id", "Should set download ID"
|
assert st.id == "test-id", "Should set download ID"
|
||||||
assert st.info == mock_config["info"], "Should set info reference"
|
assert st.info == mock_config["info"], "Should set info reference"
|
||||||
|
|
@ -1036,7 +1040,7 @@ class TestStatusTracker:
|
||||||
assert st.final_update is False, "Should initialize final_update as False"
|
assert st.final_update is False, "Should initialize final_update as False"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_status_ignores_bad_id(self, mock_config: dict) -> None:
|
async def test_status_ignores_bad_id(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
status = {"id": "wrong-id", "status": "downloading"}
|
status = {"id": "wrong-id", "status": "downloading"}
|
||||||
|
|
||||||
|
|
@ -1044,14 +1048,14 @@ class TestStatusTracker:
|
||||||
assert st.info.status != "downloading", "Should not update status for wrong ID"
|
assert st.info.status != "downloading", "Should not update status for wrong ID"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_status_ignores_short(self, mock_config: dict) -> None:
|
async def test_status_ignores_short(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
status = {"id": "test-id"}
|
status = {"id": "test-id"}
|
||||||
|
|
||||||
await st.process_status_update(status)
|
await st.process_status_update(status)
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_process_status_update_sets_status(self, mock_config: dict) -> None:
|
async def test_process_status_update_sets_status(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
status = {"id": "test-id", "status": "downloading", "downloaded_bytes": 1000}
|
status = {"id": "test-id", "status": "downloading", "downloaded_bytes": 1000}
|
||||||
|
|
||||||
|
|
@ -1059,7 +1063,7 @@ class TestStatusTracker:
|
||||||
assert st.info.status == "downloading", "Should update info status"
|
assert st.info.status == "downloading", "Should update info status"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_status_sets_skipped(self, mock_config: dict) -> None:
|
async def test_status_sets_skipped(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
status = {"id": "test-id", "status": "downloading", "download_skipped": True}
|
status = {"id": "test-id", "status": "downloading", "download_skipped": True}
|
||||||
|
|
||||||
|
|
@ -1067,7 +1071,7 @@ class TestStatusTracker:
|
||||||
assert st.info.download_skipped is True, "Should update download_skipped from status queue"
|
assert st.info.download_skipped is True, "Should update download_skipped from status queue"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_process_status_update_sets_tmpfilename(self, mock_config: dict) -> None:
|
async def test_process_status_update_sets_tmpfilename(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
status = {"id": "test-id", "status": "downloading", "tmpfilename": "/tmp/file.part"}
|
status = {"id": "test-id", "status": "downloading", "tmpfilename": "/tmp/file.part"}
|
||||||
|
|
||||||
|
|
@ -1075,7 +1079,7 @@ class TestStatusTracker:
|
||||||
assert st.tmpfilename == "/tmp/file.part", "Should update tmpfilename"
|
assert st.tmpfilename == "/tmp/file.part", "Should update tmpfilename"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_status_sets_percent(self, mock_config: dict) -> None:
|
async def test_status_sets_percent(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
status = {
|
status = {
|
||||||
"id": "test-id",
|
"id": "test-id",
|
||||||
|
|
@ -1090,7 +1094,7 @@ class TestStatusTracker:
|
||||||
assert st.info.percent == 50.0, "Should calculate percent correctly"
|
assert st.info.percent == 50.0, "Should calculate percent correctly"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_status_uses_estimate(self, mock_config: dict) -> None:
|
async def test_status_uses_estimate(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
status = {
|
status = {
|
||||||
"id": "test-id",
|
"id": "test-id",
|
||||||
|
|
@ -1104,7 +1108,7 @@ class TestStatusTracker:
|
||||||
assert st.info.percent == 30.0, "Should calculate percent from estimate"
|
assert st.info.percent == 30.0, "Should calculate percent from estimate"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_status_percent(self, mock_config: dict) -> None:
|
async def test_status_percent(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
status = {
|
status = {
|
||||||
"id": "test-id",
|
"id": "test-id",
|
||||||
|
|
@ -1117,7 +1121,7 @@ class TestStatusTracker:
|
||||||
assert st.info.percent == 50.0, "Should calculate percent correctly with valid total"
|
assert st.info.percent == 50.0, "Should calculate percent correctly with valid total"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_status_sets_speed_eta(self, mock_config: dict) -> None:
|
async def test_status_sets_speed_eta(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
status = {"id": "test-id", "status": "downloading", "speed": 1024000, "eta": 60}
|
status = {"id": "test-id", "status": "downloading", "speed": 1024000, "eta": 60}
|
||||||
|
|
||||||
|
|
@ -1126,7 +1130,7 @@ class TestStatusTracker:
|
||||||
assert st.info.eta == 60, "Should set eta"
|
assert st.info.eta == 60, "Should set eta"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_process_status_update_sets_error(self, mock_config: dict) -> None:
|
async def test_process_status_update_sets_error(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
status = {"id": "test-id", "status": "error", "error": "Download failed"}
|
status = {"id": "test-id", "status": "error", "error": "Download failed"}
|
||||||
|
|
||||||
|
|
@ -1135,7 +1139,7 @@ class TestStatusTracker:
|
||||||
assert st.info.error == "Download failed", "Should set error message"
|
assert st.info.error == "Download failed", "Should set error message"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict) -> None:
|
async def test_process_status_update_sets_final_update(self, tmp_path: Path, mock_config: dict[str, Any]) -> None:
|
||||||
test_file = tmp_path / "test.mp4"
|
test_file = tmp_path / "test.mp4"
|
||||||
test_file.write_text("test content")
|
test_file.write_text("test content")
|
||||||
|
|
||||||
|
|
@ -1148,20 +1152,20 @@ class TestStatusTracker:
|
||||||
assert st.info.filename == "test.mp4", "Should set relative filename"
|
assert st.info.filename == "test.mp4", "Should set relative filename"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_drain_queue_processes_remaining_updates(self, mock_config: dict) -> None:
|
async def test_drain_queue_processes_remaining_updates(self, mock_config: dict[str, Any]) -> None:
|
||||||
queue = DummyQueue()
|
queue = DummyQueue()
|
||||||
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 100})
|
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 100})
|
||||||
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 200})
|
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 200})
|
||||||
queue.put(Terminator())
|
queue.put(Terminator())
|
||||||
|
|
||||||
config = {**mock_config, "status_queue": queue}
|
config: dict[str, Any] = {**mock_config, "status_queue": queue}
|
||||||
st = StatusTracker(**config)
|
st = StatusTracker(**config)
|
||||||
|
|
||||||
await st.drain_queue(max_iterations=10)
|
await st.drain_queue(max_iterations=10)
|
||||||
assert st.info.downloaded_bytes == 200, "Should process all queued updates"
|
assert st.info.downloaded_bytes == 200, "Should process all queued updates"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict) -> None:
|
async def test_drain_queue_stops_on_final_update(self, tmp_path: Path, mock_config: dict[str, Any]) -> None:
|
||||||
test_file = tmp_path / "test.mp4"
|
test_file = tmp_path / "test.mp4"
|
||||||
test_file.write_text("test content")
|
test_file.write_text("test content")
|
||||||
|
|
||||||
|
|
@ -1169,25 +1173,25 @@ class TestStatusTracker:
|
||||||
queue.put({"id": "test-id", "status": "finished", "final_name": str(test_file)})
|
queue.put({"id": "test-id", "status": "finished", "final_name": str(test_file)})
|
||||||
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 999})
|
queue.put({"id": "test-id", "status": "downloading", "downloaded_bytes": 999})
|
||||||
|
|
||||||
config = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)}
|
config: dict[str, Any] = {**mock_config, "status_queue": queue, "download_dir": str(tmp_path)}
|
||||||
st = StatusTracker(**config)
|
st = StatusTracker(**config)
|
||||||
|
|
||||||
await st.drain_queue(max_iterations=10)
|
await st.drain_queue(max_iterations=10)
|
||||||
assert st.final_update is True, "Should stop draining after final update"
|
assert st.final_update is True, "Should stop draining after final update"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_drain_queue_skips_invalid(self, mock_config: dict) -> None:
|
async def test_drain_queue_skips_invalid(self, mock_config: dict[str, Any]) -> None:
|
||||||
queue = DummyQueue()
|
queue = DummyQueue()
|
||||||
queue.put({"id": "test-id", "status": "downloading"})
|
queue.put({"id": "test-id", "status": "downloading"})
|
||||||
queue.put(None)
|
queue.put(None)
|
||||||
|
|
||||||
config = {**mock_config, "status_queue": queue}
|
config: dict[str, Any] = {**mock_config, "status_queue": queue}
|
||||||
st = StatusTracker(**config)
|
st = StatusTracker(**config)
|
||||||
|
|
||||||
await st.drain_queue(max_iterations=5)
|
await st.drain_queue(max_iterations=5)
|
||||||
assert st.info.status == "downloading", "valid updates should still be processed"
|
assert st.info.status == "downloading", "valid updates should still be processed"
|
||||||
|
|
||||||
def test_cancel_update_task(self, mock_config: dict) -> None:
|
def test_cancel_update_task(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
st.update_task = Mock()
|
st.update_task = Mock()
|
||||||
st.update_task.done = Mock(return_value=False)
|
st.update_task.done = Mock(return_value=False)
|
||||||
|
|
@ -1196,23 +1200,122 @@ class TestStatusTracker:
|
||||||
st.cancel_update_task()
|
st.cancel_update_task()
|
||||||
st.update_task.cancel.assert_called_once()
|
st.update_task.cancel.assert_called_once()
|
||||||
|
|
||||||
def test_cancel_update_task_noop(self, mock_config: dict) -> None:
|
def test_cancel_update_task_noop(self, mock_config: dict[str, Any]) -> None:
|
||||||
st = StatusTracker(**mock_config)
|
st = StatusTracker(**mock_config)
|
||||||
|
|
||||||
st.cancel_update_task()
|
st.cancel_update_task()
|
||||||
assert st.update_task is None, "missing tasks should be ignored"
|
assert st.update_task is None, "missing tasks should be ignored"
|
||||||
|
|
||||||
def test_put_terminator_adds_to_queue(self, mock_config: dict) -> None:
|
def test_put_terminator_adds_to_queue(self, mock_config: dict[str, Any]) -> None:
|
||||||
queue = DummyQueue()
|
queue = DummyQueue()
|
||||||
config = {**mock_config, "status_queue": queue}
|
config: dict[str, Any] = {**mock_config, "status_queue": queue}
|
||||||
st = StatusTracker(**config)
|
st = StatusTracker(**config)
|
||||||
|
|
||||||
st.put_terminator()
|
st.put_terminator()
|
||||||
assert 1 == len(queue.items), "Should add terminator to queue"
|
assert 1 == len(queue.items), "Should add terminator to queue"
|
||||||
assert isinstance(queue.items[0], Terminator), "Should add Terminator instance"
|
assert isinstance(queue.items[0], Terminator), "Should add Terminator instance"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_progress_emits_item_progress(self, mock_config: dict[str, Any]) -> None:
|
||||||
|
st = StatusTracker(**mock_config)
|
||||||
|
st.info.status = "downloading"
|
||||||
|
calls: list = []
|
||||||
|
st._notify = Mock()
|
||||||
|
st._notify.emit = Mock(side_effect=lambda *a, **kw: calls.append((a, kw)))
|
||||||
|
|
||||||
|
await st.process_status_update(
|
||||||
|
{"id": "test-id", "status": "downloading", "downloaded_bytes": 50, "total_bytes": 100}
|
||||||
|
)
|
||||||
|
|
||||||
|
progress_calls = [c for c in calls if c[0][0] == Events.ITEM_PROGRESS]
|
||||||
|
updated_calls = [c for c in calls if c[0][0] == Events.ITEM_UPDATED]
|
||||||
|
assert len(progress_calls) == 1
|
||||||
|
assert len(updated_calls) == 0
|
||||||
|
payload = progress_calls[0][1]["data"]
|
||||||
|
assert payload["_id"] == st.info._id
|
||||||
|
assert payload["percent"] == 50.0
|
||||||
|
assert "options" not in payload
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_status_change_emits_item_updated(self, mock_config: dict[str, Any]) -> None:
|
||||||
|
st = StatusTracker(**mock_config)
|
||||||
|
st.info.status = "started"
|
||||||
|
calls: list = []
|
||||||
|
st._notify = Mock()
|
||||||
|
st._notify.emit = Mock(side_effect=lambda *a, **kw: calls.append((a, kw)))
|
||||||
|
|
||||||
|
await st.process_status_update(
|
||||||
|
{"id": "test-id", "status": "downloading", "downloaded_bytes": 10, "total_bytes": 100}
|
||||||
|
)
|
||||||
|
|
||||||
|
updated_calls = [c for c in calls if c[0][0] == Events.ITEM_UPDATED]
|
||||||
|
assert len(updated_calls) == 1
|
||||||
|
assert updated_calls[0][1]["data"] is st.info
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_progress_throttled(self, mock_config: dict[str, Any]) -> None:
|
||||||
|
st = StatusTracker(**mock_config)
|
||||||
|
st.info.status = "downloading"
|
||||||
|
st._progress_interval = 0.5
|
||||||
|
calls: list = []
|
||||||
|
st._notify = Mock()
|
||||||
|
st._notify.emit = Mock(side_effect=lambda *a, **kw: calls.append((a, kw)))
|
||||||
|
|
||||||
|
await st.process_status_update(
|
||||||
|
{"id": "test-id", "status": "downloading", "downloaded_bytes": 10, "total_bytes": 100}
|
||||||
|
)
|
||||||
|
await st.process_status_update(
|
||||||
|
{"id": "test-id", "status": "downloading", "downloaded_bytes": 20, "total_bytes": 100}
|
||||||
|
)
|
||||||
|
await st.process_status_update(
|
||||||
|
{"id": "test-id", "status": "downloading", "downloaded_bytes": 30, "total_bytes": 100}
|
||||||
|
)
|
||||||
|
|
||||||
|
progress_calls = [c for c in calls if c[0][0] == Events.ITEM_PROGRESS]
|
||||||
|
assert len(progress_calls) == 1, "Rapid ticks should be throttled to one emission"
|
||||||
|
assert st._pending_progress is True
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_flush_on_status_change(self, mock_config: dict[str, Any]) -> None:
|
||||||
|
st = StatusTracker(**mock_config)
|
||||||
|
st.info.status = "downloading"
|
||||||
|
st._progress_interval = 10.0
|
||||||
|
calls: list = []
|
||||||
|
st._notify = Mock()
|
||||||
|
st._notify.emit = Mock(side_effect=lambda *a, **kw: calls.append((a, kw)))
|
||||||
|
|
||||||
|
await st.process_status_update(
|
||||||
|
{"id": "test-id", "status": "downloading", "downloaded_bytes": 10, "total_bytes": 100}
|
||||||
|
)
|
||||||
|
await st.process_status_update(
|
||||||
|
{"id": "test-id", "status": "downloading", "downloaded_bytes": 50, "total_bytes": 100}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert st._pending_progress is True
|
||||||
|
|
||||||
|
await st.process_status_update({"id": "test-id", "status": "error", "error": "fail"})
|
||||||
|
|
||||||
|
progress_calls = [c for c in calls if c[0][0] == Events.ITEM_PROGRESS]
|
||||||
|
updated_calls = [c for c in calls if c[0][0] == Events.ITEM_UPDATED]
|
||||||
|
assert len(progress_calls) == 2, "Pending progress should be flushed before status change"
|
||||||
|
assert len(updated_calls) == 1
|
||||||
|
assert st._pending_progress is False
|
||||||
|
|
||||||
|
|
||||||
class TestQueueManager:
|
class TestQueueManager:
|
||||||
|
class LiveStore:
|
||||||
|
def __init__(self, items: dict[str, Download]) -> None:
|
||||||
|
self._items = items
|
||||||
|
|
||||||
|
def items(self):
|
||||||
|
return self._items.items()
|
||||||
|
|
||||||
|
def __contains__(self, key: str) -> bool:
|
||||||
|
return key in self._items
|
||||||
|
|
||||||
|
def __len__(self) -> int:
|
||||||
|
return len(self._items)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _video_queue() -> Mock:
|
def _video_queue() -> Mock:
|
||||||
async def put(item):
|
async def put(item):
|
||||||
|
|
@ -1246,6 +1349,45 @@ class TestQueueManager:
|
||||||
auto_start=True,
|
auto_start=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _any_video_item() -> Any:
|
||||||
|
return TestQueueManager._video_item()
|
||||||
|
|
||||||
|
def test_live_queue_caps_visible_items(self) -> None:
|
||||||
|
queue_manager = object.__new__(DownloadQueue)
|
||||||
|
items: dict[str, Any] = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
|
||||||
|
queue_manager.queue = self.LiveStore(items)
|
||||||
|
queue_manager.pool = Mock()
|
||||||
|
queue_manager.pool.get_active_downloads.return_value = {}
|
||||||
|
|
||||||
|
snapshot = DownloadQueue.live_queue(queue_manager, limit=2)
|
||||||
|
|
||||||
|
queue_view = snapshot["queue"]
|
||||||
|
assert isinstance(queue_view, dict)
|
||||||
|
assert list(queue_view.keys()) == ["id0", "id1"]
|
||||||
|
assert snapshot["queue_count"] == 5
|
||||||
|
assert snapshot["queue_loaded"] == 2
|
||||||
|
assert snapshot["queue_limit"] == 2
|
||||||
|
|
||||||
|
def test_live_queue_keeps_active(self) -> None:
|
||||||
|
queue_manager = object.__new__(DownloadQueue)
|
||||||
|
items: dict[str, Any] = {f"id{i}": Mock(info=make_item(id=f"id{i}", title=f"Video {i}")) for i in range(5)}
|
||||||
|
queue_manager.queue = self.LiveStore(items)
|
||||||
|
queue_manager.pool = Mock()
|
||||||
|
queue_manager.pool.get_active_downloads.return_value = {
|
||||||
|
"id3": Mock(info=make_item(id="id3", title="Active 3")),
|
||||||
|
"id4": Mock(info=make_item(id="id4", title="Active 4")),
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshot = DownloadQueue.live_queue(queue_manager, limit=1)
|
||||||
|
|
||||||
|
queue_view = snapshot["queue"]
|
||||||
|
assert isinstance(queue_view, dict)
|
||||||
|
assert list(queue_view.keys()) == ["id3", "id4"]
|
||||||
|
assert snapshot["queue_count"] == 5
|
||||||
|
assert snapshot["queue_loaded"] == 2
|
||||||
|
assert snapshot["queue_limit"] == 1
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_live_reextracts(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
async def test_live_reextracts(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
seen: list[dict | None] = []
|
seen: list[dict | None] = []
|
||||||
|
|
@ -1258,7 +1400,7 @@ class TestQueueManager:
|
||||||
|
|
||||||
result = await add_video(
|
result = await add_video(
|
||||||
queue=self._video_queue(),
|
queue=self._video_queue(),
|
||||||
item=self._video_item(),
|
item=self._any_video_item(),
|
||||||
entry={
|
entry={
|
||||||
"id": "live-id",
|
"id": "live-id",
|
||||||
"title": "Live stream",
|
"title": "Live stream",
|
||||||
|
|
@ -1290,7 +1432,7 @@ class TestQueueManager:
|
||||||
"formats": [{"format_id": "18"}],
|
"formats": [{"format_id": "18"}],
|
||||||
}
|
}
|
||||||
|
|
||||||
result = await add_video(queue=self._video_queue(), item=self._video_item(), entry=entry)
|
result = await add_video(queue=self._video_queue(), item=self._any_video_item(), entry=entry)
|
||||||
|
|
||||||
assert result == {"status": "ok"}
|
assert result == {"status": "ok"}
|
||||||
assert seen == [entry]
|
assert seen == [entry]
|
||||||
|
|
@ -1415,7 +1557,11 @@ class TestQueueManager:
|
||||||
done_store.bulk_delete.assert_awaited_once_with(["done-id-1", "done-id-2"])
|
done_store.bulk_delete.assert_awaited_once_with(["done-id-1", "done-id-2"])
|
||||||
queue_manager._notify.emit.assert_called_once()
|
queue_manager._notify.emit.assert_called_once()
|
||||||
assert queue_manager._notify.emit.call_args.args[0] == Events.ITEM_BULK_DELETED
|
assert queue_manager._notify.emit.call_args.args[0] == Events.ITEM_BULK_DELETED
|
||||||
assert queue_manager._notify.emit.call_args.kwargs["data"] == {"ids": ["done-id-1", "done-id-2"], "count": 2}
|
assert queue_manager._notify.emit.call_args.kwargs["data"]["count"] == 2
|
||||||
|
assert queue_manager._notify.emit.call_args.kwargs["data"]["removed_files"] == 0
|
||||||
|
assert len(queue_manager._notify.emit.call_args.kwargs["data"]["items"]) == 2
|
||||||
|
assert queue_manager._notify.emit.call_args.kwargs["data"]["items"][0]["id"] == "done-id-1"
|
||||||
|
assert queue_manager._notify.emit.call_args.kwargs["data"]["items"][1]["id"] == "done-id-2"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_clear_status_fetches(self) -> None:
|
async def test_clear_status_fetches(self) -> None:
|
||||||
|
|
@ -1423,19 +1569,25 @@ class TestQueueManager:
|
||||||
queue_manager.config = Mock(remove_files=False, download_path="/tmp")
|
queue_manager.config = Mock(remove_files=False, download_path="/tmp")
|
||||||
queue_manager._notify = Mock()
|
queue_manager._notify = Mock()
|
||||||
|
|
||||||
|
item = Mock()
|
||||||
|
item.info = make_item(id="done-id", title="Cleared clip")
|
||||||
|
|
||||||
done_store = Mock()
|
done_store = Mock()
|
||||||
done_store.bulk_delete_by_status = AsyncMock(return_value=1)
|
done_store.bulk_delete_by_status = AsyncMock(return_value=1)
|
||||||
done_store.get_many_by_status = AsyncMock()
|
done_store.get_many_by_status = AsyncMock(return_value=[("done-id", item)])
|
||||||
queue_manager.done = done_store
|
queue_manager.done = done_store
|
||||||
|
|
||||||
result = await DownloadQueue.clear_by_status(queue_manager, "finished", remove_file=False)
|
result = await DownloadQueue.clear_by_status(queue_manager, "finished", remove_file=False)
|
||||||
|
|
||||||
assert result == {"deleted": 1}
|
assert result == {"deleted": 1}
|
||||||
|
done_store.get_many_by_status.assert_awaited_once_with("finished")
|
||||||
done_store.bulk_delete_by_status.assert_awaited_once_with("finished")
|
done_store.bulk_delete_by_status.assert_awaited_once_with("finished")
|
||||||
done_store.get_many_by_status.assert_not_called()
|
|
||||||
queue_manager._notify.emit.assert_called_once()
|
queue_manager._notify.emit.assert_called_once()
|
||||||
assert queue_manager._notify.emit.call_args.args[0] == Events.ITEM_BULK_DELETED
|
assert queue_manager._notify.emit.call_args.args[0] == Events.ITEM_BULK_DELETED
|
||||||
assert queue_manager._notify.emit.call_args.kwargs["data"] == {"count": 1, "status": "finished"}
|
assert queue_manager._notify.emit.call_args.kwargs["data"]["count"] == 1
|
||||||
|
assert queue_manager._notify.emit.call_args.kwargs["data"]["status"] == "finished"
|
||||||
|
assert len(queue_manager._notify.emit.call_args.kwargs["data"]["items"]) == 1
|
||||||
|
assert queue_manager._notify.emit.call_args.kwargs["data"]["items"][0]["id"] == "done-id"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_clear_status_files_fetch(self) -> None:
|
async def test_clear_status_files_fetch(self) -> None:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import json
|
import json
|
||||||
from datetime import date
|
from datetime import date
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
@ -132,7 +133,8 @@ class TestEncoder:
|
||||||
return True
|
return True
|
||||||
return original_isinstance(obj, cls)
|
return original_isinstance(obj, cls)
|
||||||
|
|
||||||
builtins.isinstance = mock_isinstance
|
builtins_any: Any = builtins
|
||||||
|
builtins_any.isinstance = mock_isinstance
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = self.encoder.default(mock_daterange)
|
result = self.encoder.default(mock_daterange)
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue