Compare commits
443 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8eadadc1ae | ||
|
|
151bb7b542 | ||
|
|
11c0c62a7e | ||
|
|
75f23641b3 | ||
|
|
f6360e90da | ||
|
|
02a319c350 | ||
|
|
ca1348b2c2 | ||
|
|
3f924812da | ||
|
|
b916e6a2f3 | ||
|
|
c35ec43466 | ||
|
|
a30a70fafd | ||
|
|
9aee723d1a | ||
|
|
9a8550661b | ||
|
|
458bf01599 | ||
|
|
cc8cc78b97 | ||
|
|
12cec4c042 | ||
|
|
e84d1bcde5 | ||
|
|
6a1fa78995 | ||
|
|
1b3c18e4e6 | ||
|
|
eb8498a792 | ||
|
|
8fe73ac924 | ||
|
|
b55c245fe6 | ||
|
|
f380e8ac27 | ||
|
|
bacc528ba1 | ||
|
|
6531a19c7a | ||
|
|
fc045cd74e | ||
|
|
f0e1fda0fc | ||
|
|
1722618c6d | ||
|
|
3fa7b48f4c | ||
|
|
23c64e000a | ||
|
|
f734905ba1 | ||
|
|
7140b88948 | ||
|
|
5bb90053bf | ||
|
|
846d217dca | ||
|
|
10895a7c16 | ||
|
|
7d7d242e5d | ||
|
|
b404420c76 | ||
|
|
fb9459fb5c | ||
|
|
b83f314617 | ||
|
|
7a90b008f0 | ||
|
|
3254e8a6c4 | ||
|
|
c47dccc7d7 | ||
|
|
8e071719f9 | ||
|
|
00d05a04f5 | ||
|
|
568e297d36 | ||
|
|
31d284ab0d | ||
|
|
877a86c9b0 | ||
|
|
e7552d3c62 | ||
|
|
24b79dc29e | ||
|
|
a275cbfb7a | ||
|
|
bbf7abad1b | ||
|
|
378ae39bef | ||
|
|
82cbc22577 | ||
|
|
09a1646a6a | ||
|
|
4f24ac2733 | ||
|
|
cd29239b2f | ||
|
|
1feb34621e | ||
|
|
73395b9668 | ||
|
|
dffc5c2e5d | ||
|
|
5906576a13 | ||
|
|
6734b6ab22 | ||
|
|
3f344a8cd9 | ||
|
|
d019e3aca1 | ||
|
|
f1b6fd5e31 | ||
|
|
28fe3d2b0b | ||
|
|
e7b1a239b4 | ||
|
|
d984895cba | ||
|
|
a68d9755a9 | ||
|
|
8236b95db0 | ||
|
|
78874313cc | ||
|
|
a8ce359cb7 | ||
|
|
ba6065f1b3 | ||
|
|
88553a95d2 | ||
|
|
1f1a239486 | ||
|
|
d162966cee | ||
|
|
70ad5af378 | ||
|
|
3937cb4cb8 | ||
|
|
1b39e6aa3f | ||
|
|
f111b5dda6 | ||
|
|
a6b2737a5c | ||
|
|
3ef0990ffc | ||
|
|
b26f664326 | ||
|
|
66bce2b83a | ||
|
|
7fac921afc | ||
|
|
0cf8654f47 | ||
|
|
9e845e760e | ||
|
|
ed2fbf2ae4 | ||
|
|
6fe82b2a78 | ||
|
|
a16afd1f9e | ||
|
|
5a53ffc8c2 | ||
|
|
14a32f6006 | ||
|
|
cf47032660 | ||
|
|
b6320d1a30 | ||
|
|
36f944be25 | ||
|
|
0514931140 | ||
|
|
8053bf339c | ||
|
|
c35f56bffa | ||
|
|
94e06f2d50 | ||
|
|
d925c34ce5 | ||
|
|
ae68750d9e | ||
|
|
cc2fb2ff79 | ||
|
|
fd66390648 | ||
|
|
5711963a6f | ||
|
|
49714a59ea | ||
|
|
01c101d24a | ||
|
|
a52dda6a7f | ||
|
|
efa64db04d | ||
|
|
366b5a266d | ||
|
|
9bb6c4ebd0 | ||
|
|
21e1944784 | ||
|
|
bdcd929558 | ||
|
|
91eae710b4 | ||
|
|
297709baa4 | ||
|
|
bb1074cc60 | ||
|
|
49222dd0b8 | ||
|
|
7061001f66 | ||
|
|
b23a3e91d8 | ||
|
|
7cd289e7b5 | ||
|
|
8349fa78dd | ||
|
|
0eac0ea46e | ||
|
|
d6776f9f57 | ||
|
|
57f254acaf | ||
|
|
9a3ca6ba4e | ||
|
|
f6f5561d0b | ||
|
|
7e504e03c6 | ||
|
|
afe3d0d04d | ||
|
|
55658f15da | ||
|
|
1fa3e1b599 | ||
|
|
5d488bceb4 | ||
|
|
40149d09f7 | ||
|
|
13b30a5997 | ||
|
|
3698ed1ec0 | ||
|
|
483cc61e08 | ||
|
|
71d48f704d | ||
|
|
5d8db8c5a4 | ||
|
|
c78b90826a | ||
|
|
937f3094c6 | ||
|
|
3af3a1cd24 | ||
|
|
5661ccbfd2 | ||
|
|
48f6cc5e3c | ||
|
|
1c3f255a66 | ||
|
|
d25be2c3c6 | ||
|
|
0bb77bf782 | ||
|
|
42c67a6d4d | ||
|
|
cb56ee2bee | ||
|
|
7e0726d5eb | ||
|
|
fff6448a14 | ||
|
|
ceccb4ee65 | ||
|
|
9f687c061a | ||
|
|
84fac1f80a | ||
|
|
89bedaf140 | ||
|
|
86fde7d62c | ||
|
|
305020f61e | ||
|
|
bf17ef3fab | ||
|
|
3490092e3d | ||
|
|
c5bffe51a5 | ||
|
|
44b074772b | ||
|
|
958a5cd0a4 | ||
|
|
9b1d76b4ff | ||
|
|
8803c1bd0c | ||
|
|
ecd9c25e3c | ||
|
|
ac7b47e4db | ||
|
|
2a61ac6a13 | ||
|
|
1dc727f0bb | ||
|
|
e880be9910 | ||
|
|
ddde6cba68 | ||
|
|
9f38c34900 | ||
|
|
cd13b1185e | ||
|
|
23cefd3c15 | ||
|
|
975f81f476 | ||
|
|
16ba035330 | ||
|
|
441f0e9ad1 | ||
|
|
0a0859df7a | ||
|
|
4d45cc614f | ||
|
|
887ee01cb2 | ||
|
|
8392de9207 | ||
|
|
8c2f66bea9 | ||
|
|
f98ceecd68 | ||
|
|
b3f704fbfd | ||
|
|
c33810f39f | ||
|
|
6f8a2a3f7a | ||
|
|
5a95619e22 | ||
|
|
314e587094 | ||
|
|
c55a4fa10e | ||
|
|
f7f21f44c5 | ||
|
|
530d6f181f | ||
|
|
6f14f15dd6 | ||
|
|
21a1e5ddbb | ||
|
|
278ba47519 | ||
|
|
856f14824f | ||
|
|
67197ce956 | ||
|
|
8e56273c9d | ||
|
|
5506155abe | ||
|
|
4788b54c91 | ||
|
|
ba7562c5f2 | ||
|
|
abb9d0e89e | ||
|
|
113a5ded38 | ||
|
|
62b815a3fe | ||
|
|
e141e3dcbd | ||
|
|
ef3d4cfdb4 | ||
|
|
28035f0d70 | ||
|
|
eb8f803c36 | ||
|
|
cb5ae93f76 | ||
|
|
9c3d1a5709 | ||
|
|
c7b5a93290 | ||
|
|
66f3f97460 | ||
|
|
267f11c848 | ||
|
|
d91693ba53 | ||
|
|
5012f44cfa | ||
|
|
4c16448ab3 | ||
|
|
cf03c28831 | ||
|
|
a3adf5e63c | ||
|
|
21ed97ff26 | ||
|
|
91bd895e52 | ||
|
|
c00df179ba | ||
|
|
b627f9f411 | ||
|
|
b5586d3f28 | ||
|
|
59dafff96a | ||
|
|
a83df05ce2 | ||
|
|
964b9bfdd7 | ||
|
|
220f2ad4de | ||
|
|
722b80001e | ||
|
|
3235b196b7 | ||
|
|
3a24fc24b1 | ||
|
|
9c3facfe7e | ||
|
|
217486b3c7 | ||
|
|
581735ac7d | ||
|
|
9524a40615 | ||
|
|
b6d8f07516 | ||
|
|
30dc587ebf | ||
|
|
ec5af17de7 | ||
|
|
ece4fbc792 | ||
|
|
3d67f51c29 | ||
|
|
0ff0d07c22 | ||
|
|
284acc8591 | ||
|
|
9084f1b7bb | ||
|
|
0456fa893f | ||
|
|
87750e2464 | ||
|
|
bbba2c8225 | ||
|
|
9348166bb5 | ||
|
|
b3fd6037bb | ||
|
|
591138a8a1 | ||
|
|
22adc3bc55 | ||
|
|
5ab3ec90a8 | ||
|
|
8941da8b60 | ||
|
|
f4f40c161b | ||
|
|
b440f2bcdc | ||
|
|
d30149db14 | ||
|
|
e952ea5f66 | ||
|
|
68383a16b3 | ||
|
|
b7cafb8b72 | ||
|
|
6d63016566 | ||
|
|
987cee7cb3 | ||
|
|
7c1644384d | ||
|
|
f383c82beb | ||
|
|
ce84e715c3 | ||
|
|
817de567ed | ||
|
|
efcaea314c | ||
|
|
7ef794fe74 | ||
|
|
11cc9aa789 | ||
|
|
853dbcf40f | ||
|
|
e8a0e1256b | ||
|
|
4d62df5c33 | ||
|
|
e5cd5cf08c | ||
|
|
2985514627 | ||
|
|
f7bd15d019 | ||
|
|
599f36fdf3 | ||
|
|
d90ae493dd | ||
|
|
1b456d548a | ||
|
|
bae8060c09 | ||
|
|
d3c875919f | ||
|
|
772703464b | ||
|
|
bbd081c44b | ||
|
|
261416fede | ||
|
|
6d3a59c8dc | ||
|
|
110f23f555 | ||
|
|
246c650db4 | ||
|
|
80d504f94d | ||
|
|
b316d283f7 | ||
|
|
a1b9f8a827 | ||
|
|
2b3ca12310 | ||
|
|
a9ec025706 | ||
|
|
5aa792d399 | ||
|
|
8043db0ad4 | ||
|
|
5c9b43a175 | ||
|
|
97a2023139 | ||
|
|
a8ebc9b8f7 | ||
|
|
4b7a915d39 | ||
|
|
0071358125 | ||
|
|
d722224b26 | ||
|
|
02e3b9d2e4 | ||
|
|
13d44f3ba0 | ||
|
|
9a5d187c0d | ||
|
|
419b939999 | ||
|
|
e24e9bfe1e | ||
|
|
35957b426d | ||
|
|
638269d870 | ||
|
|
21163239da | ||
|
|
240e386871 | ||
|
|
964675443b | ||
|
|
58ceca86b1 | ||
|
|
1b23b7da78 | ||
|
|
4287385af6 | ||
|
|
b344303f75 | ||
|
|
d49eb967a4 | ||
|
|
248e2c32f2 | ||
|
|
ffaa36105b | ||
|
|
e5a4dda117 | ||
|
|
6d72b7ca11 | ||
|
|
57c5148207 | ||
|
|
5856eefd7c | ||
|
|
3a8c803a54 | ||
|
|
8e00670491 | ||
|
|
5b6ef295f8 | ||
|
|
80dd2ff21c | ||
|
|
8f6f992670 | ||
|
|
f93c211de3 | ||
|
|
5f1ad517c2 | ||
|
|
a4d6e78bce | ||
|
|
b7ae32220b | ||
|
|
df09111831 | ||
|
|
c9900d91b0 | ||
|
|
b2dbd6dec5 | ||
|
|
f7d1b725d7 | ||
|
|
b2adc63a6a | ||
|
|
b8de46d2ad | ||
|
|
3359e3c111 | ||
|
|
621693ecc8 | ||
|
|
b567eb4808 | ||
|
|
e5724c6faa | ||
|
|
1423e50ccd | ||
|
|
3f439e4277 | ||
|
|
2f3e4b128b | ||
|
|
be54fccc63 | ||
|
|
df8f6a2acd | ||
|
|
8bbdd712f9 | ||
|
|
c5e30530c7 | ||
|
|
9c29414e6d | ||
|
|
6fcebe7c4b | ||
|
|
595ea2bfbd | ||
|
|
e0b9245ba3 | ||
|
|
b937343768 | ||
|
|
81546cff69 | ||
|
|
e94ea39187 | ||
|
|
8441ece6f0 | ||
|
|
6eed3d7775 | ||
|
|
f725235f44 | ||
|
|
c5ff2567a8 | ||
|
|
38d48bae37 | ||
|
|
d0c8aa9338 | ||
|
|
df889c0c6e | ||
|
|
0246842000 | ||
|
|
7081dad94e | ||
|
|
769dc62b87 | ||
|
|
d6d0dc537c | ||
|
|
87f414c8c7 | ||
|
|
288d44155d | ||
|
|
b50f7c12f4 | ||
|
|
f06728b0a7 | ||
|
|
aff4ecccc6 | ||
|
|
e2a8d3339a | ||
|
|
53391372c3 | ||
|
|
0598f5fbda | ||
|
|
c565fec59d | ||
|
|
6e526d7745 | ||
|
|
efa0632883 | ||
|
|
59c88fa0db | ||
|
|
b9b3b9eed3 | ||
|
|
8002f93220 | ||
|
|
986059626f | ||
|
|
fcd4af0efd | ||
|
|
80f1051e8a | ||
|
|
878e467f69 | ||
|
|
5e8143dd1d | ||
|
|
2306a5740c | ||
|
|
e1e0e29432 | ||
|
|
c450fa1f9a | ||
|
|
20f1214b68 | ||
|
|
81f2127e71 | ||
|
|
519685fc32 | ||
|
|
5a2113bd03 | ||
|
|
50632f6392 | ||
|
|
9b607b3d1b | ||
|
|
a483219746 | ||
|
|
fc68a6e741 | ||
|
|
50042607d7 | ||
|
|
8b01ada68a | ||
|
|
daecf9dfae | ||
|
|
a62e59dd3a | ||
|
|
9d2468a50f | ||
|
|
68a65a8e3d | ||
|
|
924d4ce8a6 | ||
|
|
798721d724 | ||
|
|
0e58973d43 | ||
|
|
776afdd1fd | ||
|
|
2901e4ec4d | ||
|
|
d35bb695ae | ||
|
|
941ebd42d9 | ||
|
|
0afd37351d | ||
|
|
aef80eb52d | ||
|
|
65ff84aa6a | ||
|
|
80679b02ba | ||
|
|
c17138bf7d | ||
|
|
093e14bd5d | ||
|
|
13e03a624c | ||
|
|
9eecd6d2c4 | ||
|
|
be141d0073 | ||
|
|
68582af374 | ||
|
|
12789b6be8 | ||
|
|
fd8d305651 | ||
|
|
9ae8c243c9 | ||
|
|
405e7097e3 | ||
|
|
a5ff4a1dd8 | ||
|
|
b5d0b76fe6 | ||
|
|
da3f89314b | ||
|
|
6f653f190a | ||
|
|
9974845470 | ||
|
|
fba47e9665 | ||
|
|
9d3cf14cdd | ||
|
|
e982b19bc5 | ||
|
|
7c8f06f427 | ||
|
|
41f6558c2d | ||
|
|
576199bb2f | ||
|
|
5b0b64bf3b | ||
|
|
0d86d84307 | ||
|
|
061079f0f6 | ||
|
|
bc334df719 | ||
|
|
7ab62674ed | ||
|
|
71f126f9d2 | ||
|
|
04fb19c80c | ||
|
|
d7ab68c067 | ||
|
|
6665ecaa12 | ||
|
|
462fa50423 | ||
|
|
401a9be0ec | ||
|
|
402a1fec50 | ||
|
|
dbe35fc023 | ||
|
|
a57951c038 | ||
|
|
c84231dd4f | ||
|
|
9330d66fcd | ||
|
|
e3d3f453da | ||
|
|
f5c0f3ca31 | ||
|
|
ac4bee75ef | ||
|
|
3202197740 |
448 changed files with 53757 additions and 30553 deletions
4
.github/workflows/docker-publish.yml
vendored
4
.github/workflows/docker-publish.yml
vendored
|
|
@ -9,9 +9,9 @@ on:
|
|||
workflow_dispatch:
|
||||
inputs:
|
||||
version_tag:
|
||||
description: 'Version tag (e.g. 2.8.2)'
|
||||
description: 'Version tag (e.g. 2.7.2)'
|
||||
required: true
|
||||
default: '2.8.2'
|
||||
default: '2.7.2'
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
|
|
|
|||
15
.gitignore
vendored
15
.gitignore
vendored
|
|
@ -12,12 +12,15 @@ __pycache__/
|
|||
|
||||
# User-specific files (auto-created by the app if missing)
|
||||
config/config.json
|
||||
config/youtube_cookies.txt
|
||||
# All app databases are live user data — never commit (music_library, video_library, …)
|
||||
database/*.db
|
||||
database/*.db-shm
|
||||
database/*.db-wal
|
||||
database/*.db.backup_*
|
||||
database/music_library.db
|
||||
database/music_library.db-shm
|
||||
database/music_library.db-wal
|
||||
database/music_library.db.backup_*
|
||||
database/video_library.db
|
||||
database/video_library.db-shm
|
||||
database/video_library.db-wal
|
||||
database/video_library.db.backup_*
|
||||
database/*.histbak
|
||||
database/api_call_history.json
|
||||
storage/image_cache/
|
||||
logs/*.log
|
||||
|
|
|
|||
|
|
@ -1,74 +0,0 @@
|
|||
# discover page — best in class plan (#913 + full generator audit)
|
||||
|
||||
morning notes. did the work overnight. tl;dr at top, details below, all of it `break nothing` + tested.
|
||||
|
||||
## what i shipped tonight (done, tested, safe)
|
||||
|
||||
### 1. listening recommendations (#913) — went from BROKEN to best-in-class
|
||||
|
||||
the feature was silently producing **zero** recs on real data. dug in and found three stacked bugs in the generation:
|
||||
|
||||
- **wrong id key (the killer).** `similar_artists.source_artist_id` is a *source* id (spotify/itunes/deezer), but the scanner built its id→name map from `artists.id` (the internal row id). so every edge resolved to nothing → 0 recs. proved it on your live db: internal-id join = 0 rows, spotify-id join = 71,636 rows.
|
||||
- **consensus could never fire.** it fed the ranker `get_top_similar_artists`, which does `GROUP BY similar_artist_name` + `MAX(source_artist_id)` — collapsing every similar artist down to a *single* seed. the whole point of the ranker is "artist X is similar to 3 of your seeds = strong signal," and that signal was being flattened away before it ever reached the ranker.
|
||||
- **similarity strength thrown away.** each edge stores a 1-10 closeness rank; it was ignored (everything weighted equally).
|
||||
|
||||
the fix (all in the pure, tested core + thin scan wiring):
|
||||
- build id→name from the **source-id columns**, query the **raw per-seed edges** (consensus preserved), and thread **similarity_rank** into the score so a seed's closest matches count for more.
|
||||
- **recency-weighted seeds**: `weight = lifetime_plays + 1.5 × recent_30d_plays`. picks now track what you're into *now*, not just all-time totals.
|
||||
|
||||
result on your actual library (simulated through the real code path): **40 recommendations, 13 with multi-seed consensus, all 40 with cached art.** top picks: Arcangel (Bad Bunny + Ozuna + J Balvin), Melanie Martinez (Ariana + Billie), Maluma, De La Ghetto — all coherent, all explainable.
|
||||
|
||||
### 2. its own row on the discover page
|
||||
|
||||
new row **"Based On Your Listening"** — play-weighted, consensus-ranked artist cards with a **"Because you listen to X, Y"** line. sits right above the library-driven "Recommended For You" row. purely additive: new endpoint `/api/discover/listening-recommendations`, new loader, hides itself when empty.
|
||||
|
||||
**you need to run one watchlist scan** for the row to populate (the data regenerates during the scan — i did NOT touch your live db). before that scan the row just stays hidden; after it, it fills in.
|
||||
|
||||
> note: this is deliberately different from the existing "Recommended For You" row. that one is driven by your *whole library / watchlist*. this one is driven by your *actual listening intensity* — the ~30 artists you really play, not the thousands you happen to own.
|
||||
|
||||
### 3. Fresh Tape "only 5-10 tracks" — fixed
|
||||
|
||||
root cause: `get_discovery_recent_albums` orders `release_date DESC`, so announced-but-unreleased albums sort to the *top* and ate the 50-album budget. the scanner skipped them *after* the budget was already spent → only a handful of released albums left → 5-10 tracks. fixed by fetching a generous budget (300) **and** excluding next-year albums at the query, so released albums fill the budget. the precise same-year `is_future_release` skip stays as a second guard. downstream caps (6/artist, top 75, take 50) unchanged.
|
||||
|
||||
**tests:** 25 pure-core cases (consensus/similarity/recency) + 2 Fresh Tape regression tests, all green. full discovery suite (255) green. nothing else touched.
|
||||
|
||||
---
|
||||
|
||||
## best-in-class roadmap for listening recs (next phases — your call)
|
||||
|
||||
these are the levers to take it further. ordered by value-to-risk. none are required; tonight's work stands on its own.
|
||||
|
||||
| phase | what | value | risk | notes |
|
||||
|---|---|---|---|---|
|
||||
| **3** | **playable track row** ✅ DONE | high | low-med | shipped: "🎧 Your Listening Mix" row — a track playlist (play/queue/download/sync) right under the artist row. stored as full render-ready dicts (not pool-hydrated, so it can't shrink on pool rotation like Fresh Tape does). |
|
||||
| **4** | **direct top-tracks fetch** ✅ DONE | high | med | shipped: scan fetches each recommended artist's top tracks (Spotify/Deezer), resolving the artist id by name-search when the similar-artist row lacks one — guarded by a strict name-match so it never pulls the wrong artist. bounded (top 20 recs), per-call guarded, fail-soft to the pool. iTunes has no top-tracks API → pool-only there. needs a live scan to populate. |
|
||||
| **5** | **genre-affinity boost** | med | low | we already compute your genre breakdown. boost recs whose genres match your top genres → tighter taste alignment. pure scoring add. |
|
||||
| **6** | **adventurousness dial** | med | low | the ranker already supports `min_seed_count` (consensus floor). expose it as a "Safe ↔ Adventurous" slider on the row. |
|
||||
| **7** | **diversity pass** | low-med | low | avoid 40 recs all orbiting your single heaviest seed — cap picks-per-seed so the row spans your taste. |
|
||||
|
||||
the core is built to absorb all of these without re-plumbing — `similarity_from_rank`, `build_recency_weighted_seeds`, and the scoring formula are all pure + tested.
|
||||
|
||||
---
|
||||
|
||||
## full discover-page generator audit (every soulsync-built row, excluding last.fm + listenbrainz)
|
||||
|
||||
how each one is generated today, and whether it can be elevated. "clear win" = safe + additive. "product call" = needs your decision (changes the row's character).
|
||||
|
||||
### curated (built during the scan, then hydrated)
|
||||
- **Fresh Tape / Release Radar** — new releases from watchlist+similar artists. **FIXED tonight** (see above). one more *clear win* available: hydration silently drops any curated id no longer in the discovery pool — could fall back to the stored `track_data_json` blob so the row can't shrink at read time.
|
||||
- **The Archives / Discovery Weekly** — strong already. nice 3-tier popularity split + serendipity scoring (boost never-played artists, penalize overplayed). same hydration-drop caveat as Fresh Tape; same cheap fallback fix.
|
||||
- **Seasonal Mix** — cleanest of the bunch. hydrates from a dedicated `seasonal_tracks` table (carries its own data), so it doesn't suffer the pool-drop problem. no bug.
|
||||
|
||||
### discovery-pool generators (live queries)
|
||||
- **Popular Picks** — ranks by popularity DESC. solid. only nit: on iTunes (no popularity scale) it silently degrades to random — indistinguishable from Shuffle there. UI-label thing at most.
|
||||
- **Hidden Gems** — *clear win*. currently `ORDER BY RANDOM()` over low-popularity tracks — so it's "random obscure," not "*best* obscure." a light ranking (popularity just under the threshold, or genre-affinity to you) would make it feel curated instead of arbitrary. (a deeper *product call*: add personalization like Archives has — bigger lift, changes its "pure underground" character.)
|
||||
- **Genre Playlists** — good. pushes the genre match into SQL. `RANDOM()` ordering is fine for a browse; a popularity/affinity tiebreak (*clear win*) would make thin genres feel less arbitrary.
|
||||
- **Discovery Shuffle** — random by design, correct. only possible add: exclude tracks already shown in other rows this refresh (needs a cross-section seen-set — medium plumbing).
|
||||
- **Time Machine (by decade)** — *clear win, low risk*: decades are hardcoded, so a modern-only library shows 7 decade tabs, 5 empty. filter the tabs to decades that actually have pool data.
|
||||
- **Daily Mix** — the weakest row. the "50% your library" half permanently returns nothing (library tracks have no source ids to play), so each Daily Mix is really just a relabeled Genre Playlist. real fix = backfill source ids into library rows (*schema-level, higher risk*) — worth a dedicated pass, not a quick tweak. also silently falls back to "top artists as pseudo-genres" when genre data is missing → "Daily Mix 1" becomes mislabeled artist-radio. gate/label that (*clear win*).
|
||||
|
||||
### cross-cutting
|
||||
- **hydration fragility** (Fresh Tape + Archives): both depend on curated ids still living in the pool at read time; misses are dropped silently. Seasonal already solved this with a dedicated table. giving the two spotify-style rows the same data-blob fallback is the single most robust cross-cutting fix. low risk, clear win.
|
||||
- **RANDOM-ordering pattern** (Hidden Gems, Shuffle, Genre, Decade): intentional for variety, but leaves quality signal on the table for the non-shuffle rows. adding a light ranking pass to Hidden Gems + Genre is the biggest "best-in-class" lever after tonight's work.
|
||||
|
||||
want me to take any of these? the Hidden Gems ranking + Time Machine empty-decade filter + the Fresh Tape/Archives hydration fallback are all safe, additive, same-shape-as-tonight wins i can knock out next.
|
||||
BIN
Issue/1.png
Normal file
BIN
Issue/1.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 563 KiB |
|
|
@ -1,13 +0,0 @@
|
|||
**SoulSync 2.7.9** is out 🎉 a big one.
|
||||
|
||||
🎚️ **Best-quality downloads** — downloads now follow a ranked quality profile you drag to order (FLAC 24/192 → mp3). best-quality mode grabs the highest-quality copy across *every* source; priority mode gets an opt-in rank-based order toggle. quarantine is folded into the Downloads page + safer imports (AcoustID fail-closed, silence/truncation guards).
|
||||
|
||||
🎧 **Discover got smart** — "Based On Your Listening" ranks artists from who you *actually play*, and "Your Listening Mix" is a playable track playlist of their top tracks (works on any source). Fresh Tape fills properly now.
|
||||
|
||||
⚡ **Wing It Pool** — a new spot next to Discovery Pool to review + re-match the tracks Wing It guessed at (they used to be invisible).
|
||||
|
||||
🔁 **Auto-Sync redesign** — the scheduling board is now clean horizontal lanes instead of a side-scrolling column wall.
|
||||
|
||||
🐛 **Fixes** — multi-disc albums no longer show disc-2 as "missing" (#927), playlists no longer stuck on "Never Synced" (#925), and tracks can't import while quarantined (#928). thanks @ramonskie + @nick2000713 🙏
|
||||
|
||||
⚠️ **re-scan your library once** so the multi-disc fix can backfill disc numbers on existing tracks. enjoy! 🎶
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
**SoulSync 2.8.0** is out 🎉 a quality + reliability release.
|
||||
|
||||
🧹 **The Unverified queue, finally under control** — if you saw thousands of "unverified" rows piling up, this is for you. the AcoustID scan stops duplicating history rows, a one-time reconcile on startup clears the existing backlog from your library (no re-scan), and a new 🧹 *Clean orphaned* button sweeps dead rows whose file is gone. (#934 — thanks @nick2000713 for #938)
|
||||
|
||||
✂️ **Preview Clip Cleanup** — a new Tools job that finds the ~30s preview clips the HiFi source sometimes hands back instead of the full song, then deletes them and re-wishlists the real version. each finding has a ▶ Play button so you can confirm before approving.
|
||||
|
||||
💿 **Album Completeness handles split albums** — an album split across multiple library rows no longer shows every fragment as falsely "incomplete"; it groups the validated fragments into one correct finding. (#936 — thanks @ragnarlotus)
|
||||
|
||||
🐛 **Fixes** — pasted YouTube cookies no longer throw `unsupported browser: "custom"` on Docker (thanks HellRa1SeR); longer remasters aren't quarantined as "truncated" anymore (#937, thanks @diegocade1); "Add to Wishlist" from a discography went from ~15–30s *per track* to instant; wishlist art renders for re-downloads; and **Clear Completed** is back on the Downloads page.
|
||||
|
||||
⚡ **Performance** — trimmed the dashboard GPU usage that was hammering Firefox/Zen (and Background Particles are OFF by default now), plus bounded the runaway memory growth that could lock the app up on big libraries. (#935 / #802)
|
||||
|
||||
enjoy! 🎶
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
**SoulSync 2.8.1** is out 🎉 a feature + reliability release.
|
||||
|
||||
🎧 **Export playlists to Spotify & Deezer** — the mirrored-playlist export now has **Sync to Spotify** and **Sync to Deezer** next to the ListenBrainz / JSPF options. it builds a playlist in your account from the IDs soulsync already has (the discovery cache first, then your library), so an already-discovered playlist exports **instantly with zero API calls**. re-exporting updates the same playlist instead of duplicating it, and an optional *"match missing tracks"* toggle confidently searches for the stragglers — a wrong-artist or karaoke version is left out, never guessed. the first Spotify export asks permission once. (#945)
|
||||
|
||||
🏷️ **Library Reorganize — Rename only** — a lighter action that just **renames your files** to your naming scheme: no re-tag, no quality/AcoustID re-check, no copy-to-staging. much faster on a NAS, and only touches files whose path actually changes. pick it from the new Action dropdown. (#875 — thanks @tsoulard / @Tacobell444)
|
||||
|
||||
💿 **Broader lossless handling** — lossy-copy now covers **all lossless formats**, not just FLAC (#941); and **DSD** (`.dsf`/`.dff`) is recognized as lossless instead of false-flagged "truncated" (#939).
|
||||
|
||||
🐛 **Download + search fixes** — an unbalanced bracket no longer false-fails as "file not found"; a file we couldn't quarantine is left for retry instead of deleted; "file not found" errors are actionable now; pasted Qobuz/Tidal links inject the exact track into manual search (#932); the Wing It pool "Fix Match" works again.
|
||||
|
||||
⚡ **Reduce visual effects, refined** — it no longer freezes functional motion (spinners, progress), only the expensive GPU stuff (blur, shadows, glow). worker orbs default OFF on Firefox and run at ~30fps under reduce-effects. plus a jellyfin scan watchdog fix for big libraries.
|
||||
|
||||
🔧 **Under the hood** — settings cleanup (#943, @nick2000713), spotify oauth hardening (#942) + npm security fixes (#944, HellRa1SeR).
|
||||
|
||||
enjoy! 🎶
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
**SoulSync 2.8.2** is out 🎉 a stability + performance release.
|
||||
|
||||
🎧 **Spotify, reliably** — the Docker boot hang is fixed: with Spotify as your primary source, an unreachable Spotify API could block startup so the container bound `:8008` but never served the UI. auth probes are now deferred during boot + capped with a timeout. the "re-auth didn't stick" bug is fixed too (the OAuth callback and the app were reading different token caches), and **Sync to Spotify** now works — it asks for playlist-write permission once, on-demand, leaving your normal login untouched. (#949 — thanks HellRa1SeR)
|
||||
|
||||
⚡ **The "slow after update" fix** — the post-update lag wasn't SoulSync, it was browser password managers (Bitwarden/1Password/etc.) rebuilding their autofill overlay on *every* DOM change. non-credential fields are now marked so they skip them — **~110× less main-thread blocking** in the reporter's benchmark. plus a new **Max Performance** mode (Settings → Appearance) that kills every effect for no-GPU / Docker setups. (#948 — thanks @nick2000713)
|
||||
|
||||
📥 **Large-library imports no longer time out** — dropping a whole library into staging used to make the import page scan every file synchronously and never load. the scan runs in the background now with a live "Scanning N of M…" progress, and fills in when done. (#947 — thanks @ramonskie)
|
||||
|
||||
enjoy! 🎶
|
||||
73
api/video/__init__.py
Normal file
73
api/video/__init__.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
"""SoulSync — VIDEO side API package (isolated).
|
||||
|
||||
A SEPARATE Flask blueprint from the music API (api_v1). It reads only
|
||||
database/video_library.db via VideoDatabase and imports nothing from the music
|
||||
API or music database layer. Registered in web_server.py with a single additive
|
||||
line at url_prefix '/api/video', so music routing is untouched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
from flask import Blueprint
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api")
|
||||
|
||||
# Lazily-created, process-wide VideoDatabase handle. VideoDatabase itself guards
|
||||
# schema init once-per-path, so this just avoids re-opening the wrapper.
|
||||
_video_db = None
|
||||
_video_db_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_video_db():
|
||||
"""Return the shared VideoDatabase instance (created on first use)."""
|
||||
global _video_db
|
||||
if _video_db is None:
|
||||
with _video_db_lock:
|
||||
if _video_db is None:
|
||||
from database.video_database import VideoDatabase
|
||||
_video_db = VideoDatabase()
|
||||
return _video_db
|
||||
|
||||
|
||||
def create_video_blueprint() -> Blueprint:
|
||||
"""Build the isolated /api/video blueprint with all video sub-routes."""
|
||||
bp = Blueprint("video_api", __name__)
|
||||
|
||||
from .dashboard import register_routes as reg_dashboard
|
||||
from .scan import register_routes as reg_scan
|
||||
from .library import register_routes as reg_library
|
||||
from .libraries import register_routes as reg_libraries
|
||||
from .poster import register_routes as reg_poster
|
||||
from .enrichment import register_routes as reg_enrichment
|
||||
from .detail import register_routes as reg_detail
|
||||
from .search import register_routes as reg_search
|
||||
from .discover import register_routes as reg_discover
|
||||
from .calendar import register_routes as reg_calendar
|
||||
from .watchlist import register_routes as reg_watchlist
|
||||
from .wishlist import register_routes as reg_wishlist
|
||||
from .youtube import register_routes as reg_youtube
|
||||
from .downloads import register_routes as reg_downloads
|
||||
from .manual_import import register_routes as reg_manual_import
|
||||
from .automations import register_routes as reg_automations
|
||||
reg_dashboard(bp)
|
||||
reg_scan(bp)
|
||||
reg_library(bp)
|
||||
reg_libraries(bp)
|
||||
reg_poster(bp)
|
||||
reg_enrichment(bp)
|
||||
reg_detail(bp)
|
||||
reg_search(bp)
|
||||
reg_discover(bp)
|
||||
reg_calendar(bp)
|
||||
reg_watchlist(bp)
|
||||
reg_wishlist(bp)
|
||||
reg_youtube(bp)
|
||||
reg_downloads(bp)
|
||||
reg_manual_import(bp)
|
||||
reg_automations(bp)
|
||||
|
||||
return bp
|
||||
28
api/video/automations.py
Normal file
28
api/video/automations.py
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
"""Video automation builder support (isolated /api/video routes).
|
||||
|
||||
The automation ENGINE is app-wide (shared with the music side), but the
|
||||
builder block palette is scoped: the video builder must only ever offer
|
||||
video + generic blocks, never music-only ones. This endpoint returns that
|
||||
scoped slice via ``core.automation.blocks.blocks_for_scope('video')``.
|
||||
|
||||
ISOLATION: imports only the shared automation block definitions (which
|
||||
themselves import nothing music-specific) — no music API / music DB.
|
||||
Reading/running/toggling video automations goes through the shared
|
||||
``/api/automations`` endpoints (owned_by='video' rows); only the scoped
|
||||
block palette lives here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.automations")
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/automations/blocks", methods=["GET"])
|
||||
def video_automation_blocks():
|
||||
from core.automation.blocks import blocks_for_scope
|
||||
return jsonify(blocks_for_scope("video"))
|
||||
79
api/video/calendar.py
Normal file
79
api/video/calendar.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
"""Video Calendar — upcoming TV episodes for OWNED shows.
|
||||
|
||||
GET /api/video/calendar?days=N → episodes airing from today through today+N-1,
|
||||
grouped client-side into the agenda view. Isolated: reads only video_library.db
|
||||
via VideoDatabase, writes nothing, never touches the music side.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime, timedelta
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video.calendar")
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/calendar", methods=["GET"])
|
||||
def video_calendar():
|
||||
from . import get_video_db
|
||||
try:
|
||||
days = request.args.get("days", default=7, type=int) or 7
|
||||
days = max(1, min(days, 31)) # one week (or a few) per view
|
||||
today = date.today()
|
||||
# Optional ?start=YYYY-MM-DD for week navigation; default = today.
|
||||
start = today
|
||||
start_s = request.args.get("start")
|
||||
if start_s:
|
||||
try:
|
||||
start = datetime.strptime(start_s, "%Y-%m-%d").date()
|
||||
except ValueError:
|
||||
start = today
|
||||
if abs((start - today).days) > 400: # sane bound around today
|
||||
start = today
|
||||
end = start + timedelta(days=days - 1)
|
||||
db = get_video_db()
|
||||
|
||||
# One-time backfill: existing shows matched TVDB before air time was
|
||||
# captured, so re-queue them once (background, only those missing it).
|
||||
try:
|
||||
if (db.get_setting("airs_time_backfill") or "") != "1":
|
||||
n = db.requeue_shows_for_airtime()
|
||||
db.set_setting("airs_time_backfill", "1")
|
||||
if n:
|
||||
logger.info("calendar: queued %d shows for TVDB air-time backfill", n)
|
||||
except Exception:
|
||||
logger.exception("airs_time backfill queue failed")
|
||||
|
||||
from core.video.sources import resolve_video_server
|
||||
# scope: 'watchlist' (default — shows you follow/track) or 'all' (every
|
||||
# airing show in the library). The toggle on the page sends ?scope=.
|
||||
scope = (request.args.get("scope") or "watchlist").lower()
|
||||
eps = db.calendar_upcoming(start.isoformat(), end.isoformat(),
|
||||
server_source=resolve_video_server(),
|
||||
watchlist_only=(scope != "all"))
|
||||
|
||||
# Per-date counts drive the day-strip dots without a second query.
|
||||
counts: dict[str, int] = {}
|
||||
owned = 0
|
||||
for e in eps:
|
||||
counts[e["air_date"]] = counts.get(e["air_date"], 0) + 1
|
||||
if e.get("has_file"):
|
||||
owned += 1
|
||||
return jsonify({
|
||||
"today": today.isoformat(), # real today (for the highlight)
|
||||
"start": start.isoformat(), # window start (may be a future week)
|
||||
"end": end.isoformat(),
|
||||
"days": days,
|
||||
"scope": "all" if scope == "all" else "watchlist",
|
||||
"counts_by_date": counts,
|
||||
"total": len(eps),
|
||||
"owned": owned,
|
||||
"episodes": eps,
|
||||
})
|
||||
except Exception:
|
||||
logger.exception("video calendar failed")
|
||||
return jsonify({"error": "calendar failed"}), 500
|
||||
28
api/video/dashboard.py
Normal file
28
api/video/dashboard.py
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
"""Video dashboard endpoint — live counts from video.db.
|
||||
|
||||
GET /api/video/dashboard -> {library:{...}, downloads:{...}, watchlist, wishlist}
|
||||
With an empty database every value is a real 0.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.dashboard")
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/dashboard", methods=["GET"])
|
||||
def video_dashboard():
|
||||
from . import get_video_db
|
||||
from core.video.sources import resolve_video_server
|
||||
try:
|
||||
server = resolve_video_server() # the VIDEO server, not music's active
|
||||
stats = get_video_db().dashboard_stats(server_source=server)
|
||||
stats["server"] = server
|
||||
return jsonify(stats)
|
||||
except Exception:
|
||||
logger.exception("Failed to build video dashboard stats")
|
||||
return jsonify({"error": "Failed to load video dashboard stats"}), 500
|
||||
135
api/video/detail.py
Normal file
135
api/video/detail.py
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
"""Video detail payloads (drill-in pages).
|
||||
|
||||
GET /api/video/detail/show/<id> → show + seasons→episodes tree (owned roll-ups)
|
||||
GET /api/video/detail/movie/<id> → movie + owned/file info
|
||||
|
||||
Reads only video.db; isolated from the music API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.detail")
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/monitor", methods=["POST"])
|
||||
def video_set_monitor():
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
kind, item_id = body.get("kind"), body.get("id")
|
||||
if kind not in ("movie", "show") or not isinstance(item_id, int):
|
||||
return jsonify({"error": "bad request"}), 400
|
||||
ok = get_video_db().set_monitored(kind, item_id, bool(body.get("monitored")))
|
||||
if not ok:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
return jsonify({"success": True, "monitored": bool(body.get("monitored"))})
|
||||
|
||||
@bp.route("/detail/show/<int:show_id>", methods=["GET"])
|
||||
def video_show_detail(show_id):
|
||||
from . import get_video_db
|
||||
data = get_video_db().show_detail(show_id)
|
||||
if not data:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
return jsonify(data)
|
||||
|
||||
@bp.route("/detail/movie/<int:movie_id>", methods=["GET"])
|
||||
def video_movie_detail(movie_id):
|
||||
from . import get_video_db
|
||||
data = get_video_db().movie_detail(movie_id)
|
||||
if not data:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
return jsonify(data)
|
||||
|
||||
@bp.route("/detail/show/<int:show_id>/refresh-art", methods=["POST"])
|
||||
def video_show_refresh_art(show_id):
|
||||
"""Lazy on-view backfill: pull missing season posters / episode art from
|
||||
TMDB and cache them. Best-effort — never errors the page."""
|
||||
try:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
res = get_video_enrichment_engine().refresh_show_art(show_id)
|
||||
except Exception:
|
||||
logger.exception("refresh-art failed for show %s", show_id)
|
||||
res = {"ok": False, "reason": "error"}
|
||||
return jsonify(res)
|
||||
|
||||
@bp.route("/detail/movie/<int:movie_id>/refresh-art", methods=["POST"])
|
||||
def video_movie_refresh_art(movie_id):
|
||||
"""Lazy on-view backfill for a movie (cast / genres / backdrop / ratings)."""
|
||||
try:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
res = get_video_enrichment_engine().refresh_movie_art(movie_id)
|
||||
except Exception:
|
||||
logger.exception("refresh-art failed for movie %s", movie_id)
|
||||
res = {"ok": False, "reason": "error"}
|
||||
return jsonify(res)
|
||||
|
||||
@bp.route("/tmdb/<kind>/<int:tmdb_id>", methods=["GET"])
|
||||
def video_tmdb_detail(kind, tmdb_id):
|
||||
"""Full detail for a TMDB title not in the library (the search → detail
|
||||
view). May return {redirect:{source,kind,id}} if it's actually owned."""
|
||||
if kind not in ("movie", "show"):
|
||||
return jsonify({"error": "bad kind"}), 400
|
||||
try:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
d = get_video_enrichment_engine().tmdb_detail(kind, tmdb_id)
|
||||
except Exception:
|
||||
logger.exception("tmdb detail failed for %s %s", kind, tmdb_id)
|
||||
d = None
|
||||
if not d:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
return jsonify(d)
|
||||
|
||||
@bp.route("/tmdb/show/<int:tv_id>/season/<int:season_number>", methods=["GET"])
|
||||
def video_tmdb_season(tv_id, season_number):
|
||||
"""Lazy per-season episodes for a TMDB (un-owned) show detail."""
|
||||
try:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
d = get_video_enrichment_engine().tmdb_season(tv_id, season_number)
|
||||
except Exception:
|
||||
logger.exception("tmdb season failed for %s S%s", tv_id, season_number)
|
||||
d = None
|
||||
if not d:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
return jsonify(d)
|
||||
|
||||
@bp.route("/episode/<int:tmdb_id>/<int:season>/<int:episode>", methods=["GET"])
|
||||
def video_episode_extra(tmdb_id, season, episode):
|
||||
"""Episode expand: guest stars + bigger still (by the SHOW's tmdb id)."""
|
||||
try:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
d = get_video_enrichment_engine().episode_extra(tmdb_id, season, episode)
|
||||
except Exception:
|
||||
logger.exception("episode extra failed for %s S%sE%s", tmdb_id, season, episode)
|
||||
d = None
|
||||
if not d:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
return jsonify(d)
|
||||
|
||||
@bp.route("/person/<int:tmdb_id>", methods=["GET"])
|
||||
def video_person_detail(tmdb_id):
|
||||
"""In-app person page: bio + filmography (each credit annotated owned/not)."""
|
||||
try:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
d = get_video_enrichment_engine().person_detail(tmdb_id)
|
||||
except Exception:
|
||||
logger.exception("person detail failed for %s", tmdb_id)
|
||||
d = None
|
||||
if not d:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
return jsonify(d)
|
||||
|
||||
@bp.route("/detail/<kind>/<int:item_id>/extras", methods=["GET"])
|
||||
def video_detail_extras(kind, item_id):
|
||||
"""Live TMDB extras (trailer / where-to-watch / similar) for the detail page."""
|
||||
if kind not in ("movie", "show"):
|
||||
return jsonify({}), 400
|
||||
try:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
return jsonify(get_video_enrichment_engine().item_extras(kind, item_id))
|
||||
except Exception:
|
||||
logger.exception("extras failed for %s %s", kind, item_id)
|
||||
return jsonify({})
|
||||
386
api/video/discover.py
Normal file
386
api/video/discover.py
Normal file
|
|
@ -0,0 +1,386 @@
|
|||
"""Video discover API — browse TMDB (movies/TV the user doesn't own yet).
|
||||
|
||||
GET /api/video/discover/hero → trending titles w/ backdrops (slideshow)
|
||||
GET /api/video/discover/genres → {movie:[{id,name}], show:[{id,name}]}
|
||||
GET /api/video/discover/list?... → one shelf/grid of items, e.g.
|
||||
?key=trending → trending movies + shows
|
||||
?key=<curated>&page= → a canned list (popular_movies, top_shows…)
|
||||
?kind=movie|show&genre=&year=&decade=&sort=&page= → a filtered browse
|
||||
|
||||
Items are annotated with ``library_id`` when already owned (so the card links to
|
||||
the owned detail, not the TMDB preview). Reads only the enrichment engine +
|
||||
video.db; isolated from the music API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.discover")
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/discover/hero", methods=["GET"])
|
||||
def video_discover_hero():
|
||||
"""A few trending titles that have a backdrop — drives the hero slideshow."""
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
try:
|
||||
items = [x for x in (get_video_enrichment_engine().trending() or [])
|
||||
if x.get("backdrop")][:6]
|
||||
except Exception:
|
||||
logger.exception("discover hero failed")
|
||||
items = []
|
||||
return jsonify({"items": items})
|
||||
|
||||
@bp.route("/discover/taste", methods=["GET"])
|
||||
def video_discover_taste():
|
||||
"""The user's most-owned genres (movies + shows) → personalized rails."""
|
||||
from . import get_video_db
|
||||
try:
|
||||
from core.video.sources import resolve_video_server
|
||||
srv = resolve_video_server()
|
||||
except Exception:
|
||||
srv = None
|
||||
db = get_video_db()
|
||||
try:
|
||||
return jsonify({"movie": db.top_owned_genres("movie", srv, 6),
|
||||
"show": db.top_owned_genres("show", srv, 6)})
|
||||
except Exception:
|
||||
logger.exception("discover taste failed")
|
||||
return jsonify({"movie": [], "show": []})
|
||||
|
||||
@bp.route("/discover/morelike", methods=["GET"])
|
||||
def video_discover_morelike():
|
||||
"""'More like <owned title>' rails — TMDB recommendations seeded from a few
|
||||
random titles you already own (interleaved movie/show, max 3 rails)."""
|
||||
from . import get_video_db
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
try:
|
||||
from core.video.sources import resolve_video_server
|
||||
srv = resolve_video_server()
|
||||
except Exception:
|
||||
srv = None
|
||||
db = get_video_db()
|
||||
eng = get_video_enrichment_engine()
|
||||
try:
|
||||
seeds = db.random_owned_titles(2, srv)
|
||||
movies = [s for s in seeds if s["kind"] == "movie"]
|
||||
shows = [s for s in seeds if s["kind"] == "show"]
|
||||
ordered = []
|
||||
while (movies or shows) and len(ordered) < 3:
|
||||
if movies:
|
||||
ordered.append(movies.pop(0))
|
||||
if shows and len(ordered) < 3:
|
||||
ordered.append(shows.pop(0))
|
||||
rails = []
|
||||
for s in ordered:
|
||||
items = [it for it in eng.recommendations(s["kind"], s["tmdb_id"])
|
||||
if it.get("tmdb_id") != s["tmdb_id"]]
|
||||
if len(items) >= 4:
|
||||
rails.append({"title": "More like " + s["title"], "items": items[:30]})
|
||||
return jsonify({"rails": rails})
|
||||
except Exception:
|
||||
logger.exception("discover morelike failed")
|
||||
return jsonify({"rails": []})
|
||||
|
||||
@bp.route("/discover/foryou", methods=["GET"])
|
||||
def video_discover_foryou():
|
||||
"""A single 'Recommended for you' wall blended from many owned titles — a title
|
||||
recommended by more of your library ranks higher (consensus)."""
|
||||
from . import get_video_db
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
from core.video.discovery_recs import blend_recommendations
|
||||
try:
|
||||
from core.video.sources import resolve_video_server
|
||||
srv = resolve_video_server()
|
||||
except Exception:
|
||||
srv = None
|
||||
db = get_video_db()
|
||||
eng = get_video_enrichment_engine()
|
||||
try:
|
||||
seeds = db.random_owned_titles(6, srv) # up to 6 movies + 6 shows
|
||||
seed_ids = [s["tmdb_id"] for s in seeds if s.get("tmdb_id")]
|
||||
rec_lists = [eng.recommendations(s["kind"], s["tmdb_id"])
|
||||
for s in seeds if s.get("tmdb_id")]
|
||||
items = blend_recommendations(rec_lists, exclude_ids=seed_ids, limit=40)
|
||||
return jsonify({"items": items})
|
||||
except Exception:
|
||||
logger.exception("discover foryou failed")
|
||||
return jsonify({"items": []})
|
||||
|
||||
@bp.route("/discover/gaps", methods=["GET"])
|
||||
def video_discover_gaps():
|
||||
"""'What am I missing?' rails — franchises you've started but not finished, and
|
||||
more from the directors/creators you own the most. Powered by the gap engine."""
|
||||
from . import get_video_db
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
from core.video.discovery_gaps import collection_gaps, filmography_gaps
|
||||
try:
|
||||
from core.video.sources import resolve_video_server
|
||||
srv = resolve_video_server()
|
||||
except Exception:
|
||||
srv = None
|
||||
db = get_video_db()
|
||||
eng = get_video_enrichment_engine()
|
||||
try:
|
||||
# Lazy collection-id backfill: movies matched before the collection column
|
||||
# exists have no franchise id. Fill a small batch each load (self-healing);
|
||||
# isolated so a backfill hiccup never breaks the gap rails.
|
||||
try:
|
||||
for mv in db.movies_missing_collection(srv, limit=20):
|
||||
coll = eng.movie_collection(mv["tmdb_id"])
|
||||
if coll is not None:
|
||||
db.set_movie_collection(mv["id"], coll.get("id"), coll.get("name"))
|
||||
except Exception:
|
||||
logger.exception("collection-id backfill batch failed")
|
||||
|
||||
owned = db.owned_movie_tmdb_ids(srv)
|
||||
ignored = db.ignored_keys()
|
||||
rails = []
|
||||
# Complete your collections — top franchises you've started, missing entries.
|
||||
for coll in db.owned_movie_collections(srv, limit=8):
|
||||
missing = collection_gaps(owned, eng.collection(coll["collection_id"]))
|
||||
if missing:
|
||||
name = (coll.get("name") or "Collection").strip()
|
||||
rails.append({"title": "Complete the " + name, "kind": "collection",
|
||||
"items": missing[:30]})
|
||||
# More from the people you own the most (directors / creators).
|
||||
for person in db.top_owned_people(min_titles=2, limit=6, server_source=srv):
|
||||
p = eng.person_detail(person["tmdb_id"])
|
||||
if not p:
|
||||
continue
|
||||
missing = filmography_gaps(owned, p.get("credits") or [],
|
||||
kinds=("movie",), min_vote_count=50, limit=30)
|
||||
if ignored:
|
||||
missing = [m for m in missing
|
||||
if f"{m.get('kind')}:{m.get('tmdb_id')}" not in ignored]
|
||||
if len(missing) >= 3:
|
||||
rails.append({"title": "More from " + person["name"], "kind": "person",
|
||||
"items": missing})
|
||||
return jsonify({"rails": rails})
|
||||
except Exception:
|
||||
logger.exception("discover gaps failed")
|
||||
return jsonify({"rails": []})
|
||||
|
||||
@bp.route("/discover/trailer", methods=["GET"])
|
||||
def video_discover_trailer():
|
||||
"""Best YouTube trailer {key,name} for a tmdb title (hero 'Trailer' button)."""
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
kind = request.args.get("kind", "movie")
|
||||
try:
|
||||
tmdb_id = int(request.args.get("tmdb_id"))
|
||||
except (TypeError, ValueError):
|
||||
return jsonify({"trailer": None})
|
||||
try:
|
||||
tr = get_video_enrichment_engine().trailer(kind, tmdb_id)
|
||||
except Exception:
|
||||
logger.exception("discover trailer failed")
|
||||
tr = None
|
||||
return jsonify({"trailer": tr or None})
|
||||
|
||||
@bp.route("/discover/ignore", methods=["GET", "POST"])
|
||||
def video_discover_ignore():
|
||||
"""The Discover 'Not interested' list. GET -> {items}. POST
|
||||
{action:'add'|'remove', kind, tmdb_id, title?, year?, poster?}."""
|
||||
from . import get_video_db
|
||||
db = get_video_db()
|
||||
try:
|
||||
if request.method == "GET":
|
||||
return jsonify({"items": db.list_ignored()})
|
||||
body = request.get_json(silent=True) or {}
|
||||
kind, tmdb_id = body.get("kind"), body.get("tmdb_id")
|
||||
if body.get("action") == "remove":
|
||||
db.remove_ignored(kind, tmdb_id)
|
||||
return jsonify({"success": True})
|
||||
ok = db.add_ignored(kind, tmdb_id, body.get("title"), body.get("year"), body.get("poster"))
|
||||
return jsonify({"success": ok})
|
||||
except Exception:
|
||||
logger.exception("discover ignore failed")
|
||||
return jsonify({"success": False, "items": []})
|
||||
|
||||
@bp.route("/discover/languages", methods=["GET", "POST"])
|
||||
def video_discover_languages():
|
||||
"""Get/set the preferred original-languages for general rails (ISO-639-1 codes).
|
||||
POST {languages: ['en','ko']} (or 'en,ko'); GET returns the current list."""
|
||||
from . import get_video_db
|
||||
db = get_video_db()
|
||||
try:
|
||||
if request.method == "POST":
|
||||
body = request.get_json(silent=True) or {}
|
||||
langs = body.get("languages")
|
||||
if isinstance(langs, list):
|
||||
val = ",".join(str(c).strip().lower() for c in langs if str(c).strip())
|
||||
else:
|
||||
val = ",".join(c.strip().lower() for c in str(langs or "").split(",") if c.strip())
|
||||
db.set_setting("discover_languages", val or "en")
|
||||
return jsonify({"success": True,
|
||||
"languages": [c for c in (val or "en").split(",") if c]})
|
||||
raw = db.get_setting("discover_languages", "en") or "en"
|
||||
return jsonify({"languages": [c.strip() for c in raw.split(",") if c.strip()]})
|
||||
except Exception:
|
||||
logger.exception("discover languages get/set failed")
|
||||
return jsonify({"languages": ["en"]})
|
||||
|
||||
@bp.route("/discover/providers-pref", methods=["GET", "POST"])
|
||||
def video_discover_providers_pref():
|
||||
"""The user's subscribed streaming services (TMDB provider ids) — drives the
|
||||
'On your streaming services' rail. POST {providers:[8,9]} (or '8,9'); GET returns them."""
|
||||
from . import get_video_db
|
||||
db = get_video_db()
|
||||
try:
|
||||
if request.method == "POST":
|
||||
body = request.get_json(silent=True) or {}
|
||||
p = body.get("providers")
|
||||
if isinstance(p, list):
|
||||
val = ",".join(str(c).strip() for c in p if str(c).strip())
|
||||
else:
|
||||
val = ",".join(c.strip() for c in str(p or "").split(",") if c.strip())
|
||||
db.set_setting("discover_providers", val)
|
||||
return jsonify({"success": True, "providers": [c for c in val.split(",") if c]})
|
||||
raw = db.get_setting("discover_providers", "") or ""
|
||||
return jsonify({"providers": [c.strip() for c in raw.split(",") if c.strip()]})
|
||||
except Exception:
|
||||
logger.exception("discover providers-pref failed")
|
||||
return jsonify({"providers": []})
|
||||
|
||||
@bp.route("/discover/genres", methods=["GET"])
|
||||
def video_discover_genres():
|
||||
"""Genre id→name maps for both kinds (powers the genre rails + filter)."""
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
eng = get_video_enrichment_engine()
|
||||
try:
|
||||
return jsonify({"movie": eng.genre_list("movie"), "show": eng.genre_list("show")})
|
||||
except Exception:
|
||||
logger.exception("discover genres failed")
|
||||
return jsonify({"movie": [], "show": []})
|
||||
|
||||
@bp.route("/discover/list", methods=["GET"])
|
||||
def video_discover_list():
|
||||
"""One shelf (rail) or one page of a filtered browse — see module docstring.
|
||||
|
||||
``pages`` (1–3, default 1) fetches that many consecutive TMDB pages and
|
||||
concatenates them (deduped) in one response — so a rail can show ~40 items
|
||||
and still look full after 'Hide owned' drops the ones you have."""
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
eng = get_video_enrichment_engine()
|
||||
try:
|
||||
page = max(1, int(request.args.get("page", 1) or 1))
|
||||
except (TypeError, ValueError):
|
||||
page = 1
|
||||
try:
|
||||
pages = min(3, max(1, int(request.args.get("pages", 1) or 1)))
|
||||
except (TypeError, ValueError):
|
||||
pages = 1
|
||||
key = (request.args.get("key") or "").strip()
|
||||
kind = request.args.get("kind", "movie")
|
||||
genre = request.args.get("genre") or None
|
||||
year = request.args.get("year") or None
|
||||
decade = request.args.get("decade") or None
|
||||
providers = request.args.get("providers") or None
|
||||
if providers and "," in providers:
|
||||
providers = providers.replace(",", "|") # TMDB with_watch_providers OR-join
|
||||
sort = request.args.get("sort") or "popularity.desc"
|
||||
# Netflix-class discover extensions (all optional). Comma -> pipe = TMDB OR-join.
|
||||
keywords = (request.args.get("keywords") or "").replace(",", "|") or None
|
||||
companies = (request.args.get("companies") or "").replace(",", "|") or None
|
||||
networks = (request.args.get("networks") or "").replace(",", "|") or None
|
||||
cast = request.args.get("cast") or None
|
||||
crew = request.args.get("crew") or None
|
||||
min_runtime = request.args.get("min_runtime") or None
|
||||
max_runtime = request.args.get("max_runtime") or None
|
||||
certification = request.args.get("certification") or None
|
||||
release_window = request.args.get("release_window") or None
|
||||
try:
|
||||
vote_count_min = int(request.args["vote_count_min"]) if request.args.get("vote_count_min") else None
|
||||
except (TypeError, ValueError):
|
||||
vote_count_min = None
|
||||
lang = (request.args.get("lang") or "").strip().lower() or None # explicit (foreign rail / browse)
|
||||
# The Browse-all grid sends `lang=any` to opt OUT of the rail language preference
|
||||
# entirely (ad-hoc search shows every language); a real code (en/ko/…) filters to it.
|
||||
any_lang = lang in ("any", "all")
|
||||
if any_lang:
|
||||
lang = None
|
||||
hide_owned = (request.args.get("hide_owned") or "") in ("1", "true", "yes")
|
||||
# Preferred original-languages (multi) for GENERAL/curated rails — so the feeds
|
||||
# aren't flooded with foreign titles (e.g. Bollywood in Popular/Trending). A rail
|
||||
# with an explicit `lang` (a dedicated foreign rail) or `lang=any` (browse) bypasses
|
||||
# this. Default 'en'.
|
||||
prefer_langs = None
|
||||
if not lang and not any_lang:
|
||||
try:
|
||||
from . import get_video_db
|
||||
raw = get_video_db().get_setting("discover_languages", "en") or "en"
|
||||
prefer_langs = {c.strip().lower() for c in raw.split(",") if c.strip()} or None
|
||||
except Exception:
|
||||
prefer_langs = {"en"}
|
||||
|
||||
def fetch(p):
|
||||
if key == "trending":
|
||||
return eng.trending()
|
||||
if key == "trending_today":
|
||||
return eng.trending(window="day") # the ranked 'Top 10 today' chart (mixed)
|
||||
if key == "trending_movies_today":
|
||||
return eng.trending(window="day", kind="movie") # split 'Top 10 Movies Today'
|
||||
if key == "trending_tv_today":
|
||||
return eng.trending(window="day", kind="show") # split 'Top 10 TV Shows Today'
|
||||
if key:
|
||||
return eng.discover_curated(key, page=p)
|
||||
return eng.discover_filter(
|
||||
kind, genre=genre, year=year, decade=decade, providers=providers,
|
||||
sort_by=sort, page=p, language=lang, keywords=keywords, companies=companies,
|
||||
networks=networks, cast=cast, crew=crew, min_runtime=min_runtime,
|
||||
max_runtime=max_runtime, certification=certification,
|
||||
vote_count_min=vote_count_min, release_window=release_window)
|
||||
|
||||
try:
|
||||
items, seen = [], set()
|
||||
|
||||
def consume(batch):
|
||||
"""Filter a page into `items` (dedup + hide-owned + language). Returns False
|
||||
once TMDB hands back an empty page (it ran out — stop paging)."""
|
||||
if not batch:
|
||||
return False
|
||||
for it in batch:
|
||||
dk = (it.get("kind"), it.get("tmdb_id"))
|
||||
if dk in seen:
|
||||
continue
|
||||
seen.add(dk)
|
||||
if hide_owned and it.get("library_id") is not None:
|
||||
continue
|
||||
if prefer_langs:
|
||||
ol = (it.get("original_language") or "").lower()
|
||||
if ol and ol not in prefer_langs:
|
||||
continue # known foreign language not in your preference
|
||||
items.append(it)
|
||||
return True
|
||||
|
||||
# When filtering (hide-owned or language), page DEEPER and drop items server-side
|
||||
# so a heavy library's rail still fills to ~target instead of showing 4 cards. We
|
||||
# fetch the extra pages in concurrent WAVES (TTLCache + the TMDB client are
|
||||
# thread-safe) so digging 20 pages deep costs ~4 page-latencies, not 20.
|
||||
need_fill = hide_owned or bool(prefer_langs)
|
||||
target = 20 if need_fill else 0
|
||||
if key in ("trending", "trending_today", "trending_movies_today", "trending_tv_today"):
|
||||
consume(fetch(page)) # a single fixed chart; never paged
|
||||
elif not need_fill:
|
||||
for offset in range(pages): # no filtering: respect requested page count
|
||||
if not consume(fetch(page + offset)):
|
||||
break
|
||||
else:
|
||||
MAX_PAGES, WAVE = 20, 5
|
||||
offset, ran_out = 0, False
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=WAVE) as ex:
|
||||
while offset < MAX_PAGES and len(items) < target and not ran_out:
|
||||
batches = list(ex.map(fetch, [page + offset + i for i in range(WAVE)]))
|
||||
for batch in batches:
|
||||
if not consume(batch):
|
||||
ran_out = True # an empty page = TMDB has no more
|
||||
offset += WAVE
|
||||
return jsonify({"items": items, "page": page})
|
||||
except Exception:
|
||||
logger.exception("discover list failed (key=%s)", key)
|
||||
return jsonify({"items": [], "page": page})
|
||||
514
api/video/downloads.py
Normal file
514
api/video/downloads.py
Normal file
|
|
@ -0,0 +1,514 @@
|
|||
"""Video-side download SETTINGS (isolated).
|
||||
|
||||
Persists the video download configuration in video.db's ``video_settings`` KV
|
||||
table — fully separate from the music ``soulseek.*`` paths so the two libraries
|
||||
never share a folder or collide. The actual download fulfillment engine (wishlist
|
||||
→ search → grab) is a later roadmap phase; these endpoints just store/serve the
|
||||
config the Settings → Downloads tab edits.
|
||||
|
||||
Folders:
|
||||
- INPUT (download) folder is SHARED with the music side — it's the same
|
||||
``config_manager`` key the music Download Settings use (``soulseek.download_path``),
|
||||
so changing it on either side changes both (one physical download dir, simpler
|
||||
Docker mounts). We only READ/WRITE that shared key; no music code is touched.
|
||||
- OUTPUT (library) folders are video-specific and live in video.db, one per type:
|
||||
``movies_path`` / ``tv_path`` / ``youtube_path``.
|
||||
The engine routes a finished download to the library path matching its type. (Legacy
|
||||
single video ``transfer_path`` is migrated into ``movies_path`` on first read.)
|
||||
|
||||
Connection settings that are genuinely SHARED with music (the slskd instance, the
|
||||
torrent/usenet clients, Prowlarr indexers) are NOT stored here — those live in the
|
||||
music config_manager and are surfaced on the shared Indexers tab + shared slskd
|
||||
block (a deliberate shared boundary, since they're one physical resource).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.downloads")
|
||||
|
||||
# Video-specific OUTPUT library folders (video.db). The INPUT folder is the shared
|
||||
# music key below — not in this list.
|
||||
_PATH_KEYS = ("movies_path", "tv_path", "youtube_path")
|
||||
# The shared input/download dir — the SAME config key the music side reads/writes.
|
||||
_SHARED_DOWNLOAD_KEY = "soulseek.download_path"
|
||||
|
||||
# slskd CONNECTION settings genuinely SHARED with music — one slskd instance serves
|
||||
# both sides, so these live in the app-wide config_manager (soulseek.*), NOT video.db.
|
||||
# Deliberately excludes the music download/transfer PATHS and source mode/quality —
|
||||
# those are video-specific (stored in video.db). Maps the video field name -> the
|
||||
# shared config key + default. (config_manager is shared app config, not music code.)
|
||||
_SLSKD_KEYS = {
|
||||
"slskd_url": ("soulseek.slskd_url", "http://localhost:5030"),
|
||||
"api_key": ("soulseek.api_key", ""),
|
||||
"search_timeout": ("soulseek.search_timeout", 60),
|
||||
"search_timeout_buffer": ("soulseek.search_timeout_buffer", 15),
|
||||
"search_min_delay_seconds": ("soulseek.search_min_delay_seconds", 0),
|
||||
"min_peer_upload_speed": ("soulseek.min_peer_upload_speed", 0),
|
||||
"max_peer_queue": ("soulseek.max_peer_queue", 0),
|
||||
"download_timeout": ("soulseek.download_timeout", 600), # seconds (UI shows minutes)
|
||||
"auto_clear_searches": ("soulseek.auto_clear_searches", True),
|
||||
}
|
||||
|
||||
|
||||
def _evaluate_hits(raw, profile, scope, want_season, want_episode) -> list:
|
||||
"""Parse → evaluate → rank a list of raw indexer hits against the quality profile.
|
||||
Shared by the mock search and the live slskd start/poll endpoints."""
|
||||
from core.video.quality_eval import evaluate_release
|
||||
from core.video.release_parse import parse_release
|
||||
results = []
|
||||
for hit in raw:
|
||||
parsed = parse_release(hit.get("title"))
|
||||
size_gb = round((hit.get("size_bytes") or 0) / (1024 ** 3), 1)
|
||||
verdict = evaluate_release(parsed, profile, scope=scope, want_season=want_season,
|
||||
want_episode=want_episode, size_gb=size_gb)
|
||||
# Availability = how downloadable the source is (slskd: free slot/queue/speed score
|
||||
# from group_video_files; torrents/mock: seeders/peers). Ranks within a quality tier
|
||||
# so we grab a free-slot/empty-queue release over one stuck behind a huge queue.
|
||||
avail = hit.get("availability")
|
||||
if avail is None:
|
||||
avail = hit.get("seeders") if hit.get("seeders") is not None else (hit.get("peers") or 0)
|
||||
results.append({
|
||||
"title": hit.get("title"), "size_gb": size_gb, "size_bytes": hit.get("size_bytes") or 0,
|
||||
"seeders": hit.get("seeders"), "peers": hit.get("peers"),
|
||||
"username": hit.get("username"), "slots": hit.get("slots"),
|
||||
"queue": hit.get("queue"), "speed": hit.get("speed"),
|
||||
"filename": hit.get("filename"), "_avail": avail,
|
||||
"quality_label": verdict["quality_label"], "accepted": verdict["accepted"],
|
||||
"rejected": verdict["rejected"], "score": verdict["score"],
|
||||
"resolution": parsed.get("resolution"), "source": parsed.get("source"),
|
||||
"codec": parsed.get("codec"), "hdr": parsed.get("hdr"),
|
||||
"audio": parsed.get("audio"), "group": parsed.get("group"),
|
||||
"repack": parsed.get("repack") or parsed.get("proper"),
|
||||
})
|
||||
# accepted first, then quality-profile score, then availability, then bigger file.
|
||||
results.sort(key=lambda r: (r["accepted"], r["score"], r["_avail"], r["size_bytes"]), reverse=True)
|
||||
for r in results:
|
||||
r.pop("_avail", None)
|
||||
return results[:40]
|
||||
|
||||
|
||||
def _search_ints(body):
|
||||
def _int(v):
|
||||
try:
|
||||
return int(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return _int(body.get("season")), _int(body.get("episode")), _int(body.get("season_end"))
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/downloads/config", methods=["GET"])
|
||||
def video_downloads_config():
|
||||
from . import get_video_db
|
||||
from core.video.download_config import load as load_source
|
||||
from config.settings import config_manager
|
||||
db = get_video_db()
|
||||
out = {k: db.get_setting(k) or "" for k in _PATH_KEYS}
|
||||
if not out["movies_path"]: # migrate the legacy single transfer folder → Movies
|
||||
out["movies_path"] = db.get_setting("transfer_path") or ""
|
||||
out["download_path"] = config_manager.get(_SHARED_DOWNLOAD_KEY, "") or "" # shared w/ music
|
||||
out.update(load_source(db)) # download_mode + hybrid_order
|
||||
return jsonify(out)
|
||||
|
||||
@bp.route("/downloads/config", methods=["POST"])
|
||||
def video_downloads_config_save():
|
||||
from . import get_video_db
|
||||
from core.video.download_config import save as save_source
|
||||
db = get_video_db()
|
||||
body = request.get_json(silent=True) or {}
|
||||
for key in _PATH_KEYS:
|
||||
if key in body:
|
||||
db.set_setting(key, (str(body.get(key) or "")).strip())
|
||||
if "download_path" in body: # SHARED with music — write the same config key
|
||||
from config.settings import config_manager
|
||||
config_manager.set(_SHARED_DOWNLOAD_KEY, (str(body.get("download_path") or "")).strip())
|
||||
save_source(db, body) # download_mode + hybrid_order (validated)
|
||||
return jsonify({"status": "saved"})
|
||||
|
||||
@bp.route("/downloads/history", methods=["GET"])
|
||||
def video_downloads_history():
|
||||
"""Paged permanent history of grabs (movies + episodes). ?kind=movie|show,
|
||||
?search=, ?outcome=, ?page=, ?limit=. Always returns counts for the tabs."""
|
||||
from . import get_video_db
|
||||
try:
|
||||
db = get_video_db()
|
||||
kind = request.args.get("kind")
|
||||
res = db.query_download_history(
|
||||
kind=kind if kind in ("movie", "show") else None,
|
||||
search=request.args.get("search", ""),
|
||||
outcome=request.args.get("outcome") or None,
|
||||
page=request.args.get("page", 1), limit=request.args.get("limit", 40))
|
||||
return jsonify({"success": True, "counts": db.download_history_counts(), **res})
|
||||
except Exception:
|
||||
logger.exception("Failed to list video download history")
|
||||
return jsonify({"success": False, "error": "Failed to load history"}), 500
|
||||
|
||||
@bp.route("/downloads/history/<int:history_id>", methods=["GET"])
|
||||
def video_downloads_history_detail(history_id):
|
||||
from . import get_video_db
|
||||
d = get_video_db().download_history_detail(history_id)
|
||||
if not d:
|
||||
return jsonify({"success": False, "error": "not found"}), 404
|
||||
return jsonify({"success": True, "item": d})
|
||||
|
||||
@bp.route("/downloads/history/<int:history_id>", methods=["DELETE"])
|
||||
def video_downloads_history_delete(history_id):
|
||||
"""Forget one grab — the 'Re-download' action. Removing its history row lets the
|
||||
scans re-add + re-grab it (useful after you've deleted the file)."""
|
||||
from . import get_video_db
|
||||
return jsonify({"success": get_video_db().delete_download_history(history_id)})
|
||||
|
||||
@bp.route("/downloads/history/clear", methods=["POST"])
|
||||
def video_downloads_history_clear():
|
||||
"""Clear the permanent history (all, or one kind via {kind})."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
n = get_video_db().clear_download_history(kind=body.get("kind"))
|
||||
return jsonify({"success": True, "removed": n})
|
||||
|
||||
@bp.route("/downloads/meta/<kind>/<int:tmdb_id>", methods=["GET"])
|
||||
def video_download_meta(kind, tmdb_id):
|
||||
"""Lazy TMDB detail for a download's expand drawer (logo, cast w/ photos, trailer,
|
||||
where-to-watch, rating/runtime/genres…), keyed by the grabbed title's TMDB id.
|
||||
Best-effort — the drawer still shows the download facts without it."""
|
||||
if kind not in ("movie", "show"):
|
||||
return jsonify({}), 400
|
||||
try:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
d = get_video_enrichment_engine().tmdb_full_detail(kind, tmdb_id) or {}
|
||||
if not d:
|
||||
return jsonify({})
|
||||
extras = d.get("_extras") or {}
|
||||
tr = extras.get("trailer") or {}
|
||||
director = next((c.get("name") for c in (d.get("crew") or [])
|
||||
if (c.get("job") or "").lower() in ("director", "creator")), None)
|
||||
# episode-specific detail (still + that episode's own title/overview/air date) when a
|
||||
# specific episode is downloading — more relevant than the show synopsis.
|
||||
episode = None
|
||||
sn, en = request.args.get("season"), request.args.get("episode")
|
||||
if kind == "show" and sn and en:
|
||||
try:
|
||||
season = get_video_enrichment_engine().tmdb_season(tmdb_id, int(sn)) or {}
|
||||
ep = next((e for e in (season.get("episodes") or [])
|
||||
if str(e.get("episode_number")) == str(int(en))), None)
|
||||
if ep:
|
||||
episode = {"season": int(sn), "episode": int(en), "title": ep.get("title"),
|
||||
"overview": ep.get("overview"), "air_date": ep.get("air_date"),
|
||||
"still_url": ep.get("still_url")}
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return jsonify({
|
||||
"title": d.get("title"), "overview": d.get("overview"), "tagline": d.get("tagline"),
|
||||
"backdrop_url": d.get("backdrop_url"), "logo": d.get("logo"),
|
||||
"genres": d.get("genres") or [], "rating": d.get("rating"),
|
||||
"runtime_minutes": d.get("runtime_minutes"), "year": d.get("year"),
|
||||
"network": d.get("network"), "studio": d.get("studio"),
|
||||
"status": d.get("status"), "director": director, "episode": episode,
|
||||
"cast": [{"name": c.get("name"), "character": c.get("character"), "photo": c.get("photo")}
|
||||
for c in (d.get("cast") or [])[:10]],
|
||||
"trailer_url": ("https://www.youtube.com/watch?v=" + tr["key"]) if tr.get("key") else None,
|
||||
"providers": (extras.get("providers") or [])[:6],
|
||||
"providers_link": extras.get("providers_link"),
|
||||
})
|
||||
except Exception:
|
||||
logger.exception("download meta failed for %s %s", kind, tmdb_id)
|
||||
return jsonify({})
|
||||
|
||||
@bp.route("/downloads/yt-meta/<video_id>", methods=["GET"])
|
||||
def video_download_yt_meta(video_id):
|
||||
"""Cached extra detail for a YouTube download's drawer (duration / views / thumbnail)."""
|
||||
from . import get_video_db
|
||||
try:
|
||||
return jsonify(get_video_db().youtube_video_detail(video_id) or {})
|
||||
except Exception:
|
||||
logger.exception("yt meta failed for %s", video_id)
|
||||
return jsonify({})
|
||||
|
||||
@bp.route("/downloads/quality", methods=["GET"])
|
||||
def video_quality_profile():
|
||||
from . import get_video_db
|
||||
from core.video.quality_profile import load
|
||||
return jsonify(load(get_video_db()))
|
||||
|
||||
@bp.route("/downloads/quality", methods=["POST"])
|
||||
def video_quality_profile_save():
|
||||
from . import get_video_db
|
||||
from core.video.quality_profile import save
|
||||
body = request.get_json(silent=True) or {}
|
||||
return jsonify(save(get_video_db(), body))
|
||||
|
||||
@bp.route("/organization", methods=["GET"])
|
||||
def video_organization():
|
||||
"""The library-organisation settings: naming templates + post-process toggles."""
|
||||
from . import get_video_db
|
||||
from core.video.organization import load
|
||||
return jsonify(load(get_video_db()))
|
||||
|
||||
@bp.route("/organization", methods=["POST"])
|
||||
def video_organization_save():
|
||||
from . import get_video_db
|
||||
from core.video.organization import save
|
||||
body = request.get_json(silent=True) or {}
|
||||
return jsonify(save(get_video_db(), body))
|
||||
|
||||
@bp.route("/downloads/evaluate", methods=["POST"])
|
||||
def video_quality_evaluate():
|
||||
"""Judge a video file the user already owns against their quality profile —
|
||||
powers the Download modal's 'In your library · … (below your target)' line.
|
||||
Body: {"file": {resolution, video_codec, …}}."""
|
||||
from . import get_video_db
|
||||
from core.video.quality_eval import evaluate_owned
|
||||
from core.video.quality_profile import load
|
||||
body = request.get_json(silent=True) or {}
|
||||
profile = load(get_video_db())
|
||||
return jsonify(evaluate_owned(body.get("file"), profile))
|
||||
|
||||
@bp.route("/downloads/search", methods=["POST"])
|
||||
def video_downloads_search():
|
||||
"""Search a scope (movie / episode / season / series) and return candidates
|
||||
ranked + filtered against the stored quality profile. The indexer is mocked
|
||||
for now (core.video.mock_search) — the parse→evaluate→rank pipeline is real,
|
||||
so swapping in slskd/Prowlarr later needs no change here.
|
||||
Body: {scope, title, year?, season?, episode?, season_end?}."""
|
||||
from . import get_video_db
|
||||
from core.video.mock_search import mock_search
|
||||
from core.video.quality_profile import load as load_profile
|
||||
|
||||
body = request.get_json(silent=True) or {}
|
||||
scope = str(body.get("scope") or "movie").lower()
|
||||
title = body.get("title") or ""
|
||||
source = str(body.get("source") or "").lower()
|
||||
want_season, want_episode, season_end = _search_ints(body)
|
||||
profile = load_profile(get_video_db())
|
||||
live = False
|
||||
if source == "soulseek":
|
||||
from core.video.slskd_search import build_query, slskd_search
|
||||
sres = slskd_search(build_query(scope, title, year=body.get("year"),
|
||||
season=want_season, episode=want_episode))
|
||||
if not sres.get("configured"):
|
||||
return jsonify({"scope": scope, "results": [], "error": "slskd isn't configured — set its URL on Settings → Downloads."})
|
||||
if sres.get("error"):
|
||||
return jsonify({"scope": scope, "results": [], "error": "slskd: " + str(sres["error"])})
|
||||
raw, live = sres["hits"], True
|
||||
else:
|
||||
raw = mock_search(scope, title, year=body.get("year"), season=want_season,
|
||||
episode=want_episode, season_end=season_end, source=source)
|
||||
return jsonify({"scope": scope, "live": live,
|
||||
"results": _evaluate_hits(raw, profile, scope, want_season, want_episode)})
|
||||
|
||||
@bp.route("/downloads/search/start", methods=["POST"])
|
||||
def video_downloads_search_start():
|
||||
"""Begin a search. For mock sources the results come back immediately; for
|
||||
Soulseek it returns a slskd search id to poll (results trickle in over ~30s,
|
||||
like the music side) — fixes 'no results' from waiting too briefly."""
|
||||
from . import get_video_db
|
||||
from core.video.mock_search import mock_search
|
||||
from core.video.quality_profile import load as load_profile
|
||||
body = request.get_json(silent=True) or {}
|
||||
scope = str(body.get("scope") or "movie").lower()
|
||||
title = body.get("title") or ""
|
||||
source = str(body.get("source") or "").lower()
|
||||
want_season, want_episode, season_end = _search_ints(body)
|
||||
|
||||
if source == "soulseek":
|
||||
from core.video.slskd_search import build_query, search_timeout_ms, start_search
|
||||
res = start_search(build_query(scope, title, year=body.get("year"),
|
||||
season=want_season, episode=want_episode))
|
||||
if not res.get("configured"):
|
||||
return jsonify({"error": "slskd isn't configured — set its URL on Settings → Downloads."})
|
||||
if res.get("error"):
|
||||
return jsonify({"error": "slskd: " + str(res["error"])})
|
||||
# how long the client should keep polling (slskd keeps searching this long).
|
||||
return jsonify({"id": res["id"], "live": True, "complete": False,
|
||||
"poll_ms": search_timeout_ms() + 8000})
|
||||
# mock sources resolve in one shot
|
||||
profile = load_profile(get_video_db())
|
||||
raw = mock_search(scope, title, year=body.get("year"), season=want_season,
|
||||
episode=want_episode, season_end=season_end, source=source)
|
||||
return jsonify({"id": None, "live": False, "complete": True,
|
||||
"results": _evaluate_hits(raw, profile, scope, want_season, want_episode)})
|
||||
|
||||
@bp.route("/downloads/search/poll", methods=["GET"])
|
||||
def video_downloads_search_poll():
|
||||
"""Current ranked results for an in-flight slskd search. Query: id, scope,
|
||||
title, season?, episode?. The client polls until it stops growing or times out."""
|
||||
from . import get_video_db
|
||||
from core.video.quality_profile import load as load_profile
|
||||
from core.video.slskd_search import poll_search
|
||||
sid = request.args.get("id")
|
||||
scope = str(request.args.get("scope") or "movie").lower()
|
||||
want_season, want_episode, _ = _search_ints(request.args)
|
||||
if not sid:
|
||||
return jsonify({"results": [], "live": True, "total_files": 0})
|
||||
profile = load_profile(get_video_db())
|
||||
polled = poll_search(sid)
|
||||
return jsonify({"live": True, "total_files": polled["total_files"],
|
||||
"results": _evaluate_hits(polled["hits"], profile, scope, want_season, want_episode)})
|
||||
|
||||
@bp.route("/downloads/grab", methods=["POST"])
|
||||
def video_downloads_grab():
|
||||
"""Start a real download of a chosen release and track it. v1: Soulseek only.
|
||||
Body: {kind, title, release_title, source, username, filename, size_bytes,
|
||||
quality_label}."""
|
||||
from . import get_video_db
|
||||
from config.settings import config_manager
|
||||
from core.video.download_monitor import ensure_started
|
||||
from core.video.download_pipeline import target_dir_for
|
||||
from core.video.slskd_download import start_download
|
||||
|
||||
body = request.get_json(silent=True) or {}
|
||||
source = str(body.get("source") or "soulseek").lower()
|
||||
if source != "soulseek":
|
||||
return jsonify({"ok": False, "error": "Only Soulseek grabs are wired up so far."}), 400
|
||||
username, filename = body.get("username"), body.get("filename")
|
||||
if not username or not filename:
|
||||
return jsonify({"ok": False, "error": "Missing the release's source info."}), 400
|
||||
|
||||
db = get_video_db()
|
||||
paths = {k: db.get_setting(k) or "" for k in ("movies_path", "tv_path", "youtube_path")}
|
||||
if not paths["movies_path"]:
|
||||
paths["movies_path"] = db.get_setting("transfer_path") or ""
|
||||
target = target_dir_for(body.get("kind"), paths)
|
||||
if not target:
|
||||
return jsonify({"ok": False, "error": "Set the library folder for this type on Settings → Downloads."}), 400
|
||||
|
||||
started = start_download(username, filename, body.get("size_bytes") or 0)
|
||||
if not started.get("ok"):
|
||||
return jsonify({"ok": False, "error": started.get("error") or "slskd refused the download."}), 502
|
||||
|
||||
import json as _json
|
||||
from core.video.slskd_search import build_query
|
||||
# The OTHER accepted results become the retry pool; the search context drives
|
||||
# the alternate-query requery when the pool runs dry.
|
||||
ctx = body.get("search_ctx") if isinstance(body.get("search_ctx"), dict) else {}
|
||||
candidates = [c for c in (body.get("candidates") or []) if isinstance(c, dict) and c.get("filename") != filename]
|
||||
first_query = build_query(ctx.get("scope") or body.get("kind") or "movie", ctx.get("title") or body.get("title"),
|
||||
year=ctx.get("year"), season=ctx.get("season"), episode=ctx.get("episode"))
|
||||
dl_id = db.add_video_download({
|
||||
"kind": str(body.get("kind") or "movie"), "title": body.get("title"),
|
||||
"release_title": body.get("release_title") or body.get("filename"),
|
||||
"source": "soulseek", "username": username, "filename": filename,
|
||||
"size_bytes": int(body.get("size_bytes") or 0), "quality_label": body.get("quality_label"),
|
||||
"target_dir": target, "status": "downloading",
|
||||
"media_id": (str(body.get("media_id")) if body.get("media_id") is not None else None),
|
||||
"media_source": body.get("media_source"), "year": body.get("year"),
|
||||
"poster_url": body.get("poster_url"),
|
||||
"candidates": _json.dumps(candidates), "search_ctx": _json.dumps(ctx),
|
||||
"tried_queries": _json.dumps([first_query] if first_query else []),
|
||||
"tried_files": _json.dumps([filename]), "attempts": 0,
|
||||
})
|
||||
ensure_started(get_video_db)
|
||||
return jsonify({"ok": True, "id": dl_id})
|
||||
|
||||
@bp.route("/downloads/active", methods=["GET"])
|
||||
def video_downloads_active():
|
||||
from . import get_video_db
|
||||
from core.video.download_monitor import ensure_started
|
||||
db = get_video_db()
|
||||
ensure_started(get_video_db) # also (re)start the monitor when the page is open
|
||||
return jsonify({"downloads": db.list_video_downloads()})
|
||||
|
||||
@bp.route("/downloads/status", methods=["GET"])
|
||||
def video_downloads_status():
|
||||
"""Lightweight live-tracking lookup — used by the Download modal's result
|
||||
card (by ``id``) and a movie/show detail page (by ``media_id`` +
|
||||
``media_source``, returning that title's most relevant download so the page
|
||||
can show live progress). Returns ``{"download": {...}|null}``."""
|
||||
from . import get_video_db
|
||||
from core.video.download_monitor import ensure_started
|
||||
db = get_video_db()
|
||||
ensure_started(get_video_db)
|
||||
dl_id = request.args.get("id")
|
||||
if dl_id:
|
||||
try:
|
||||
return jsonify({"download": db.get_video_download(int(dl_id))})
|
||||
except (TypeError, ValueError):
|
||||
return jsonify({"download": None})
|
||||
media_id = request.args.get("media_id")
|
||||
if media_id:
|
||||
media_source = request.args.get("media_source")
|
||||
match = [r for r in db.list_video_downloads()
|
||||
if str(r.get("media_id")) == str(media_id)
|
||||
and (not media_source or r.get("media_source") == media_source)]
|
||||
# list_video_downloads orders active-first then newest — so an active
|
||||
# download (or else the most recent) for this title is simply match[0].
|
||||
return jsonify({"download": match[0] if match else None})
|
||||
return jsonify({"download": None})
|
||||
|
||||
@bp.route("/downloads/cancel", methods=["POST"])
|
||||
def video_downloads_cancel():
|
||||
from . import get_video_db
|
||||
from core.video.slskd_download import cancel_download
|
||||
body = request.get_json(silent=True) or {}
|
||||
db = get_video_db()
|
||||
dl = db.get_video_download(body.get("id"))
|
||||
if not dl:
|
||||
return jsonify({"ok": False, "error": "Download not found."}), 404
|
||||
if dl["status"] in ("completed", "failed", "cancelled"):
|
||||
return jsonify({"ok": True, "already": True})
|
||||
cancel_download(dl.get("username"), dl.get("filename")) # best-effort; mark regardless
|
||||
import time
|
||||
db.update_video_download(dl["id"], status="cancelled", error="Cancelled",
|
||||
completed_at=time.strftime("%Y-%m-%d %H:%M:%S"))
|
||||
return jsonify({"ok": True})
|
||||
|
||||
@bp.route("/downloads/retry", methods=["POST"])
|
||||
def video_downloads_retry():
|
||||
"""Re-grab the SAME release (basic retry). Auto-retry + alternate-query retry
|
||||
come in a later phase."""
|
||||
from . import get_video_db
|
||||
from core.video.download_monitor import ensure_started
|
||||
from core.video.slskd_download import start_download
|
||||
body = request.get_json(silent=True) or {}
|
||||
db = get_video_db()
|
||||
dl = db.get_video_download(body.get("id"))
|
||||
if not dl:
|
||||
return jsonify({"ok": False, "error": "Download not found."}), 404
|
||||
if not dl.get("username") or not dl.get("filename"):
|
||||
return jsonify({"ok": False, "error": "Nothing to retry from."}), 400
|
||||
started = start_download(dl["username"], dl["filename"], dl.get("size_bytes") or 0)
|
||||
if not started.get("ok"):
|
||||
return jsonify({"ok": False, "error": started.get("error") or "slskd refused the download."}), 502
|
||||
db.update_video_download(dl["id"], status="downloading", progress=0, error=None,
|
||||
dest_path=None, completed_at=None)
|
||||
ensure_started(get_video_db)
|
||||
return jsonify({"ok": True})
|
||||
|
||||
@bp.route("/downloads/clear", methods=["POST"])
|
||||
def video_downloads_clear():
|
||||
from . import get_video_db
|
||||
return jsonify({"cleared": get_video_db().clear_finished_video_downloads()})
|
||||
|
||||
@bp.route("/downloads/youtube-quality", methods=["GET"])
|
||||
def video_youtube_quality():
|
||||
# Separate, smaller profile — YouTube is yt-dlp, not scene/p2p releases.
|
||||
from . import get_video_db
|
||||
from core.video.youtube_quality import load
|
||||
return jsonify(load(get_video_db()))
|
||||
|
||||
@bp.route("/downloads/youtube-quality", methods=["POST"])
|
||||
def video_youtube_quality_save():
|
||||
from . import get_video_db
|
||||
from core.video.youtube_quality import save
|
||||
body = request.get_json(silent=True) or {}
|
||||
return jsonify(save(get_video_db(), body))
|
||||
|
||||
@bp.route("/downloads/slskd", methods=["GET"])
|
||||
def video_slskd_config():
|
||||
# SHARED with music — same slskd instance. Reads the app-wide config_manager.
|
||||
from config.settings import config_manager
|
||||
return jsonify({k: config_manager.get(cfg, default)
|
||||
for k, (cfg, default) in _SLSKD_KEYS.items()})
|
||||
|
||||
@bp.route("/downloads/slskd", methods=["POST"])
|
||||
def video_slskd_config_save():
|
||||
from config.settings import config_manager
|
||||
body = request.get_json(silent=True) or {}
|
||||
for k, (cfg, _default) in _SLSKD_KEYS.items():
|
||||
if k in body:
|
||||
config_manager.set(cfg, body.get(k))
|
||||
return jsonify({"status": "saved", "shared": True})
|
||||
219
api/video/enrichment.py
Normal file
219
api/video/enrichment.py
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
"""Video enrichment API — mirrors the music enrichment endpoints so the shared
|
||||
Manage-Workers modal can drive video workers by pointing at /api/video/...
|
||||
|
||||
GET /api/video/enrichment/services
|
||||
GET /api/video/enrichment/<service>/status
|
||||
POST /api/video/enrichment/<service>/pause | /resume
|
||||
GET /api/video/enrichment/<service>/breakdown
|
||||
GET /api/video/enrichment/<service>/unmatched?kind=&status=&q=&limit=&offset=
|
||||
POST /api/video/enrichment/<service>/retry {kind, scope, item_id}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.enrichment")
|
||||
|
||||
|
||||
def _int(val, default):
|
||||
try:
|
||||
return int(val)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
def engine():
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
return get_video_enrichment_engine()
|
||||
|
||||
def _yt_enricher():
|
||||
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||
return get_youtube_date_enricher()
|
||||
|
||||
@bp.route("/enrichment/services", methods=["GET"])
|
||||
def video_enrichment_services():
|
||||
try:
|
||||
return jsonify({"services": engine().services()})
|
||||
except Exception:
|
||||
logger.exception("video enrichment services failed")
|
||||
return jsonify({"services": []})
|
||||
|
||||
@bp.route("/enrichment/config", methods=["GET"])
|
||||
def video_enrichment_config():
|
||||
from . import get_video_db
|
||||
db = get_video_db()
|
||||
return jsonify({
|
||||
"tmdb_api_key": db.get_setting("tmdb_api_key") or "",
|
||||
"tvdb_api_key": db.get_setting("tvdb_api_key") or "",
|
||||
"omdb_api_key": db.get_setting("omdb_api_key") or "",
|
||||
# Backfill-worker keys (free, optional) + no-key toggles.
|
||||
"fanart_api_key": db.get_setting("fanart_api_key") or "",
|
||||
"opensubtitles_api_key": db.get_setting("opensubtitles_api_key") or "",
|
||||
"trakt_api_key": db.get_setting("trakt_api_key") or "",
|
||||
"ryd_enabled": (db.get_setting("ryd_enabled") or "1") == "1",
|
||||
"sponsorblock_enabled": (db.get_setting("sponsorblock_enabled") or "1") == "1",
|
||||
"dearrow_enabled": (db.get_setting("dearrow_enabled") or "1") == "1",
|
||||
"tvmaze_enabled": (db.get_setting("tvmaze_enabled") or "1") == "1",
|
||||
"anilist_enabled": (db.get_setting("anilist_enabled") or "0") == "1",
|
||||
"wikidata_enabled": (db.get_setting("wikidata_enabled") or "1") == "1",
|
||||
"billboard_autoplay": (db.get_setting("billboard_autoplay") or "1") == "1",
|
||||
"watch_region": (db.get_setting("watch_region") or "US").upper(),
|
||||
})
|
||||
|
||||
@bp.route("/prefs", methods=["GET"])
|
||||
def video_prefs():
|
||||
# Lightweight UI prefs for the detail page (no API keys).
|
||||
from . import get_video_db
|
||||
db = get_video_db()
|
||||
return jsonify({
|
||||
"billboard_autoplay": (db.get_setting("billboard_autoplay") or "1") == "1",
|
||||
"watch_region": (db.get_setting("watch_region") or "US").upper(),
|
||||
})
|
||||
|
||||
@bp.route("/enrichment/config", methods=["POST"])
|
||||
def video_enrichment_config_save():
|
||||
from . import get_video_db
|
||||
db = get_video_db()
|
||||
body = request.get_json(silent=True) or {}
|
||||
keys_changed = False
|
||||
|
||||
def put_key(field):
|
||||
nonlocal keys_changed
|
||||
if field in body:
|
||||
val = body.get(field) or ""
|
||||
if val != (db.get_setting(field) or ""):
|
||||
keys_changed = True
|
||||
db.set_setting(field, val)
|
||||
|
||||
put_key("tmdb_api_key")
|
||||
put_key("tvdb_api_key")
|
||||
put_key("fanart_api_key")
|
||||
put_key("opensubtitles_api_key")
|
||||
put_key("trakt_api_key")
|
||||
# No-key worker on/off toggles (read live by the worker — no rebuild needed).
|
||||
for flag in ("ryd_enabled", "sponsorblock_enabled", "dearrow_enabled",
|
||||
"tvmaze_enabled", "anilist_enabled", "wikidata_enabled"):
|
||||
if flag in body:
|
||||
db.set_setting(flag, "1" if body.get(flag) else "0")
|
||||
if "billboard_autoplay" in body:
|
||||
db.set_setting("billboard_autoplay", "1" if body.get("billboard_autoplay") else "0")
|
||||
if "watch_region" in body:
|
||||
region = (body.get("watch_region") or "US").strip().upper()[:2] or "US"
|
||||
db.set_setting("watch_region", region)
|
||||
if "omdb_api_key" in body:
|
||||
new_key = body.get("omdb_api_key") or ""
|
||||
changed = new_key != (db.get_setting("omdb_api_key") or "")
|
||||
db.set_setting("omdb_api_key", new_key)
|
||||
keys_changed = keys_changed or changed
|
||||
# A new/changed OMDb key → re-try every title that still has no rating
|
||||
# (covers items wrongly marked 'synced' during a prior bad-key run).
|
||||
if new_key and changed:
|
||||
try:
|
||||
for kind in ("movie", "show"):
|
||||
db.enrichment_retry("omdb", kind, scope="failed")
|
||||
except Exception:
|
||||
logger.exception("video enrichment: omdb re-try reset failed")
|
||||
# Only rebuild the workers when an API KEY actually changed — a prefs-only
|
||||
# save (autoplay/region) must not churn the running enrichment engine
|
||||
# (that restart was re-logging the OMDb limit warning on every save).
|
||||
if keys_changed:
|
||||
try:
|
||||
from core.video.enrichment.engine import rebuild_video_enrichment_engine
|
||||
rebuild_video_enrichment_engine()
|
||||
except Exception:
|
||||
logger.exception("video enrichment: engine rebuild after key change failed")
|
||||
return jsonify({"status": "saved"})
|
||||
|
||||
@bp.route("/enrichment/<service>/status", methods=["GET"])
|
||||
def video_enrichment_status(service):
|
||||
if service == "youtube": # the standalone date enricher (not an engine worker)
|
||||
return jsonify(_yt_enricher().stats())
|
||||
w = engine().worker(service)
|
||||
if not w:
|
||||
return jsonify({"error": "unknown service"}), 404
|
||||
return jsonify(w.get_stats())
|
||||
|
||||
@bp.route("/enrichment/<service>/pause", methods=["POST"])
|
||||
def video_enrichment_pause(service):
|
||||
if service == "youtube":
|
||||
_yt_enricher().pause()
|
||||
return jsonify({"status": "paused"})
|
||||
w = engine().worker(service)
|
||||
if not w:
|
||||
return jsonify({"error": "unknown service"}), 404
|
||||
w.pause()
|
||||
return jsonify({"status": "paused"})
|
||||
|
||||
@bp.route("/enrichment/<service>/resume", methods=["POST"])
|
||||
def video_enrichment_resume(service):
|
||||
if service == "youtube":
|
||||
_yt_enricher().resume()
|
||||
return jsonify({"status": "running"})
|
||||
w = engine().worker(service)
|
||||
if not w:
|
||||
return jsonify({"error": "unknown service"}), 404
|
||||
w.resume()
|
||||
return jsonify({"status": "running"})
|
||||
|
||||
@bp.route("/enrichment/priority", methods=["GET", "POST"])
|
||||
def video_enrichment_priority():
|
||||
from . import get_video_db
|
||||
db = get_video_db()
|
||||
if request.method == "POST":
|
||||
body = request.get_json(silent=True) or {}
|
||||
kind = body.get("priority") or ""
|
||||
if kind not in ("", "movie", "show"):
|
||||
return jsonify({"error": "bad priority"}), 400
|
||||
db.set_setting("enrichment_priority", kind)
|
||||
return jsonify({"success": True, "priority": kind})
|
||||
return jsonify({"priority": db.get_setting("enrichment_priority") or ""})
|
||||
|
||||
@bp.route("/enrichment/<service>/test", methods=["POST"])
|
||||
def video_enrichment_test(service):
|
||||
w = engine().worker(service)
|
||||
if not w:
|
||||
return jsonify({"success": False, "error": "unknown service"}), 404
|
||||
try:
|
||||
ok, msg = w.client.test()
|
||||
return jsonify({"success": bool(ok), "message": msg, "error": None if ok else msg})
|
||||
except Exception:
|
||||
logger.exception("video enrichment test failed for %s", service)
|
||||
return jsonify({"success": False, "error": "Test failed"})
|
||||
|
||||
@bp.route("/enrichment/<service>/breakdown", methods=["GET"])
|
||||
def video_enrichment_breakdown(service):
|
||||
from . import get_video_db
|
||||
return jsonify({"service": service, "breakdown": get_video_db().enrichment_breakdown(service)})
|
||||
|
||||
@bp.route("/enrichment/<service>/unmatched", methods=["GET"])
|
||||
def video_enrichment_unmatched(service):
|
||||
from . import get_video_db
|
||||
kind = request.args.get("kind", "movie")
|
||||
res = get_video_db().enrichment_unmatched(
|
||||
service, kind,
|
||||
status=request.args.get("status", "not_found"),
|
||||
search=request.args.get("q") or None,
|
||||
limit=_int(request.args.get("limit"), 50),
|
||||
offset=_int(request.args.get("offset"), 0))
|
||||
res.update({"service": service, "kind": kind})
|
||||
return jsonify(res)
|
||||
|
||||
@bp.route("/enrichment/retry-all-failed", methods=["POST"])
|
||||
def video_enrichment_retry_all_failed():
|
||||
"""Global re-queue: reset every failed/not_found item across ALL workers and
|
||||
kinds (one-click recovery after an outage). Returns the total re-queued."""
|
||||
from . import get_video_db
|
||||
return jsonify({"success": True, "reset": get_video_db().retry_all_failed()})
|
||||
|
||||
@bp.route("/enrichment/<service>/retry", methods=["POST"])
|
||||
def video_enrichment_retry(service):
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
n = get_video_db().enrichment_retry(
|
||||
service, body.get("kind", "movie"),
|
||||
scope=body.get("scope", "failed"), item_id=body.get("item_id"))
|
||||
return jsonify({"success": True, "reset": n})
|
||||
182
api/video/libraries.py
Normal file
182
api/video/libraries.py
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
"""Video library mapping endpoints.
|
||||
|
||||
GET /api/video/libraries -> discover the active server's Movies/TV libraries
|
||||
+ the user's current selection.
|
||||
POST /api/video/libraries -> save {movies, tv} (library titles) for the active
|
||||
server. The scanner then reads only those.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.libraries")
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/libraries", methods=["GET"])
|
||||
def video_libraries():
|
||||
from . import get_video_db
|
||||
try:
|
||||
from core.video.sources import list_video_libraries, resolve_video_server
|
||||
libs = list_video_libraries() or {"server": None, "movies": [], "tv": []}
|
||||
server = libs.get("server") or resolve_video_server()
|
||||
libs["selected"] = (get_video_db().get_library_selection(server)
|
||||
if server else {"movies": None, "tv": None})
|
||||
return jsonify(libs)
|
||||
except Exception:
|
||||
logger.exception("Failed to list video libraries")
|
||||
return jsonify({"error": "Failed to list video libraries"}), 500
|
||||
|
||||
@bp.route("/server", methods=["GET"])
|
||||
def video_server_status():
|
||||
"""Which server the video side uses + which of Plex/Jellyfin are configured
|
||||
(so the UI can show a picker, or a 'connect a server' message)."""
|
||||
try:
|
||||
from core.video.sources import (resolve_video_server,
|
||||
video_plex_config, video_jellyfin_config)
|
||||
plex = bool(video_plex_config().get("base_url"))
|
||||
jelly = bool(video_jellyfin_config().get("base_url"))
|
||||
return jsonify({"server": resolve_video_server(), "plex": plex, "jellyfin": jelly})
|
||||
except Exception:
|
||||
logger.exception("video server status failed")
|
||||
return jsonify({"server": None, "plex": False, "jellyfin": False})
|
||||
|
||||
@bp.route("/server", methods=["POST"])
|
||||
def video_server_set():
|
||||
"""Set the explicit video-side server pick (only meaningful when both Plex
|
||||
and Jellyfin are configured)."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
choice = body.get("server")
|
||||
if choice not in ("plex", "jellyfin"):
|
||||
return jsonify({"error": "bad server"}), 400
|
||||
get_video_db().set_setting("video_server", choice)
|
||||
return jsonify({"status": "saved", "server": choice})
|
||||
|
||||
@bp.route("/server-config", methods=["GET"])
|
||||
def video_server_config_get():
|
||||
"""The video side's OWN server connection — its stored creds when set, else
|
||||
the values INHERITED (read-only) from music. 'inherited' flags tell the UI a
|
||||
field is a placeholder it can override; tokens/keys are returned masked."""
|
||||
try:
|
||||
from core.video.sources import video_plex_config, video_jellyfin_config
|
||||
p, j = video_plex_config(), video_jellyfin_config()
|
||||
|
||||
def mask(v):
|
||||
v = v or ""
|
||||
return ("•" * 12) if v else ""
|
||||
return jsonify({
|
||||
"plex": {"base_url": p.get("base_url") or "", "token": mask(p.get("token")),
|
||||
"has_token": bool(p.get("token")), "inherited": p.get("source") == "music"},
|
||||
"jellyfin": {"base_url": j.get("base_url") or "", "api_key": mask(j.get("api_key")),
|
||||
"has_key": bool(j.get("api_key")), "inherited": j.get("source") == "music"},
|
||||
})
|
||||
except Exception:
|
||||
logger.exception("video server-config get failed")
|
||||
return jsonify({"plex": {}, "jellyfin": {}})
|
||||
|
||||
@bp.route("/server-config", methods=["POST"])
|
||||
def video_server_config_set():
|
||||
"""Save the video side's OWN Plex/Jellyfin creds to video.db (NEVER the music
|
||||
config). An empty/blank field clears that override → video falls back to
|
||||
inheriting music's value. A masked token (all •) is left untouched."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
db = get_video_db()
|
||||
|
||||
def is_mask(v):
|
||||
return bool(v) and set(str(v)) == {"•"}
|
||||
|
||||
def put(key, val):
|
||||
if is_mask(val):
|
||||
return # unchanged masked secret — keep what's stored
|
||||
db.set_setting(key, (val or "").strip())
|
||||
|
||||
plex = body.get("plex") or {}
|
||||
jelly = body.get("jellyfin") or {}
|
||||
if "base_url" in plex:
|
||||
put("video_plex_url", plex.get("base_url"))
|
||||
if "token" in plex:
|
||||
put("video_plex_token", plex.get("token"))
|
||||
if "base_url" in jelly:
|
||||
put("video_jellyfin_url", jelly.get("base_url"))
|
||||
if "api_key" in jelly:
|
||||
put("video_jellyfin_key", jelly.get("api_key"))
|
||||
return jsonify({"status": "saved"})
|
||||
|
||||
@bp.route("/server-config/test", methods=["POST"])
|
||||
def video_server_config_test():
|
||||
"""Test the video side's effective connection for one server, using its OWN
|
||||
stored/inherited creds (so it verifies exactly what the video scan will use)."""
|
||||
body = request.get_json(silent=True) or {}
|
||||
which = body.get("server")
|
||||
if which not in ("plex", "jellyfin"):
|
||||
return jsonify({"success": False, "error": "bad server"}), 400
|
||||
try:
|
||||
if which == "plex":
|
||||
from core.video.sources import video_plex_config, PLEX_SCAN_TIMEOUT
|
||||
cfg = video_plex_config()
|
||||
if not cfg.get("base_url") or not cfg.get("token"):
|
||||
return jsonify({"success": False, "error": "Plex URL/token not set"})
|
||||
from plexapi.server import PlexServer
|
||||
srv = PlexServer(cfg["base_url"], cfg["token"], timeout=PLEX_SCAN_TIMEOUT)
|
||||
return jsonify({"success": True, "message": "Connected to " + (srv.friendlyName or "Plex")})
|
||||
from core.video.sources import video_jellyfin_config, video_jellyfin_test
|
||||
ok, message = video_jellyfin_test(video_jellyfin_config())
|
||||
if ok:
|
||||
return jsonify({"success": True, "message": message})
|
||||
return jsonify({"success": False, "error": message})
|
||||
except Exception as e:
|
||||
return jsonify({"success": False, "error": str(e) or "connection failed"})
|
||||
|
||||
@bp.route("/jellyfin/users", methods=["GET"])
|
||||
def video_jellyfin_users():
|
||||
"""List the Jellyfin server's users so the video side can pick one (its
|
||||
libraries are scoped to that user) — mirrors the music user picker. Uses
|
||||
video's own effective Jellyfin creds."""
|
||||
from . import get_video_db
|
||||
try:
|
||||
from core.video.sources import video_jellyfin_config
|
||||
cfg = video_jellyfin_config()
|
||||
base = (cfg.get("base_url") or "").rstrip("/")
|
||||
key = cfg.get("api_key") or ""
|
||||
if not base or not key:
|
||||
return jsonify({"success": False, "users": []})
|
||||
import requests
|
||||
r = requests.get(base + "/Users", headers={"X-Emby-Token": key}, timeout=8)
|
||||
if r.status_code != 200:
|
||||
return jsonify({"success": False, "users": [], "error": "HTTP %d" % r.status_code})
|
||||
users = r.json() or []
|
||||
out = [{"id": u.get("Id"), "name": u.get("Name"),
|
||||
"admin": bool((u.get("Policy") or {}).get("IsAdministrator"))}
|
||||
for u in users if u.get("Id")]
|
||||
selected = get_video_db().get_setting("video_jellyfin_user") or ""
|
||||
return jsonify({"success": True, "users": out, "selected": selected})
|
||||
except Exception as e:
|
||||
return jsonify({"success": False, "users": [], "error": str(e)})
|
||||
|
||||
@bp.route("/jellyfin/user", methods=["POST"])
|
||||
def video_jellyfin_user_set():
|
||||
"""Persist the chosen Jellyfin user (its Id) for the video side."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
get_video_db().set_setting("video_jellyfin_user", (body.get("user") or "").strip())
|
||||
return jsonify({"status": "saved"})
|
||||
|
||||
@bp.route("/libraries", methods=["POST"])
|
||||
def save_video_libraries():
|
||||
from . import get_video_db
|
||||
try:
|
||||
from core.video.sources import resolve_video_server
|
||||
body = request.get_json(silent=True) or {}
|
||||
server = resolve_video_server()
|
||||
if not server:
|
||||
return jsonify({"error": "no video server"}), 400
|
||||
get_video_db().set_library_selection(server, body.get("movies"), body.get("tv"))
|
||||
return jsonify({"status": "saved", "server": server})
|
||||
except Exception:
|
||||
logger.exception("Failed to save video library selection")
|
||||
return jsonify({"error": "Failed to save selection"}), 500
|
||||
34
api/video/library.py
Normal file
34
api/video/library.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
"""Video library listing endpoint.
|
||||
|
||||
GET /api/video/library -> {"movies": [...], "shows": [...]}
|
||||
Reads what the last scan mirrored from the media server into video.db.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.library")
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/library", methods=["GET"])
|
||||
def video_library():
|
||||
from . import get_video_db
|
||||
from core.video.sources import resolve_video_server
|
||||
try:
|
||||
return jsonify(get_video_db().query_library(
|
||||
request.args.get("kind", "movies"),
|
||||
search=request.args.get("search") or None,
|
||||
letter=request.args.get("letter") or None,
|
||||
sort=request.args.get("sort", "title"),
|
||||
status=request.args.get("status", "all"),
|
||||
page=request.args.get("page", 1),
|
||||
limit=request.args.get("limit", 75),
|
||||
server_source=resolve_video_server(),
|
||||
))
|
||||
except Exception:
|
||||
logger.exception("Failed to query video library")
|
||||
return jsonify({"error": "Failed to load video library"}), 500
|
||||
160
api/video/manual_import.py
Normal file
160
api/video/manual_import.py
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
"""Manual / failed-import resolution API (isolated).
|
||||
|
||||
When a finished download can't be auto-placed it's parked as ``import_failed`` with
|
||||
the file left on disk (``dest_path`` points at it). The Import page surfaces these and
|
||||
lets the user place them by hand:
|
||||
|
||||
GET /api/video/import/failed → the queue of unplaced downloads
|
||||
POST /api/video/import/<id>/place → force-import to the user's chosen identity
|
||||
POST /api/video/import/<id>/dismiss → drop the row (optionally delete the file)
|
||||
|
||||
The identity picker on the page reuses the existing /api/video/search (TMDB, with a
|
||||
``library_id`` annotation for owned titles) — no new search endpoint needed here.
|
||||
Reads only the video engine + video.db; nothing from the music side.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.manual_import")
|
||||
|
||||
_KIND_FOR_SCOPE = {"movie": "movie", "episode": "show"}
|
||||
|
||||
|
||||
def _ctx(row):
|
||||
try:
|
||||
c = json.loads(row.get("search_ctx") or "{}")
|
||||
return c if isinstance(c, dict) else {}
|
||||
except (ValueError, TypeError):
|
||||
return {}
|
||||
|
||||
|
||||
def _failed_view(row):
|
||||
"""The render-ready shape for one unplaced download."""
|
||||
c = _ctx(row)
|
||||
return {
|
||||
"id": row.get("id"),
|
||||
"title": row.get("title"),
|
||||
"kind": row.get("kind"),
|
||||
"year": row.get("year"),
|
||||
"reason": row.get("error"),
|
||||
"file": row.get("dest_path"), # where the file is sitting, unplaced
|
||||
"release_title": row.get("release_title"),
|
||||
"poster_url": row.get("poster_url"),
|
||||
"scope": c.get("scope"),
|
||||
"season": c.get("season"),
|
||||
"episode": c.get("episode"),
|
||||
}
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/import/failed", methods=["GET"])
|
||||
def video_import_failed():
|
||||
from . import get_video_db
|
||||
rows = get_video_db().get_import_failed_video_downloads()
|
||||
return jsonify({"success": True, "items": [_failed_view(r) for r in rows]})
|
||||
|
||||
@bp.route("/import/<int:dl_id>/place", methods=["POST"])
|
||||
def video_import_place(dl_id):
|
||||
"""Force-import an unplaced file to the user's chosen identity. Body:
|
||||
{scope, title, year, season, episode, episode_title, media_id}."""
|
||||
from . import get_video_db
|
||||
from core.video import organization
|
||||
from core.video.download_pipeline import target_dir_for
|
||||
from core.video.importer import real_fs, run_import
|
||||
from core.video.mediainfo import probe
|
||||
|
||||
db = get_video_db()
|
||||
row = db.get_video_download(dl_id)
|
||||
if not row or row.get("status") != "import_failed":
|
||||
return jsonify({"success": False, "error": "Not an unplaced import."}), 404
|
||||
src = row.get("dest_path")
|
||||
if not src or not os.path.exists(src):
|
||||
return jsonify({"success": False, "error": "The file is no longer on disk."}), 410
|
||||
|
||||
body = request.get_json(silent=True) or {}
|
||||
scope = str(body.get("scope") or "").lower()
|
||||
if scope not in ("movie", "episode"):
|
||||
return jsonify({"success": False, "error": "Choose a movie or an episode."}), 400
|
||||
|
||||
paths = {k: db.get_setting(k) or "" for k in ("movies_path", "tv_path", "youtube_path")}
|
||||
if not paths["movies_path"]:
|
||||
paths["movies_path"] = db.get_setting("transfer_path") or ""
|
||||
override = {
|
||||
"scope": scope,
|
||||
"title": body.get("title"),
|
||||
"year": body.get("year"),
|
||||
"season": body.get("season"),
|
||||
"episode": body.get("episode"),
|
||||
"episode_title": body.get("episode_title"),
|
||||
"media_id": body.get("media_id"),
|
||||
"target_dir": target_dir_for(_KIND_FOR_SCOPE[scope], paths),
|
||||
}
|
||||
settings = organization.load(db)
|
||||
prober = probe if settings.get("verify_with_ffprobe", True) else None
|
||||
patch = run_import(row, src, fs=real_fs(), prober=prober, settings=settings,
|
||||
force=True, override=override)
|
||||
try:
|
||||
db.update_video_download(dl_id, **patch)
|
||||
except Exception:
|
||||
logger.exception("manual place: failed to persist import %s", dl_id)
|
||||
return jsonify({"success": False, "error": "Couldn't save the result."}), 500
|
||||
ok = patch.get("status") == "completed"
|
||||
if ok:
|
||||
# Write NFO + artwork sidecars for the chosen identity (best-effort), then
|
||||
# refresh the server + DB the same way the auto-download path does
|
||||
# (batch-complete → scan chain), so the manually-placed title shows up
|
||||
# without waiting for a scheduled scan.
|
||||
sidecar_dl = {"kind": _KIND_FOR_SCOPE[scope], "media_source": "tmdb",
|
||||
"media_id": override.get("media_id"),
|
||||
"poster_url": row.get("poster_url"),
|
||||
"search_ctx": json.dumps({"scope": scope, "season": override.get("season"),
|
||||
"episode": override.get("episode")})}
|
||||
if settings.get("save_artwork") or settings.get("write_nfo"):
|
||||
try:
|
||||
from core.video.download_monitor import write_sidecars
|
||||
from core.video.importer import real_fs
|
||||
write_sidecars(db, sidecar_dl, patch["dest_path"], settings, real_fs())
|
||||
except Exception:
|
||||
logger.exception("manual place: sidecar write failed for %s", dl_id)
|
||||
if settings.get("download_subtitles"):
|
||||
try:
|
||||
from core.video.download_monitor import write_subtitles_for
|
||||
from core.video.importer import real_fs
|
||||
write_subtitles_for(db, sidecar_dl, patch["dest_path"], settings, real_fs())
|
||||
except Exception:
|
||||
logger.exception("manual place: subtitle fetch failed for %s", dl_id)
|
||||
try:
|
||||
from core.video.download_events import notify_batch_complete
|
||||
notify_batch_complete({"completed": 1, "manual": True})
|
||||
except Exception:
|
||||
logger.exception("manual place: batch-complete notify failed for %s", dl_id)
|
||||
return jsonify({"success": ok, "status": patch.get("status"),
|
||||
"dest_path": patch.get("dest_path"), "error": patch.get("error")})
|
||||
|
||||
@bp.route("/import/<int:dl_id>/dismiss", methods=["POST"])
|
||||
def video_import_dismiss(dl_id):
|
||||
"""Drop a failed-import row. Body {delete_file: bool} optionally removes the
|
||||
unplaced file from disk too."""
|
||||
from . import get_video_db
|
||||
db = get_video_db()
|
||||
row = db.get_video_download(dl_id)
|
||||
if not row or row.get("status") != "import_failed":
|
||||
return jsonify({"success": False, "error": "Not an unplaced import."}), 404
|
||||
body = request.get_json(silent=True) or {}
|
||||
if body.get("delete_file"):
|
||||
src = row.get("dest_path")
|
||||
if src and os.path.exists(src):
|
||||
try:
|
||||
os.remove(src)
|
||||
except OSError:
|
||||
logger.warning("dismiss: could not delete %s", src)
|
||||
db.update_video_download(dl_id, status="cancelled",
|
||||
error="Dismissed from manual import")
|
||||
return jsonify({"success": True})
|
||||
152
api/video/poster.py
Normal file
152
api/video/poster.py
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
"""Video poster proxy.
|
||||
|
||||
GET /api/video/poster/<kind>/<id> streams a movie/show poster from the media
|
||||
server, server-side (so the Plex token / Jellyfin key never reaches the
|
||||
browser). Falls back to 404 so the frontend shows its placeholder.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import Response, abort, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.poster")
|
||||
|
||||
|
||||
def _req_width():
|
||||
"""Optional ?w= thumbnail width (clamped) so the calendar/library don't pull
|
||||
full-size art into tiny cells. None = original."""
|
||||
try:
|
||||
w = request.args.get("w", type=int)
|
||||
except Exception:
|
||||
w = None
|
||||
if not w:
|
||||
return None
|
||||
return max(48, min(1600, w))
|
||||
|
||||
|
||||
def _tmdb_resize(url, w, backdrop):
|
||||
"""Rewrite a TMDB image URL's size segment (/t/p/<size>/...) to a bucket near
|
||||
``w`` so episode stills load small. Best-effort — unchanged if it doesn't match."""
|
||||
import re
|
||||
buckets = [300, 780, 1280] if backdrop else [185, 342, 500, 780]
|
||||
pick = next((b for b in buckets if w <= b), buckets[-1])
|
||||
return re.sub(r"/t/p/[^/]+/", "/t/p/w%d/" % pick, url, count=1)
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
def _stream_art(kind, item_id, art):
|
||||
from . import get_video_db
|
||||
ref = get_video_db().get_art_ref(kind, item_id, art)
|
||||
if not ref or not ref.get("poster_url"):
|
||||
abort(404)
|
||||
try:
|
||||
import requests
|
||||
w = _req_width()
|
||||
backdrop = art == "backdrop"
|
||||
path = ref["poster_url"]
|
||||
# Enrichment can store a full external URL (e.g. a TMDB season poster) —
|
||||
# stream it directly; otherwise it's a server path needing the token.
|
||||
if path.startswith("http://") or path.startswith("https://"):
|
||||
if w and "image.tmdb.org" in path:
|
||||
path = _tmdb_resize(path, w, backdrop)
|
||||
upstream = requests.get(path, timeout=15, stream=True)
|
||||
if upstream.status_code != 200:
|
||||
abort(404)
|
||||
resp = Response(upstream.iter_content(8192),
|
||||
content_type=upstream.headers.get("Content-Type", "image/jpeg"))
|
||||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||
return resp
|
||||
# Art is served from the VIDEO side's effective connection (its own
|
||||
# creds, or inherited from music) — that's where the item was scanned.
|
||||
from core.video.sources import video_plex_config, video_jellyfin_config
|
||||
source = ref.get("server_source")
|
||||
fallback = None
|
||||
if source == "plex":
|
||||
cfg = video_plex_config()
|
||||
base, token = cfg.get("base_url"), cfg.get("token")
|
||||
if not base or not token:
|
||||
abort(404)
|
||||
base = base.rstrip("/")
|
||||
params = {"X-Plex-Token": token}
|
||||
if w:
|
||||
# Plex photo transcoder → a right-sized JPEG instead of full art;
|
||||
# fall back to the original if a server has transcoding disabled.
|
||||
from urllib.parse import quote
|
||||
h = int(w * (9 / 16 if backdrop else 3 / 2))
|
||||
url = (base + "/photo/:/transcode?width=%d&height=%d&minSize=1&upscale=1&url=%s"
|
||||
% (w, h, quote(ref["poster_url"], safe="")))
|
||||
fallback = base + ref["poster_url"]
|
||||
else:
|
||||
url = base + ref["poster_url"]
|
||||
elif source == "jellyfin":
|
||||
cfg = video_jellyfin_config()
|
||||
base, key = cfg.get("base_url"), cfg.get("api_key")
|
||||
if not base:
|
||||
abort(404)
|
||||
image = "Backdrop" if backdrop else "Primary"
|
||||
url = base.rstrip("/") + f"/Items/{ref['server_id']}/Images/{image}"
|
||||
params = {"api_key": key} if key else {}
|
||||
if w:
|
||||
params["maxWidth"] = w
|
||||
params["quality"] = 90
|
||||
else:
|
||||
abort(404)
|
||||
|
||||
upstream = requests.get(url, params=params, timeout=15, stream=True)
|
||||
if upstream.status_code != 200 and fallback:
|
||||
upstream = requests.get(fallback, params=params, timeout=15, stream=True)
|
||||
if upstream.status_code != 200:
|
||||
abort(404)
|
||||
ctype = upstream.headers.get("Content-Type", "image/jpeg")
|
||||
resp = Response(upstream.iter_content(8192), content_type=ctype)
|
||||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||
return resp
|
||||
except Exception:
|
||||
logger.exception("video %s proxy failed for %s/%s", art, kind, item_id)
|
||||
abort(404)
|
||||
|
||||
@bp.route("/poster/<kind>/<int:item_id>", methods=["GET"])
|
||||
def video_poster(kind, item_id):
|
||||
return _stream_art(kind, item_id, "poster")
|
||||
|
||||
@bp.route("/backdrop/<kind>/<int:item_id>", methods=["GET"])
|
||||
def video_backdrop(kind, item_id):
|
||||
return _stream_art(kind, item_id, "backdrop")
|
||||
|
||||
@bp.route("/img", methods=["GET"])
|
||||
def video_img_proxy():
|
||||
"""Same-origin image proxy (SSRF-safe allowlist). TMDB for accent-sampling,
|
||||
and YouTube CDN (avatars/banners/thumbnails) so channel art loads reliably
|
||||
regardless of hotlink/CORS policy."""
|
||||
from flask import request
|
||||
from urllib.parse import urlparse
|
||||
url = request.args.get("u", "")
|
||||
if not url.startswith("https://"):
|
||||
abort(404)
|
||||
host = (urlparse(url).hostname or "").lower()
|
||||
# image.tmdb.org + any YouTube image host (yt3/yt4.ggpht, googleusercontent, *.ytimg)
|
||||
ok = host == "image.tmdb.org" or any(
|
||||
host == s or host.endswith("." + s) for s in ("ytimg.com", "ggpht.com", "googleusercontent.com"))
|
||||
if not ok:
|
||||
abort(404)
|
||||
try:
|
||||
import requests
|
||||
# A browser UA — Google's image CDN (yt3/googleusercontent) 403s some
|
||||
# avatars when fetched with no User-Agent, which blanked search results.
|
||||
upstream = requests.get(url, timeout=15, stream=True, headers={
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
|
||||
})
|
||||
if upstream.status_code != 200:
|
||||
abort(404)
|
||||
resp = Response(upstream.iter_content(8192),
|
||||
content_type=upstream.headers.get("Content-Type", "image/jpeg"))
|
||||
resp.headers["Cache-Control"] = "public, max-age=604800"
|
||||
resp.headers["Access-Control-Allow-Origin"] = "*"
|
||||
return resp
|
||||
except Exception:
|
||||
logger.exception("video image proxy failed for %s", url)
|
||||
abort(404)
|
||||
61
api/video/scan.py
Normal file
61
api/video/scan.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
"""Video library scan endpoints.
|
||||
|
||||
POST /api/video/scan/request -> start a background scan of the active server
|
||||
GET /api/video/scan/status -> current scan progress/state
|
||||
|
||||
The scan READS the media server (source of truth) into video.db. Triggering the
|
||||
server's own rescan (post-download) is wired separately into the download flow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.scan")
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/scan/request", methods=["POST"])
|
||||
def video_scan_request():
|
||||
from . import get_video_db
|
||||
from core.video.scanner import get_video_scanner
|
||||
from core.video.sources import get_active_video_source
|
||||
body = request.get_json(silent=True) or {}
|
||||
mode = body.get("mode", "full")
|
||||
# Which library to scan — movies and TV are independent libraries, so the
|
||||
# UI can target one or both. The scanner normalizes/validates it.
|
||||
media_type = body.get("media_type", "all")
|
||||
scanner = get_video_scanner(get_video_db())
|
||||
return jsonify(scanner.request_scan(get_active_video_source, mode, media_type))
|
||||
|
||||
@bp.route("/scan/status", methods=["GET"])
|
||||
def video_scan_status():
|
||||
from . import get_video_db
|
||||
from core.video.scanner import get_video_scanner
|
||||
return jsonify(get_video_scanner(get_video_db()).get_status())
|
||||
|
||||
@bp.route("/scan/stop", methods=["POST"])
|
||||
def video_scan_stop():
|
||||
from . import get_video_db
|
||||
from core.video.scanner import get_video_scanner
|
||||
return jsonify(get_video_scanner(get_video_db()).cancel())
|
||||
|
||||
# ── Server-side scan (distinct from the SoulSync-reads-server scan above) ──
|
||||
# This tells the media server (Plex/Jellyfin) to rescan its OWN folders so
|
||||
# newly-downloaded files get indexed — the manual twin of the post-download
|
||||
# 'Scan Video Server' automation. media_type scopes it to Movies / TV / both.
|
||||
@bp.route("/scan/server", methods=["POST"])
|
||||
def video_scan_server():
|
||||
from core.video.sources import refresh_video_server_sections
|
||||
body = request.get_json(silent=True) or {}
|
||||
media_type = body.get("media_type", "all")
|
||||
return jsonify(refresh_video_server_sections(media_type))
|
||||
|
||||
@bp.route("/scan/server/status", methods=["GET"])
|
||||
def video_scan_server_status():
|
||||
# {scanning: true|false|null} — null when the adapter can't report state.
|
||||
from core.video.sources import video_server_scan_in_progress
|
||||
media_type = request.args.get("media_type", "all")
|
||||
return jsonify({"scanning": video_server_scan_in_progress(media_type)})
|
||||
43
api/video/search.py
Normal file
43
api/video/search.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
"""Video search API (in-app, isolated).
|
||||
|
||||
GET /api/video/search?q=... → TMDB multi-search (movies / shows / people),
|
||||
movie/show results annotated with library_id
|
||||
if already owned.
|
||||
|
||||
Everything resolves back into SoulSync — results link to the library detail
|
||||
(owned) or the TMDB-backed detail (not owned); people open the in-app person
|
||||
page. Reads only the video engine + video.db.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.search")
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/search", methods=["GET"])
|
||||
def video_search():
|
||||
q = (request.args.get("q") or "").strip()
|
||||
if not q:
|
||||
return jsonify({"results": [], "query": ""})
|
||||
try:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
results = get_video_enrichment_engine().search(q)
|
||||
except Exception:
|
||||
logger.exception("video search failed for %r", q)
|
||||
results = []
|
||||
return jsonify({"results": results, "query": q})
|
||||
|
||||
@bp.route("/trending", methods=["GET"])
|
||||
def video_trending():
|
||||
try:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
results = get_video_enrichment_engine().trending()
|
||||
except Exception:
|
||||
logger.exception("video trending failed")
|
||||
results = []
|
||||
return jsonify({"results": results})
|
||||
122
api/video/watchlist.py
Normal file
122
api/video/watchlist.py
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
"""Video watchlist API — the user's curated follow-list of shows + people.
|
||||
|
||||
Mirrors the music watchlist's add/remove/list/check shape. v1 just manages
|
||||
membership (so cards can toggle + the Watchlist page can render); the
|
||||
monitoring/discovery engine that turns a follow into new downloads is a later
|
||||
phase. Reads/writes only video_library.db via the shared VideoDatabase.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.watchlist")
|
||||
|
||||
_KINDS = ("show", "person")
|
||||
|
||||
|
||||
def _server():
|
||||
"""Active video server_source (scopes the airing-show default). None on error."""
|
||||
try:
|
||||
from core.video.sources import resolve_video_server
|
||||
return resolve_video_server()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/watchlist", methods=["GET"])
|
||||
def video_watchlist_list():
|
||||
"""All watchlist entries grouped by kind (for the tabbed page).
|
||||
|
||||
Shows include actively-airing library shows by default, scoped to the
|
||||
active video server so Plex/Jellyfin libraries don't commingle."""
|
||||
from . import get_video_db
|
||||
try:
|
||||
db = get_video_db()
|
||||
server = _server()
|
||||
kind = request.args.get("kind")
|
||||
counts = db.watchlist_counts(server_source=server)
|
||||
if kind in _KINDS:
|
||||
# Paged + searchable, like the library page.
|
||||
res = db.query_watchlist(
|
||||
kind, search=request.args.get("search", ""),
|
||||
sort=request.args.get("sort", "default"),
|
||||
page=request.args.get("page", 1), limit=request.args.get("limit", 60),
|
||||
server_source=server)
|
||||
return jsonify({"success": True, "kind": kind, "counts": counts, **res})
|
||||
# No kind → grouped (counts + first-glance lists).
|
||||
rows = db.list_watchlist(server_source=server)
|
||||
shows = [r for r in rows if r.get("kind") == "show"]
|
||||
people = [r for r in rows if r.get("kind") == "person"]
|
||||
return jsonify({"success": True, "shows": shows, "people": people, "counts": counts})
|
||||
except Exception:
|
||||
logger.exception("Failed to list video watchlist")
|
||||
return jsonify({"success": False, "error": "Failed to load watchlist"}), 500
|
||||
|
||||
@bp.route("/watchlist/counts", methods=["GET"])
|
||||
def video_watchlist_counts():
|
||||
from . import get_video_db
|
||||
try:
|
||||
return jsonify({"success": True, **get_video_db().watchlist_counts(server_source=_server())})
|
||||
except Exception:
|
||||
logger.exception("Failed to count video watchlist")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/watchlist/add", methods=["POST"])
|
||||
def video_watchlist_add():
|
||||
"""Add a show/person. Body: {kind, tmdb_id, title, poster_url?, library_id?}."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
kind = body.get("kind")
|
||||
tmdb_id = body.get("tmdb_id")
|
||||
title = (body.get("title") or "").strip()
|
||||
if kind not in _KINDS or not tmdb_id or not title:
|
||||
return jsonify({"success": False, "error": "kind, tmdb_id and title are required"}), 400
|
||||
try:
|
||||
ok = get_video_db().add_to_watchlist(
|
||||
kind, int(tmdb_id), title,
|
||||
poster_url=body.get("poster_url") or None,
|
||||
library_id=body.get("library_id") or None)
|
||||
if not ok:
|
||||
return jsonify({"success": False, "error": "Could not add to watchlist"}), 400
|
||||
return jsonify({"success": True, "watched": True})
|
||||
except Exception:
|
||||
logger.exception("Failed to add to video watchlist")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/watchlist/remove", methods=["POST"])
|
||||
def video_watchlist_remove():
|
||||
"""Remove a show/person. Body: {kind, tmdb_id}."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
kind = body.get("kind")
|
||||
tmdb_id = body.get("tmdb_id")
|
||||
if kind not in _KINDS or not tmdb_id:
|
||||
return jsonify({"success": False, "error": "kind and tmdb_id are required"}), 400
|
||||
try:
|
||||
removed = get_video_db().remove_from_watchlist(kind, int(tmdb_id))
|
||||
return jsonify({"success": True, "watched": False, "removed": removed})
|
||||
except Exception:
|
||||
logger.exception("Failed to remove from video watchlist")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/watchlist/check", methods=["POST"])
|
||||
def video_watchlist_check():
|
||||
"""Hydrate cards. Body: {kind, tmdb_ids: [...]} → {results: {id: true}}.
|
||||
Only watched ids appear in results (absent = not watched)."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
kind = body.get("kind")
|
||||
ids = body.get("tmdb_ids") or []
|
||||
if kind not in _KINDS:
|
||||
return jsonify({"success": False, "error": "kind is required"}), 400
|
||||
try:
|
||||
state = get_video_db().watchlist_state(kind, ids, server_source=_server())
|
||||
# JSON object keys must be strings.
|
||||
return jsonify({"success": True, "results": {str(k): True for k in state}})
|
||||
except Exception:
|
||||
logger.exception("Failed to check video watchlist")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
185
api/video/wishlist.py
Normal file
185
api/video/wishlist.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
"""Video wishlist API — the curated 'get this' list (movies + episodes).
|
||||
|
||||
Atomic units are movies and episodes; adding a whole show or a season just hands
|
||||
us the explicit episodes to expand into rows. v1 manages membership + the tabbed
|
||||
Movies/TV page; the search/download engine that fulfils a wish is a later phase.
|
||||
Reads/writes only video_library.db via the shared VideoDatabase.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.wishlist")
|
||||
|
||||
_KINDS = ("movie", "show")
|
||||
_SCOPES = ("movie", "show", "season", "episode")
|
||||
|
||||
|
||||
def _server():
|
||||
"""Active video server_source (stored on rows, informational). None on error."""
|
||||
try:
|
||||
from core.video.sources import resolve_video_server
|
||||
return resolve_video_server()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/wishlist", methods=["GET"])
|
||||
def video_wishlist_list():
|
||||
"""Paged slice for a tab (kind='movie'|'show'), or counts-only with no kind.
|
||||
Shows are grouped show→season→episode with wanted/done roll-ups."""
|
||||
from . import get_video_db
|
||||
try:
|
||||
db = get_video_db()
|
||||
counts = db.wishlist_counts()
|
||||
kind = request.args.get("kind")
|
||||
if kind in _KINDS:
|
||||
res = db.query_wishlist(
|
||||
kind, search=request.args.get("search", ""), sort=request.args.get("sort", "added"),
|
||||
page=request.args.get("page", 1), limit=request.args.get("limit", 60))
|
||||
return jsonify({"success": True, "kind": kind, "counts": counts, **res})
|
||||
return jsonify({"success": True, "counts": counts})
|
||||
except Exception:
|
||||
logger.exception("Failed to list video wishlist")
|
||||
return jsonify({"success": False, "error": "Failed to load wishlist"}), 500
|
||||
|
||||
@bp.route("/wishlist/counts", methods=["GET"])
|
||||
def video_wishlist_counts():
|
||||
from . import get_video_db
|
||||
try:
|
||||
db = get_video_db()
|
||||
counts = db.wishlist_counts() # {movie, show, episode, total(movie+ep)}
|
||||
yt = db.youtube_wishlist_counts() # {channel, video} (its own table-shape)
|
||||
counts["video"] = yt.get("video", 0)
|
||||
counts["channel"] = yt.get("channel", 0)
|
||||
# The header/sidebar badge is the WHOLE wishlist — movies + episodes + YouTube
|
||||
# videos — so fold the YouTube count into the total (it was movies+episodes only,
|
||||
# which is why a YouTube-only wishlist showed no badge).
|
||||
counts["total"] = counts.get("total", 0) + counts["video"]
|
||||
return jsonify({"success": True, **counts})
|
||||
except Exception:
|
||||
logger.exception("Failed to count video wishlist")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/wishlist/add", methods=["POST"])
|
||||
def video_wishlist_add():
|
||||
"""Add a movie or a set of a show's episodes. Body is one of:
|
||||
{"movie": {tmdb_id, title, year?, poster_url?, library_id?}}
|
||||
{"show": {tmdb_id, title, poster_url?, library_id?},
|
||||
"episodes": [{season_number, episode_number, title?, air_date?}, …]}"""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
srv = _server()
|
||||
db = get_video_db()
|
||||
try:
|
||||
movie = body.get("movie")
|
||||
if movie and movie.get("tmdb_id") and (movie.get("title") or "").strip():
|
||||
ok = db.add_movie_to_wishlist(
|
||||
int(movie["tmdb_id"]), movie["title"].strip(), year=movie.get("year"),
|
||||
poster_url=movie.get("poster_url") or None,
|
||||
library_id=movie.get("library_id") or None, server_source=srv)
|
||||
return jsonify({"success": ok, "added": 1 if ok else 0, "counts": db.wishlist_counts()})
|
||||
|
||||
show = body.get("show")
|
||||
episodes = body.get("episodes") or []
|
||||
if show and show.get("tmdb_id") and (show.get("title") or "").strip() and episodes:
|
||||
n = db.add_episodes_to_wishlist(
|
||||
int(show["tmdb_id"]), show["title"].strip(), episodes,
|
||||
poster_url=show.get("poster_url") or None,
|
||||
library_id=show.get("library_id") or None, server_source=srv)
|
||||
return jsonify({"success": n > 0, "added": n, "counts": db.wishlist_counts()})
|
||||
|
||||
return jsonify({"success": False, "error": "movie or show+episodes required"}), 400
|
||||
except Exception:
|
||||
logger.exception("Failed to add to video wishlist")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/wishlist/remove", methods=["POST"])
|
||||
def video_wishlist_remove():
|
||||
"""Remove at any granularity. Body: {scope, tmdb_id, season_number?, episode_number?}
|
||||
where scope ∈ movie|show|season|episode."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
scope, tmdb_id = body.get("scope"), body.get("tmdb_id")
|
||||
if scope not in _SCOPES or not tmdb_id:
|
||||
return jsonify({"success": False, "error": "scope and tmdb_id are required"}), 400
|
||||
try:
|
||||
db = get_video_db()
|
||||
removed = db.remove_from_wishlist(
|
||||
scope, tmdb_id=int(tmdb_id),
|
||||
season_number=body.get("season_number"), episode_number=body.get("episode_number"))
|
||||
return jsonify({"success": True, "removed": removed, "counts": db.wishlist_counts()})
|
||||
except Exception:
|
||||
logger.exception("Failed to remove from video wishlist")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/wishlist/clear", methods=["POST"])
|
||||
def video_wishlist_clear():
|
||||
"""Empty an entire wishlist tab. Body: {kind} where kind ∈ movie|show|youtube."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
kind = body.get("kind")
|
||||
if kind not in ("movie", "show", "youtube"):
|
||||
return jsonify({"success": False, "error": "kind must be movie|show|youtube"}), 400
|
||||
try:
|
||||
db = get_video_db()
|
||||
removed = db.clear_wishlist(kind)
|
||||
return jsonify({"success": True, "removed": removed,
|
||||
"counts": db.wishlist_counts(), "youtube_counts": db.youtube_wishlist_counts()})
|
||||
except Exception:
|
||||
logger.exception("Failed to clear video wishlist")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/wishlist/backfill-art", methods=["POST"])
|
||||
def video_wishlist_backfill_art():
|
||||
"""Fill episode stills + season posters for rows that predate art-capture.
|
||||
One tmdb_season call per (show, season); best-effort. Returns rows filled."""
|
||||
from . import get_video_db
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
db = get_video_db()
|
||||
eng = get_video_enrichment_engine()
|
||||
updated = 0
|
||||
try:
|
||||
for grp in db.wishlist_art_backfill_targets():
|
||||
try:
|
||||
se = eng.tmdb_season(grp["tmdb_id"], grp["season_number"]) or {}
|
||||
except Exception:
|
||||
continue
|
||||
if se.get("poster_url"):
|
||||
updated += db.set_wishlist_season_poster(grp["tmdb_id"], grp["season_number"], se["poster_url"])
|
||||
for ep in (se.get("episodes") or []):
|
||||
en = ep.get("episode_number")
|
||||
if en is None:
|
||||
continue
|
||||
if ep.get("still_url") and db.set_wishlist_still(grp["tmdb_id"], grp["season_number"], en, ep["still_url"]):
|
||||
updated += 1
|
||||
if ep.get("overview"):
|
||||
db.set_wishlist_episode_overview(grp["tmdb_id"], grp["season_number"], en, ep["overview"])
|
||||
return jsonify({"success": True, "updated": updated})
|
||||
except Exception:
|
||||
logger.exception("wishlist art backfill failed")
|
||||
return jsonify({"success": False, "updated": updated})
|
||||
|
||||
@bp.route("/wishlist/check", methods=["POST"])
|
||||
def video_wishlist_check():
|
||||
"""Hydrate cards/modal. Body: {movie_ids: [...], show_tmdb_id?} →
|
||||
{movies: [ids already wished], episodes: ['S_E' already wished]}."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
try:
|
||||
db = get_video_db()
|
||||
st = db.wishlist_state(
|
||||
movie_ids=body.get("movie_ids") or [], show_tmdb_id=body.get("show_tmdb_id"))
|
||||
out = {"success": True, "movies": sorted(st["movies"]), "episodes": sorted(st["episodes"])}
|
||||
shows = body.get("shows") # multi-show membership for the calendar button
|
||||
if shows:
|
||||
keys = db.wishlist_keys_for_shows(shows)
|
||||
out["by_show"] = {str(tid): sorted(ks) for tid, ks in keys.items()}
|
||||
return jsonify(out)
|
||||
except Exception:
|
||||
logger.exception("Failed to check video wishlist")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
494
api/video/youtube.py
Normal file
494
api/video/youtube.py
Normal file
|
|
@ -0,0 +1,494 @@
|
|||
"""Video YouTube API — follow a channel as a "show", its uploads flow to the
|
||||
wishlist (visual-first: no downloading yet).
|
||||
|
||||
GET /youtube/resolve?url=... → preview a pasted channel (meta + recent
|
||||
uploads) + whether it's already followed.
|
||||
POST /youtube/follow → {url} (or a pre-resolved channel) → follow +
|
||||
wish its videos. Returns the channel + counts.
|
||||
POST /youtube/unfollow → {youtube_id} → un-follow (keeps wished videos).
|
||||
GET /youtube/channels → followed channels (for the watchlist page).
|
||||
GET /youtube/wishlist → wished videos grouped by channel (+ counts).
|
||||
POST /youtube/wishlist/remove → {scope: channel|video, source_id}.
|
||||
|
||||
yt-dlp only; reads/writes only video_library.db.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("video_api.youtube")
|
||||
|
||||
# Cap how many recent uploads a follow pulls into the wishlist. Channels can have
|
||||
# thousands of videos; flat listing is cheap but we don't want to wish them all.
|
||||
_FOLLOW_LIMIT = 30
|
||||
_RESOLVE_LIMIT = 24
|
||||
|
||||
|
||||
def _server():
|
||||
try:
|
||||
from core.video.sources import resolve_video_server
|
||||
return resolve_video_server()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def register_routes(bp):
|
||||
@bp.route("/youtube/video/<video_id>/segments", methods=["GET"])
|
||||
def video_youtube_segments(video_id):
|
||||
"""SponsorBlock crowd segments stored for a video (sponsor/intro/outro/…),
|
||||
so the player can offer skips. [] until the SponsorBlock worker enriches it."""
|
||||
from . import get_video_db
|
||||
return jsonify({"video_id": video_id,
|
||||
"segments": get_video_db().youtube_video_segments(video_id)})
|
||||
|
||||
@bp.route("/youtube/resolve", methods=["GET"])
|
||||
def video_youtube_resolve():
|
||||
"""Preview a pasted channel URL without committing. Returns the channel +
|
||||
recent uploads, plus ``following`` so the button can hydrate."""
|
||||
from . import get_video_db
|
||||
from core.video import youtube as yt
|
||||
url = (request.args.get("url") or "").strip()
|
||||
if not url:
|
||||
return jsonify({"success": False, "error": "url is required"}), 400
|
||||
try:
|
||||
limit = int(request.args.get("limit") or _RESOLVE_LIMIT)
|
||||
except (TypeError, ValueError):
|
||||
limit = _RESOLVE_LIMIT
|
||||
try:
|
||||
# A playlist link → resolve as a playlist (curator-ordered, partial set).
|
||||
if yt.parse_playlist_id(url):
|
||||
pl = yt.resolve_playlist(url, limit=max(1, min(50, limit)))
|
||||
if not pl:
|
||||
return jsonify({"success": False, "error": "Could not read that playlist"}), 404
|
||||
following = bool(get_video_db().playlist_watch_state([pl["playlist_id"]]))
|
||||
return jsonify({"success": True, "playlist": pl, "following": following})
|
||||
channel = yt.resolve_channel(url, limit=max(1, min(50, limit)))
|
||||
if not channel:
|
||||
return jsonify({"success": False,
|
||||
"error": "Not a YouTube channel or playlist link (paste a channel "
|
||||
"URL like youtube.com/@handle, or a playlist link)"}), 404
|
||||
following = bool(get_video_db().channel_watch_state([channel["youtube_id"]]))
|
||||
return jsonify({"success": True, "channel": channel, "following": following})
|
||||
except Exception:
|
||||
logger.exception("youtube resolve failed for %r", url)
|
||||
return jsonify({"success": False, "error": "Could not read that link"}), 500
|
||||
|
||||
@bp.route("/youtube/follow", methods=["POST"])
|
||||
def video_youtube_follow():
|
||||
"""Follow a channel + wish its recent uploads. Body: {url} (re-resolved) or
|
||||
{channel: {...}} already resolved (avoids a second yt-dlp call)."""
|
||||
from . import get_video_db
|
||||
from core.video import youtube as yt
|
||||
body = request.get_json(silent=True) or {}
|
||||
db = get_video_db()
|
||||
try:
|
||||
channel = body.get("channel")
|
||||
if not channel:
|
||||
url = (body.get("url") or "").strip()
|
||||
if not url:
|
||||
return jsonify({"success": False, "error": "url or channel required"}), 400
|
||||
channel = yt.resolve_channel(url, limit=_FOLLOW_LIMIT)
|
||||
if not channel or not channel.get("youtube_id"):
|
||||
return jsonify({"success": False, "error": "Could not resolve channel"}), 404
|
||||
|
||||
followed = db.add_channel_to_watchlist(channel)
|
||||
added = db.add_videos_to_wishlist(channel, channel.get("videos") or [], server_source=_server())
|
||||
if followed: # followed channels get their full upload-date catalog in the background
|
||||
try:
|
||||
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||
get_youtube_date_enricher().enqueue(channel.get("youtube_id"), channel.get("title"))
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify({"success": followed, "following": followed, "added_videos": added,
|
||||
"channel": {k: channel.get(k) for k in ("youtube_id", "title", "avatar_url")},
|
||||
"counts": db.youtube_wishlist_counts()})
|
||||
except Exception:
|
||||
logger.exception("youtube follow failed")
|
||||
return jsonify({"success": False, "error": "Failed to follow channel"}), 500
|
||||
|
||||
@bp.route("/youtube/unfollow", methods=["POST"])
|
||||
def video_youtube_unfollow():
|
||||
"""Un-follow a channel. Body: {youtube_id}. Wished videos are left in place."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
cid = (body.get("youtube_id") or "").strip()
|
||||
if not cid:
|
||||
return jsonify({"success": False, "error": "youtube_id is required"}), 400
|
||||
try:
|
||||
get_video_db().remove_channel_from_watchlist(cid)
|
||||
return jsonify({"success": True, "following": False})
|
||||
except Exception:
|
||||
logger.exception("youtube unfollow failed")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/youtube/playlist/follow", methods=["POST"])
|
||||
def video_youtube_playlist_follow():
|
||||
"""Follow a YouTube playlist. Body: {playlist:{...}} (already resolved) or {url}."""
|
||||
from . import get_video_db
|
||||
from core.video import youtube as yt
|
||||
body = request.get_json(silent=True) or {}
|
||||
db = get_video_db()
|
||||
try:
|
||||
playlist = body.get("playlist")
|
||||
if not playlist:
|
||||
url = (body.get("url") or "").strip()
|
||||
playlist = yt.resolve_playlist(url) if url else None
|
||||
if not playlist or not playlist.get("playlist_id"):
|
||||
return jsonify({"success": False, "error": "Could not resolve playlist"}), 404
|
||||
ok = db.add_playlist_to_watchlist(playlist)
|
||||
if ok and playlist.get("videos"): # remember the count straight away
|
||||
try:
|
||||
db.cache_channel_videos(playlist["playlist_id"], playlist["videos"])
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify({"success": ok, "following": ok,
|
||||
"playlist": {k: playlist.get(k) for k in ("playlist_id", "title", "thumbnail_url")}})
|
||||
except Exception:
|
||||
logger.exception("youtube playlist follow failed")
|
||||
return jsonify({"success": False, "error": "Failed to follow playlist"}), 500
|
||||
|
||||
@bp.route("/youtube/playlist/unfollow", methods=["POST"])
|
||||
def video_youtube_playlist_unfollow():
|
||||
from . import get_video_db
|
||||
pid = ((request.get_json(silent=True) or {}).get("playlist_id") or "").strip()
|
||||
if not pid:
|
||||
return jsonify({"success": False, "error": "playlist_id is required"}), 400
|
||||
try:
|
||||
get_video_db().remove_playlist_from_watchlist(pid)
|
||||
return jsonify({"success": True, "following": False})
|
||||
except Exception:
|
||||
logger.exception("youtube playlist unfollow failed")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/youtube/channels", methods=["GET"])
|
||||
def video_youtube_channels():
|
||||
"""Followed channels (newest first) for the watchlist page. Also sweeps:
|
||||
any followed channel not date-enriched recently gets queued for the
|
||||
background enricher (so existing follows get picked up, not just new ones)."""
|
||||
from . import get_video_db
|
||||
try:
|
||||
db = get_video_db()
|
||||
channels = db.list_watchlist_channels()
|
||||
playlists = db.list_watchlist_playlists()
|
||||
try:
|
||||
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||
enr = get_youtube_date_enricher()
|
||||
for c in channels:
|
||||
if not db.channel_dates_enriched_recently(c["youtube_id"]):
|
||||
enr.enqueue(c["youtube_id"], c.get("title"))
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify({"success": True, "channels": channels, "playlists": playlists,
|
||||
"counts": db.youtube_wishlist_counts()})
|
||||
except Exception:
|
||||
logger.exception("youtube channels list failed")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/youtube/channel/<channel_id>/settings", methods=["GET"])
|
||||
def video_youtube_channel_settings(channel_id):
|
||||
"""Per-channel overrides (custom show-name + quality), plus the global quality
|
||||
default so the modal can show 'using default' until the user overrides."""
|
||||
from . import get_video_db
|
||||
from core.video.youtube_quality import default_profile, load as load_quality
|
||||
db = get_video_db()
|
||||
return jsonify({"success": True, "settings": db.get_channel_settings(channel_id) or {},
|
||||
"default_quality": load_quality(db) or default_profile()})
|
||||
|
||||
@bp.route("/youtube/channel/<channel_id>/settings", methods=["POST"])
|
||||
def video_youtube_channel_settings_save(channel_id):
|
||||
"""Save per-channel overrides. A blank custom_name / absent quality clears that
|
||||
override (falls back to the channel title / global quality). Body:
|
||||
{custom_name?, quality?{...}}."""
|
||||
from . import get_video_db
|
||||
from core.video.youtube_quality import normalize as normalize_quality
|
||||
db = get_video_db()
|
||||
body = request.get_json(silent=True) or {}
|
||||
out = {}
|
||||
name = str(body.get("custom_name") or "").strip()
|
||||
if name:
|
||||
out["custom_name"] = name
|
||||
if body.get("quality"): # falsy/absent → no override (use the global default)
|
||||
out["quality"] = normalize_quality(body.get("quality"))
|
||||
keep = str(body.get("retention") or "").strip()
|
||||
if keep and keep != "all": # blank/'all' → no policy (keep everything)
|
||||
out["retention"] = keep
|
||||
db.set_channel_settings(channel_id, out)
|
||||
return jsonify({"success": True, "settings": out})
|
||||
|
||||
@bp.route("/youtube/wishlist", methods=["GET"])
|
||||
def video_youtube_wishlist():
|
||||
"""Wished videos grouped by channel (channel = group, videos = feed)."""
|
||||
from . import get_video_db
|
||||
try:
|
||||
db = get_video_db()
|
||||
res = db.query_youtube_wishlist(
|
||||
search=request.args.get("search", ""), sort=request.args.get("sort", "added"),
|
||||
page=request.args.get("page", 1), limit=request.args.get("limit", 60))
|
||||
return jsonify({"success": True, "counts": db.youtube_wishlist_counts(), **res})
|
||||
except Exception:
|
||||
logger.exception("youtube wishlist list failed")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/youtube/channel/<channel_id>", methods=["GET"])
|
||||
def video_youtube_channel_detail(channel_id):
|
||||
"""Full channel detail for the in-app channel page: meta + a deeper page of
|
||||
uploads, the follow state, and per-video wished flags. Resolves live."""
|
||||
from . import get_video_db
|
||||
from core.video import youtube as yt
|
||||
try:
|
||||
limit = int(request.args.get("limit") or 60)
|
||||
except (TypeError, ValueError):
|
||||
limit = 60
|
||||
try:
|
||||
db = get_video_db()
|
||||
cid = str(channel_id).strip()
|
||||
following = bool(db.channel_watch_state([cid]))
|
||||
# Opening a channel page → (re)remember it in the background (followed or
|
||||
# not — you're looking at it). The enricher caches list + meta + dates.
|
||||
try:
|
||||
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||
get_youtube_date_enricher().enqueue(cid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# CACHE-FIRST: a remembered channel renders instantly (no yt-dlp). The
|
||||
# page's background re-stream + the enricher keep it fresh.
|
||||
meta = db.get_channel_meta(cid)
|
||||
cached_vids = db.get_channel_videos(cid)
|
||||
from_cache = bool(meta and cached_vids)
|
||||
if from_cache:
|
||||
channel = {"youtube_id": cid, "title": meta.get("title"), "handle": meta.get("handle"),
|
||||
"description": meta.get("description"), "avatar_url": meta.get("avatar_url"),
|
||||
"banner_url": meta.get("banner_url"), "subscriber_count": meta.get("subscriber_count"),
|
||||
"view_count": meta.get("view_count"), "tags": meta.get("tags") or [],
|
||||
"videos": cached_vids}
|
||||
else:
|
||||
# MISS → fetch live (yt-dlp header + recent uploads) and remember it.
|
||||
channel = yt.resolve_channel("https://www.youtube.com/channel/" + cid,
|
||||
limit=max(1, min(90, limit)))
|
||||
if not channel or not channel.get("youtube_id"):
|
||||
return jsonify({"success": False, "error": "Channel not found"}), 404
|
||||
cid = channel["youtube_id"]
|
||||
db.cache_channel_meta(cid, channel)
|
||||
db.cache_channel_videos(cid, channel.get("videos") or [])
|
||||
if channel.get("avatar_url"):
|
||||
try:
|
||||
db.set_wishlist_channel_poster(cid, channel["avatar_url"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
vids = channel.get("videos") or []
|
||||
ids = [v.get("youtube_id") for v in vids]
|
||||
wished = db.youtube_video_wish_state(ids)
|
||||
# Dates → year-seasons. Cached list already carries them; only pull a
|
||||
# fresh RSS (recent ~15) on a live MISS so the cache-hit stays instant.
|
||||
dates = db.get_video_dates(ids)
|
||||
if not from_cache:
|
||||
try:
|
||||
dates.update(yt.channel_recent_dates(cid) or {})
|
||||
except Exception:
|
||||
pass
|
||||
for v in vids:
|
||||
v["wished"] = v.get("youtube_id") in wished
|
||||
if not v.get("published_at") and dates.get(v.get("youtube_id")):
|
||||
v["published_at"] = dates[v["youtube_id"]]
|
||||
try:
|
||||
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")}
|
||||
for v in vids if v.get("published_at")])
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify({"success": True, "kind": "channel", "source": "youtube",
|
||||
"channel": channel, "following": following, "from_cache": from_cache})
|
||||
except Exception:
|
||||
logger.exception("youtube channel detail failed for %r", channel_id)
|
||||
return jsonify({"success": False, "error": "Could not load channel"}), 500
|
||||
|
||||
@bp.route("/youtube/channel/<channel_id>/videos", methods=["POST"])
|
||||
def video_youtube_channel_videos(channel_id):
|
||||
"""ONE InnerTube page of a channel's videos — the channel page streams the
|
||||
WHOLE catalog by re-POSTing with the continuation token from each response
|
||||
(each page fetched once; no yt-dlp re-scan). POST (not GET) keeps the giant
|
||||
continuation token out of the URL/access logs; the frontend paces the calls,
|
||||
so each is fast and never trips the slow-request warning. Returns the videos
|
||||
(dates refined from cache) + next ``continuation`` (null = no more)."""
|
||||
from . import get_video_db
|
||||
from core.video import youtube as yt
|
||||
body = request.get_json(silent=True) or {}
|
||||
cont = (body.get("continuation") or "").strip() or None
|
||||
try:
|
||||
db = get_video_db()
|
||||
page = yt.innertube_channel_videos_page(channel_id, continuation=cont)
|
||||
videos, token = page.get("videos") or [], page.get("continuation")
|
||||
ids = [v.get("youtube_id") for v in videos if v.get("youtube_id")]
|
||||
cached = db.get_video_dates(ids)
|
||||
wished = db.youtube_video_wish_state(ids)
|
||||
for v in videos:
|
||||
vid = v.get("youtube_id")
|
||||
if cached.get(vid): # a cached (possibly exact) date wins
|
||||
v["published_at"] = cached[vid]
|
||||
v["wished"] = vid in wished
|
||||
try:
|
||||
db.cache_channel_videos(channel_id, videos) # remember the list
|
||||
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")}
|
||||
for v in videos if v.get("youtube_id") and v.get("published_at")])
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify({"success": True, "videos": videos, "continuation": token})
|
||||
except Exception:
|
||||
logger.exception("youtube channel videos batch failed for %r", channel_id)
|
||||
return jsonify({"success": False, "videos": [], "continuation": None}), 200
|
||||
|
||||
@bp.route("/youtube/video/<video_id>", methods=["GET"])
|
||||
def video_youtube_video_detail(video_id):
|
||||
"""Full metadata for one video (description, views, likes, duration, tags) —
|
||||
fetched lazily when a video is selected. Persists the description onto its
|
||||
wishlist row so re-opening is instant."""
|
||||
from . import get_video_db
|
||||
from core.video import youtube as yt
|
||||
try:
|
||||
v = yt.video_detail(video_id)
|
||||
if not v or not v.get("youtube_id"):
|
||||
return jsonify({"success": False, "error": "Video not found"}), 404
|
||||
db = get_video_db()
|
||||
# DeArrow crowd-sourced better title (if enriched) — surfaced as an
|
||||
# alternative on the detail page.
|
||||
try:
|
||||
v["dearrow_title"] = db.youtube_video_dearrow_title(v["youtube_id"])
|
||||
except Exception:
|
||||
v["dearrow_title"] = None
|
||||
if v.get("description"):
|
||||
try:
|
||||
db.set_wishlist_video_overview(v["youtube_id"], v["description"])
|
||||
except Exception:
|
||||
pass # persistence is best-effort; the detail still returns
|
||||
if v.get("published_at"): # learned a real date → cache it for year-seasons
|
||||
try:
|
||||
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v["published_at"]}])
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify({"success": True, "video": v})
|
||||
except Exception:
|
||||
logger.exception("youtube video detail failed for %r", video_id)
|
||||
return jsonify({"success": False, "error": "Could not load video"}), 500
|
||||
|
||||
@bp.route("/youtube/search", methods=["GET"])
|
||||
def video_youtube_search():
|
||||
"""Channel search results for the search page (alongside TMDB), each with a
|
||||
followed flag so the card can hydrate. Best-effort."""
|
||||
from . import get_video_db
|
||||
from core.video import youtube as yt
|
||||
q = (request.args.get("q") or "").strip()
|
||||
if not q:
|
||||
return jsonify({"success": True, "channels": []})
|
||||
try:
|
||||
chans = yt.search_channels(q)
|
||||
following = get_video_db().channel_watch_state([c["youtube_id"] for c in chans])
|
||||
for c in chans:
|
||||
c["following"] = c["youtube_id"] in following
|
||||
return jsonify({"success": True, "channels": chans})
|
||||
except Exception:
|
||||
logger.exception("youtube search failed for %r", q)
|
||||
return jsonify({"success": False, "channels": []})
|
||||
|
||||
@bp.route("/youtube/playlists/<channel_id>", methods=["GET"])
|
||||
def video_youtube_playlists(channel_id):
|
||||
"""The channel's playlists (collapsible sections on the channel page), each
|
||||
flagged ``following`` so its Add-to-watchlist button hydrates."""
|
||||
from . import get_video_db
|
||||
from core.video import youtube as yt
|
||||
try:
|
||||
pls = yt.channel_playlists(channel_id)
|
||||
followed = get_video_db().playlist_watch_state([p.get("playlist_id") for p in pls])
|
||||
for p in pls:
|
||||
p["following"] = p.get("playlist_id") in followed
|
||||
return jsonify({"success": True, "playlists": pls})
|
||||
except Exception:
|
||||
logger.exception("youtube playlists failed for %r", channel_id)
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/youtube/playlist/<playlist_id>", methods=["GET"])
|
||||
def video_youtube_playlist(playlist_id):
|
||||
"""A playlist's videos + metadata + follow state. Serves BOTH the channel-page
|
||||
playlist expansion (reads ``videos``) and the standalone playlist detail view
|
||||
(reads ``playlist`` + ``following``). Curator-ordered — a partial set, NOT
|
||||
grouped by year. Per-video wished + cached-date flags hydrate the toggles."""
|
||||
from . import get_video_db
|
||||
from core.video import youtube as yt
|
||||
try:
|
||||
limit = int(request.args.get("limit") or 200)
|
||||
except (TypeError, ValueError):
|
||||
limit = 200
|
||||
try:
|
||||
db = get_video_db()
|
||||
# Pull the WHOLE playlist. With cookies configured (youtube.cookies_browser,
|
||||
# like ytdl-sub) yt-dlp pages the full list; anonymous, YouTube throttles
|
||||
# us to ~100, so the InnerTube catalog (~200 + the true header count) is the
|
||||
# better source. Use whichever got more, with the true total.
|
||||
pl = yt.resolve_playlist("https://www.youtube.com/playlist?list=" + playlist_id, limit=5000)
|
||||
if not pl:
|
||||
return jsonify({"success": False, "error": "Playlist not found"}), 404
|
||||
try:
|
||||
cat = yt.innertube_playlist_catalog(playlist_id)
|
||||
if len(cat.get("videos") or []) > len(pl.get("videos") or []):
|
||||
pl["videos"] = cat["videos"]
|
||||
total = max(pl.get("video_count") or 0, cat.get("total") or 0)
|
||||
if total:
|
||||
pl["video_count"] = total
|
||||
except Exception:
|
||||
pass
|
||||
vids = pl.get("videos") or []
|
||||
ids = [v.get("youtube_id") for v in vids]
|
||||
wished = db.youtube_video_wish_state(ids)
|
||||
dates = db.get_video_dates(ids)
|
||||
for v in vids:
|
||||
v["wished"] = v.get("youtube_id") in wished
|
||||
if not v.get("published_at") and dates.get(v.get("youtube_id")):
|
||||
v["published_at"] = dates[v["youtube_id"]]
|
||||
try:
|
||||
db.cache_channel_videos(playlist_id, vids) # remember the count for the watchlist card
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify({"success": True, "videos": vids, "playlist": pl,
|
||||
"following": bool(db.playlist_watch_state([pl["playlist_id"]])),
|
||||
"kind": "playlist", "source": "youtube"})
|
||||
except Exception:
|
||||
logger.exception("youtube playlist failed for %r", playlist_id)
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/youtube/wishlist/add", methods=["POST"])
|
||||
def video_youtube_wishlist_add():
|
||||
"""Wish specific videos (per-video add from the channel page). Body:
|
||||
{channel: {youtube_id, title, avatar_url?}, videos: [{youtube_id, title, …}]}."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
channel = body.get("channel") or {}
|
||||
videos = body.get("videos") or []
|
||||
if not channel.get("youtube_id") or not videos:
|
||||
return jsonify({"success": False, "error": "channel and videos required"}), 400
|
||||
try:
|
||||
db = get_video_db()
|
||||
n = db.add_videos_to_wishlist(channel, videos, server_source=_server())
|
||||
return jsonify({"success": n > 0, "added": n, "counts": db.youtube_wishlist_counts()})
|
||||
except Exception:
|
||||
logger.exception("youtube wishlist add failed")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
||||
@bp.route("/youtube/wishlist/remove", methods=["POST"])
|
||||
def video_youtube_wishlist_remove():
|
||||
"""Remove wished videos. Body: {scope: 'channel'|'video', source_id}."""
|
||||
from . import get_video_db
|
||||
body = request.get_json(silent=True) or {}
|
||||
scope = body.get("scope")
|
||||
source_id = (body.get("source_id") or "").strip()
|
||||
if scope not in ("channel", "video") or not source_id:
|
||||
return jsonify({"success": False, "error": "scope and source_id are required"}), 400
|
||||
try:
|
||||
db = get_video_db()
|
||||
removed = db.remove_youtube_from_wishlist(scope, source_id)
|
||||
return jsonify({"success": True, "removed": removed, "counts": db.youtube_wishlist_counts()})
|
||||
except Exception:
|
||||
logger.exception("youtube wishlist remove failed")
|
||||
return jsonify({"success": False, "error": "Failed"}), 500
|
||||
|
|
@ -44,8 +44,7 @@
|
|||
},
|
||||
"metadata_enhancement": {
|
||||
"enabled": true,
|
||||
"embed_album_art": true,
|
||||
"single_to_album": false
|
||||
"embed_album_art": true
|
||||
},
|
||||
"file_organization": {
|
||||
"enabled": true,
|
||||
|
|
|
|||
|
|
@ -662,12 +662,7 @@ class ConfigManager:
|
|||
# source whose art is smaller is skipped so the next source is
|
||||
# tried — stops a low-res Cover Art Archive upload from winning.
|
||||
# 0 disables the size gate.
|
||||
"min_art_size": 1000,
|
||||
# When a track matches a SINGLE release, look up the parent ALBUM
|
||||
# that contains it and tag it as that album, so it groups with its
|
||||
# album-mates and gets the album cover (not the single's). Off by
|
||||
# default — it's an extra per-import metadata lookup.
|
||||
"single_to_album": False
|
||||
"min_art_size": 1000
|
||||
},
|
||||
"musicbrainz": {
|
||||
"embed_tags": True
|
||||
|
|
@ -705,12 +700,6 @@ class ConfigManager:
|
|||
},
|
||||
"import": {
|
||||
"staging_path": "./Staging",
|
||||
# Master toggle for quality-filtering on import. On by default:
|
||||
# downloaded files that don't meet the quality profile are
|
||||
# quarantined instead of imported (same gate the download
|
||||
# pipeline uses). Off → import everything regardless of quality;
|
||||
# the library Quality Upgrade Scanner still flags them.
|
||||
"quality_filter_enabled": True,
|
||||
"replace_lower_quality": False,
|
||||
# Use the top Staging folder as the artist (Artist/Album layouts,
|
||||
# mixtapes). On by default to preserve the long-standing import
|
||||
|
|
|
|||
|
|
@ -32,8 +32,6 @@ from config.settings import config_manager
|
|||
from core.amazon_client import AmazonClient, AmazonClientError
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
from core.quality.model import AudioQuality
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("amazon_download_client")
|
||||
|
|
@ -78,12 +76,9 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get("soulseek.download_path", "./downloads")
|
||||
self.download_path = Path(download_path)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._quality = quality_tier_for_source("amazon", default="flac")
|
||||
self._quality = config_manager.get("amazon_download.quality", "flac")
|
||||
self._allow_fallback = config_manager.get("amazon_download.allow_fallback", True)
|
||||
|
||||
self._client = AmazonClient(preferred_codec=self._quality)
|
||||
|
|
@ -138,17 +133,11 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
album_map: Dict[str, AlbumResult] = {}
|
||||
album_order: List[str] = []
|
||||
preferred = self._client.preferred_codec
|
||||
# Search results only carry the codec (real sample_rate arrives at
|
||||
# stream time). Claim the format honestly — FLAC for the lossless
|
||||
# codec, lossy otherwise — so audio_quality derives a real format
|
||||
# instead of the display label ("Lossless"), and the post-download
|
||||
# probe pins the actual sample_rate/bit_depth.
|
||||
amazon_q = AudioQuality(format='flac' if _codec_key(preferred) == 'flac' else 'aac')
|
||||
|
||||
for item in items:
|
||||
quality = _quality_label(preferred)
|
||||
if item.is_track:
|
||||
tr = TrackResult(
|
||||
track_results.append(TrackResult(
|
||||
username="amazon",
|
||||
filename=f"{item.asin}||{item.artist_name} - {item.title}",
|
||||
size=0,
|
||||
|
|
@ -166,9 +155,7 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
"album_asin": item.album_asin,
|
||||
"isrc": item.isrc,
|
||||
},
|
||||
)
|
||||
tr.set_quality(amazon_q)
|
||||
track_results.append(tr)
|
||||
))
|
||||
elif item.is_album:
|
||||
album_asin = item.album_asin or item.asin
|
||||
if album_asin not in album_map:
|
||||
|
|
@ -186,7 +173,6 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
title=item.title,
|
||||
album=item.album_name,
|
||||
)
|
||||
placeholder.set_quality(amazon_q)
|
||||
album_map[album_asin] = AlbumResult(
|
||||
username="amazon",
|
||||
album_path=album_asin,
|
||||
|
|
|
|||
|
|
@ -660,13 +660,8 @@ class AutoImportWorker:
|
|||
auto_process = self._config_manager.get('auto_import.auto_process', True)
|
||||
|
||||
try:
|
||||
# Phase 3: Identify.
|
||||
# Re-identify (#889): if the user designated this exact file's release in
|
||||
# the Re-identify modal, a hint short-circuits the guessing — we match
|
||||
# straight against the chosen album. No hint → byte-identical to before.
|
||||
rematch_hint, identification = self._resolve_rematch_hint(candidate)
|
||||
if identification is None:
|
||||
identification = self._identify_folder(candidate)
|
||||
# Phase 3: Identify
|
||||
identification = self._identify_folder(candidate)
|
||||
if not identification:
|
||||
self._record_result(candidate, 'needs_identification', 0.0,
|
||||
error_message='Could not identify album from tags, folder name, or fingerprint')
|
||||
|
|
@ -695,10 +690,7 @@ class AutoImportWorker:
|
|||
high_conf_matches = [m for m in match_result.get('matches', []) if m['confidence'] >= 0.8]
|
||||
has_strong_individual_matches = len(high_conf_matches) > 0
|
||||
|
||||
# A re-identify is an explicit user choice — let it auto-process like a
|
||||
# strong match (still gated on the global auto_process preference).
|
||||
if (confidence >= threshold or has_strong_individual_matches
|
||||
or rematch_hint is not None) and auto_process:
|
||||
if (confidence >= threshold or has_strong_individual_matches) and auto_process:
|
||||
# Phase 5: Auto-process — insert an in-progress row
|
||||
# so the UI sees the import the moment it starts,
|
||||
# then update it with the final status when done.
|
||||
|
|
@ -717,13 +709,6 @@ class AutoImportWorker:
|
|||
confidence = max(confidence, effective_conf)
|
||||
if success:
|
||||
self._bump_stat('auto_processed')
|
||||
# Re-identify (#889): only NOW that the new home exists do we
|
||||
# consume the hint and (if replace was chosen) delete the old
|
||||
# row + file — so a failed import never loses the original. Pass
|
||||
# the landing paths so we never delete a file the re-import landed
|
||||
# at the SAME place (picking the release it's already in).
|
||||
if rematch_hint is not None:
|
||||
self._finalize_rematch_hint(rematch_hint, getattr(candidate, '_reid_final_paths', None))
|
||||
else:
|
||||
self._bump_stat('failed')
|
||||
|
||||
|
|
@ -1018,75 +1003,6 @@ class AutoImportWorker:
|
|||
except Exception:
|
||||
return False
|
||||
|
||||
# ── Re-identify hints (#889) ──
|
||||
|
||||
def _resolve_rematch_hint(self, candidate: 'FolderCandidate'):
|
||||
"""If this staged file carries a user-designated re-identify hint, return
|
||||
``(hint, identification)`` so matching skips the guessing tiers; otherwise
|
||||
``(None, None)`` and the caller falls back to normal identification.
|
||||
|
||||
Fail-safe: ANY error (no table, DB hiccup) returns ``(None, None)`` so a
|
||||
re-identify problem can never break ordinary auto-import. Only single-file
|
||||
candidates are eligible — a re-identify always stages exactly one track."""
|
||||
try:
|
||||
files = candidate.audio_files or []
|
||||
if len(files) != 1:
|
||||
return None, None
|
||||
from core.imports.rematch_hints import (
|
||||
build_identification_from_hint,
|
||||
find_hint_for_file,
|
||||
quick_file_signature,
|
||||
)
|
||||
file_path = files[0]
|
||||
sig = quick_file_signature(file_path)
|
||||
conn = self.database._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
hint = find_hint_for_file(cursor, file_path, sig)
|
||||
finally:
|
||||
conn.close()
|
||||
if hint is None:
|
||||
return None, None
|
||||
logger.info("[Auto-Import] Re-identify hint for %s → %s '%s' (%s)",
|
||||
candidate.name, hint.album_type or 'release',
|
||||
hint.album_name or '?', hint.source)
|
||||
return hint, build_identification_from_hint(hint)
|
||||
except Exception as e:
|
||||
logger.debug("[Auto-Import] rematch-hint lookup skipped: %s", e)
|
||||
return None, None
|
||||
|
||||
def _finalize_rematch_hint(self, hint, new_paths=None) -> None:
|
||||
"""Post-success: delete the replaced library row + file (if the user chose
|
||||
replace) and consume the hint so it's single-use. ``new_paths`` are where the
|
||||
re-import landed — passed through so the same-home guard never deletes a file
|
||||
the import wrote at the old location. Best-effort — a cleanup failure is
|
||||
logged, never raised, since the re-import already succeeded."""
|
||||
try:
|
||||
from core.imports.rematch_hints import consume_hint, delete_replaced_track
|
||||
|
||||
def _resolve_old(stored):
|
||||
# The old row's path is a STORED path (Docker/media-server view) — map
|
||||
# it to a file this process can actually unlink, same as everywhere else.
|
||||
try:
|
||||
from core.library.path_resolver import resolve_library_file_path
|
||||
return resolve_library_file_path(stored, config_manager=getattr(self, '_config_manager', None))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
conn = self.database._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
removed = delete_replaced_track(cursor, hint.replace_track_id,
|
||||
resolve_fn=_resolve_old, new_paths=new_paths)
|
||||
consume_hint(cursor, hint.id)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
if removed:
|
||||
logger.info("[Auto-Import] Re-identify replaced old track — removed %s", removed)
|
||||
except Exception as e:
|
||||
logger.warning("[Auto-Import] rematch-hint finalize failed (import still OK): %s", e)
|
||||
|
||||
# ── Identification ──
|
||||
|
||||
def _identify_folder(self, candidate: FolderCandidate) -> Optional[Dict]:
|
||||
|
|
@ -1515,11 +1431,8 @@ class AutoImportWorker:
|
|||
|
||||
def _match_tracks(self, candidate: FolderCandidate, identification: Dict) -> Optional[Dict]:
|
||||
"""Match staging files to the identified album's tracklist."""
|
||||
# Singles: no album tracklist to match against — the file IS the match.
|
||||
# force_album_match (set by a re-identify hint) overrides this: even a lone
|
||||
# staged file is matched INTO the chosen album, so it inherits the album's
|
||||
# year / track number / art instead of the bare singles stub (#889).
|
||||
if not identification.get('force_album_match') and (candidate.is_single or identification.get('is_single')):
|
||||
# Singles: no album tracklist to match against — the file IS the match
|
||||
if candidate.is_single or identification.get('is_single'):
|
||||
conf = identification.get('identification_confidence', 0.7)
|
||||
track_data = {
|
||||
'name': identification.get('track_name', identification.get('album_name', '')),
|
||||
|
|
@ -1687,7 +1600,6 @@ class AutoImportWorker:
|
|||
|
||||
processed = 0
|
||||
errors = []
|
||||
reid_final_paths = [] # #889: where the pipeline landed each file (same-home guard)
|
||||
all_matches = list(match_result.get('matches', []))
|
||||
|
||||
# Album total duration — sum of every matched track's duration.
|
||||
|
|
@ -1868,11 +1780,6 @@ class AutoImportWorker:
|
|||
|
||||
self._process_callback(context_key, context, file_path)
|
||||
processed += 1
|
||||
# Capture where the pipeline actually landed the file (#889 same-home
|
||||
# guard) — the pipeline writes it back into the mutable context.
|
||||
_landed = context.get('_final_processed_path')
|
||||
if _landed:
|
||||
reid_final_paths.append(_landed)
|
||||
logger.info(f"[Auto-Import] Processed: {track_number}. {track_name}")
|
||||
|
||||
except Exception as e:
|
||||
|
|
@ -1896,13 +1803,6 @@ class AutoImportWorker:
|
|||
except Exception as e:
|
||||
logger.debug("automation emit failed: %s", e)
|
||||
|
||||
# Stash landing paths on the candidate so _finalize_rematch_hint can avoid
|
||||
# deleting a file the re-import landed at the SAME place (#889).
|
||||
try:
|
||||
candidate._reid_final_paths = reid_final_paths
|
||||
except Exception as e:
|
||||
logger.debug("could not stash reid final paths: %s", e)
|
||||
|
||||
return processed > 0
|
||||
|
||||
# ── Database ──
|
||||
|
|
|
|||
|
|
@ -10,30 +10,37 @@ Three top-level lists:
|
|||
- `ACTIONS` — DO blocks: process_wishlist, scan_library, etc.
|
||||
- `NOTIFICATIONS` — THEN blocks: discord/pushbullet/telegram/webhook,
|
||||
plus fire_signal and run_script then-actions.
|
||||
|
||||
Each block carries an optional ``scope`` tag so the SAME definitions can
|
||||
feed both the music and the (isolated) video automation builders:
|
||||
- ``"both"`` — generic; shown on both sides (schedule, notifications, …).
|
||||
- ``"video"`` — video-only; shown only on the video builder.
|
||||
- absent — treated as music-only (the default); never shown on video.
|
||||
Use :func:`blocks_for_scope` to get the filtered lists for one side.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
TRIGGERS: list[dict] = [
|
||||
{"type": "schedule", "label": "Schedule", "icon": "clock", "description": "Run on a timer interval", "available": True,
|
||||
{"type": "schedule", "label": "Schedule", "icon": "clock", "scope": "both", "description": "Run on a timer interval", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "interval", "type": "number", "label": "Every", "default": 6, "min": 1},
|
||||
{"key": "unit", "type": "select", "label": "Unit",
|
||||
"options": [{"value": "minutes", "label": "Minutes"}, {"value": "hours", "label": "Hours"}, {"value": "days", "label": "Days"}],
|
||||
"default": "hours"}
|
||||
]},
|
||||
{"type": "daily_time", "label": "Daily Time", "icon": "clock", "description": "Run every day at a specific time", "available": True,
|
||||
{"type": "daily_time", "label": "Daily Time", "icon": "clock", "scope": "both", "description": "Run every day at a specific time", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "time", "type": "time", "label": "At", "default": "03:00"}
|
||||
]},
|
||||
{"type": "weekly_time", "label": "Weekly Schedule", "icon": "calendar", "description": "Run on specific days of the week at a set time", "available": True,
|
||||
{"type": "weekly_time", "label": "Weekly Schedule", "icon": "calendar", "scope": "both", "description": "Run on specific days of the week at a set time", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "time", "type": "time", "label": "At", "default": "03:00"},
|
||||
{"key": "days", "type": "multi_select", "label": "Days",
|
||||
"options": [{"value": "mon", "label": "Mon"}, {"value": "tue", "label": "Tue"}, {"value": "wed", "label": "Wed"},
|
||||
{"value": "thu", "label": "Thu"}, {"value": "fri", "label": "Fri"}, {"value": "sat", "label": "Sat"}, {"value": "sun", "label": "Sun"}]}
|
||||
]},
|
||||
{"type": "app_started", "label": "App Started", "icon": "power", "description": "When SoulSync starts up", "available": True},
|
||||
{"type": "app_started", "label": "App Started", "icon": "power", "scope": "both", "description": "When SoulSync starts up", "available": True},
|
||||
{"type": "track_downloaded", "label": "Track Downloaded", "icon": "download", "description": "When a track finishes downloading", "available": True,
|
||||
"has_conditions": True,
|
||||
"condition_fields": ["artist", "title", "album", "quality"],
|
||||
|
|
@ -106,16 +113,24 @@ TRIGGERS: list[dict] = [
|
|||
"description": "When duplicate cleaner finishes", "available": True,
|
||||
"variables": ["files_scanned", "duplicates_found", "space_freed"]},
|
||||
# Signal trigger
|
||||
{"type": "signal_received", "label": "Signal Received", "icon": "zap",
|
||||
{"type": "signal_received", "label": "Signal Received", "icon": "zap", "scope": "both",
|
||||
"description": "When another automation fires a named signal", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "signal_name", "type": "signal_input", "label": "Signal Name"}
|
||||
],
|
||||
"variables": ["signal_name"]},
|
||||
# Webhook trigger
|
||||
{"type": "webhook_received", "label": "Webhook Received", "icon": "globe",
|
||||
{"type": "webhook_received", "label": "Webhook Received", "icon": "globe", "scope": "both",
|
||||
"description": "When an external API request is received (POST /api/v1/request)", "available": True,
|
||||
"variables": ["query", "request_id", "source"]},
|
||||
|
||||
# ── Video side (scope='video') — the post-download scan chain triggers ──
|
||||
{"type": "video_batch_complete", "label": "Video Download Batch Done", "icon": "check-circle", "scope": "video",
|
||||
"description": "When a batch of video downloads finishes", "available": True,
|
||||
"variables": ["completed"]},
|
||||
{"type": "video_library_scan_completed", "label": "Video Library Scan Done", "icon": "hard-drive", "scope": "video",
|
||||
"description": "When the media server finishes rescanning your video sections", "available": True,
|
||||
"variables": ["server"]},
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -155,7 +170,7 @@ ACTIONS: list[dict] = [
|
|||
{"key": "refresh_first", "type": "checkbox", "label": "Refresh playlists before sync (regenerate snapshots)", "default": False},
|
||||
{"key": "skip_wishlist", "type": "checkbox", "label": "Skip wishlist processing", "default": False},
|
||||
]},
|
||||
{"type": "notify_only", "label": "Notify Only", "icon": "bell", "description": "No action — just send notification", "available": True},
|
||||
{"type": "notify_only", "label": "Notify Only", "icon": "bell", "scope": "both", "description": "No action — just send notification", "available": True},
|
||||
# Phase 3 actions
|
||||
{"type": "start_database_update", "label": "Update Database", "icon": "database",
|
||||
"description": "Trigger library database refresh", "available": True,
|
||||
|
|
@ -184,7 +199,7 @@ ACTIONS: list[dict] = [
|
|||
"description": "Clear quarantine, download queue, import folder, and search history in one sweep", "available": True},
|
||||
{"type": "deep_scan_library", "label": "Deep Scan Library", "icon": "search",
|
||||
"description": "Full library comparison without losing enrichment data", "available": True},
|
||||
{"type": "run_script", "label": "Run Script", "icon": "terminal",
|
||||
{"type": "run_script", "label": "Run Script", "icon": "terminal", "scope": "both",
|
||||
"description": "Execute a script from the scripts folder", "available": True},
|
||||
{"type": "search_and_download", "label": "Search & Download", "icon": "download",
|
||||
"description": "Search for a track and download the best match", "available": True,
|
||||
|
|
@ -192,28 +207,154 @@ ACTIONS: list[dict] = [
|
|||
{"key": "query", "type": "text", "label": "Search Query",
|
||||
"placeholder": "Artist - Track (leave empty to use trigger's query)"}
|
||||
]},
|
||||
|
||||
# ── Video side (isolated app, shared engine) ──────────────────────────
|
||||
# Tagged scope='video' so they appear ONLY on the video automation
|
||||
# builder, never the music one. Their handlers bridge into core.video.
|
||||
{"type": "video_scan_library", "label": "Scan Video Library", "icon": "refresh", "scope": "video",
|
||||
"description": "Tell the media server to rescan your selected movie/TV sections, then read what it found into SoulSync", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "mode", "type": "select", "label": "Mode",
|
||||
"options": [{"value": "full", "label": "Full (add + refresh)"},
|
||||
{"value": "incremental", "label": "Incremental (recent only)"},
|
||||
{"value": "deep", "label": "Deep (also remove missing)"}],
|
||||
"default": "full"},
|
||||
{"key": "media_type", "type": "select", "label": "Library",
|
||||
"options": [{"value": "all", "label": "Movies + TV"},
|
||||
{"value": "movie", "label": "Movies only"},
|
||||
{"value": "show", "label": "TV only"}],
|
||||
"default": "all"}
|
||||
]},
|
||||
# Post-download chain actions (two stages, like music's scan_library +
|
||||
# start_database_update). Stage 1 nudges the server; stage 2 reads it in.
|
||||
{"type": "video_scan_server", "label": "Scan Video Server", "icon": "refresh", "scope": "video",
|
||||
"description": "Get the server to index new downloads (skips the scan if it already has them), waits until it finishes, then fires 'Video Library Scan Done'", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "media_type", "type": "select", "label": "Library",
|
||||
"options": [{"value": "all", "label": "Movies + TV"},
|
||||
{"value": "movie", "label": "Movies only"},
|
||||
{"value": "show", "label": "TV only"}],
|
||||
"default": "all"},
|
||||
{"key": "skip_if_present", "type": "checkbox", "label": "Skip the scan if the server already has the download", "default": True},
|
||||
{"key": "probe_grace_minutes", "type": "number", "label": "Give the server's auto-scan this long to ingest first (min)", "default": 2, "min": 0},
|
||||
{"key": "max_wait_minutes", "type": "number", "label": "Max wait for scan (min)", "default": 60, "min": 1},
|
||||
{"key": "debounce_seconds", "type": "number", "label": "Fallback wait if status unknown (sec)", "default": 120, "min": 10}
|
||||
]},
|
||||
{"type": "video_update_database", "label": "Update Video Database", "icon": "database", "scope": "video",
|
||||
"description": "Read newly-indexed media from the server into SoulSync (incremental)", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "mode", "type": "select", "label": "Mode",
|
||||
"options": [{"value": "incremental", "label": "Incremental (recent only)"},
|
||||
{"value": "full", "label": "Full (add + refresh)"}],
|
||||
"default": "incremental"},
|
||||
{"key": "media_type", "type": "select", "label": "Library",
|
||||
"options": [{"value": "all", "label": "Movies + TV"},
|
||||
{"value": "movie", "label": "Movies only"},
|
||||
{"value": "show", "label": "TV only"}],
|
||||
"default": "all"}
|
||||
]},
|
||||
{"type": "video_update_database_hourly", "label": "Update Video Database (Hourly)", "icon": "database", "scope": "video",
|
||||
"description": "The same incremental server-read on an hourly schedule, so manual library additions (which Plex auto-scans) appear within the hour instead of waiting for the weekly deep scan. Pair with a 1-hour Schedule trigger.", "available": True},
|
||||
# Per-library deep-scan presets (the system 'Auto-Deep Scan TV/Movie Library' run
|
||||
# these). Scope + deep mode are baked in by the registration wrapper, so no config
|
||||
# fields — drag one in and it just deep-scans that library.
|
||||
{"type": "video_deep_scan_tv", "label": "Deep Scan TV Library", "icon": "search", "scope": "video",
|
||||
"description": "Full reconcile of the TV library: re-read every show from the server and drop ones it no longer has (a read, NOT a Plex disk-scan; never touches movies)", "available": True},
|
||||
{"type": "video_deep_scan_movies", "label": "Deep Scan Movie Library", "icon": "search", "scope": "video",
|
||||
"description": "Full reconcile of the Movie library: re-read every movie from the server and drop ones it no longer has (a read, NOT a Plex disk-scan; never touches TV)", "available": True},
|
||||
{"type": "video_refresh_airing_schedules", "label": "Refresh Airing TV Schedules", "icon": "calendar", "scope": "video",
|
||||
"description": "Re-pull the latest TMDB episode schedules (air dates, stills) for the still-airing shows on your watchlist, so the calendar the airing automation reads is current — newly-announced or rescheduled episodes get picked up instead of being missed. Skips ended/canceled shows. Pair with a daily Schedule a couple hours before the airing run.", "available": True},
|
||||
{"type": "video_clean_youtube_episodes", "label": "Clean Old YouTube Episodes", "icon": "trash", "scope": "video",
|
||||
"description": "Delete downloaded YouTube channel episodes that fall outside each channel's keep window (set per channel via its cog → Keep: e.g. last 30 episodes / last 3–6 months). Removes the video + its sidecars but keeps the history record so it's never re-downloaded. No-op for channels left on 'keep everything' (the default). Playlists are excluded. Pair with a daily Schedule.", "available": True},
|
||||
{"type": "video_add_airing_episodes", "label": "Wishlist Today's Airings", "icon": "calendar", "scope": "video",
|
||||
"description": "Sonarr-style: add every episode airing today (for shows you follow) to the wishlist, skipping ones you already own. Also tidies the watchlist by dropping shows that have ended/been canceled.", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "prune_ended", "type": "checkbox", "label": "Also remove ended/canceled shows from the watchlist", "default": True}
|
||||
]},
|
||||
# ── Watchlist → Wishlist pipeline ────────────────────────────────────────
|
||||
# Stage 1 — scans that FILL the wishlist from what you follow.
|
||||
{"type": "video_scan_watchlist_people", "label": "Scan Watchlist People", "icon": "users", "scope": "video",
|
||||
"description": "For everyone you follow on the watchlist, wishlist every movie they acted in or directed that you don't already own — the whole back catalog plus anything upcoming (kept as 'monitored' until it's released). First run backlogs everything; later runs are fast.", "available": True},
|
||||
{"type": "video_scan_watchlist_channels", "label": "Scan Watchlist Channels", "icon": "youtube", "scope": "video",
|
||||
"description": "For every YouTube channel you follow, wishlist its new long-form uploads (Shorts excluded). Forward-looking from when you followed, plus a safety net that always keeps the last N videos. Pair with a 6-hourly Schedule trigger — channels post at all hours.", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "backfill_count", "type": "number", "label": "Always keep the last N videos from each channel", "default": 10, "min": 0}
|
||||
]},
|
||||
{"type": "video_scan_watchlist_playlists", "label": "Scan Watchlist Playlists", "icon": "list", "scope": "video",
|
||||
"description": "For every YouTube playlist you follow, wishlist its videos you don't have yet (and anything later added to it) — mirrors the whole list. Each playlist becomes its own show in the library (playlist-as-show). Shorts excluded. Pair with a schedule.", "available": True},
|
||||
# Stage 2 — processors that DRAIN the wishlist by downloading.
|
||||
{"type": "video_process_movie_wishlist", "label": "Process Movie Wishlist", "icon": "download", "scope": "video",
|
||||
"description": "Auto-grab your wished, released MOVIES from Soulseek: searches each, picks the best release per your quality profile, and downloads it (the monitor finishes + organises it). Skips unreleased ('monitored') ones and any already downloading. Needs the Movie library folder + slskd set. Pair with an hourly schedule.", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "max_concurrent", "type": "number", "label": "Max simultaneous searches", "default": 3, "min": 1}
|
||||
]},
|
||||
{"type": "video_process_episode_wishlist", "label": "Process Episode Wishlist", "icon": "download", "scope": "video",
|
||||
"description": "Auto-grab your wished EPISODES from Soulseek: searches each, picks the best release per your quality profile, and downloads it. Fed by the 'Wishlist Today's Airings' scan. Needs the TV library folder + slskd set. Pair with an hourly schedule.", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "max_concurrent", "type": "number", "label": "Max simultaneous searches", "default": 3, "min": 1}
|
||||
]},
|
||||
{"type": "video_process_youtube_wishlist", "label": "Process YouTube Wishlist", "icon": "download", "scope": "video",
|
||||
"description": "Download wished YouTube videos (yt-dlp), organised as a Plex 'TV by date' show (channel/year/date). Queues the WHOLE wishlist — the setting only limits how many download at the same time; each finished one starts the next, so it all drains. A completed download leaves the wishlist. Needs the YouTube library folder set on Settings → Downloads.", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "max_concurrent", "type": "number", "label": "Max simultaneous downloads", "default": 3, "min": 1}
|
||||
]},
|
||||
# Video twins of the music maintenance actions. Distinct action_type (the
|
||||
# system seeder keys on action_type, so a shared key would collide with the
|
||||
# music row) but the SAME shared handler — the cleanup operates on the common
|
||||
# download/search state, so behaviour stays identical. scope='video' keeps
|
||||
# them on the video builder only; the music blocks above are untouched.
|
||||
{"type": "video_clean_search_history", "label": "Clean Search History", "icon": "trash-2", "scope": "video",
|
||||
"description": "Remove old searches from Soulseek", "available": True},
|
||||
{"type": "video_clean_completed_downloads", "label": "Clean Completed Downloads", "icon": "check-square", "scope": "video",
|
||||
"description": "Clear completed downloads and empty directories", "available": True},
|
||||
{"type": "video_full_cleanup", "label": "Full Cleanup", "icon": "trash", "scope": "video",
|
||||
"description": "Clear quarantine, download queue, import folder, and search history in one sweep", "available": True},
|
||||
# Custom (NOT a shared handler): backs up video_library.db, not the music DB.
|
||||
{"type": "video_backup_database", "label": "Backup Database", "icon": "save", "scope": "video",
|
||||
"description": "Create a timestamped backup of the video library database", "available": True},
|
||||
]
|
||||
|
||||
|
||||
NOTIFICATIONS: list[dict] = [
|
||||
{"type": "discord_webhook", "label": "Discord Webhook", "icon": "message", "description": "Send a Discord notification", "available": True,
|
||||
{"type": "discord_webhook", "label": "Discord Webhook", "icon": "message", "scope": "both", "description": "Send a Discord notification", "available": True,
|
||||
"variables": ["time", "name", "run_count", "status"]},
|
||||
{"type": "pushbullet", "label": "Pushbullet", "icon": "push", "description": "Push notification to phone/desktop", "available": True,
|
||||
{"type": "pushbullet", "label": "Pushbullet", "icon": "push", "scope": "both", "description": "Push notification to phone/desktop", "available": True,
|
||||
"variables": ["time", "name", "run_count", "status"]},
|
||||
{"type": "telegram", "label": "Telegram", "icon": "message", "description": "Send a Telegram message", "available": True,
|
||||
{"type": "telegram", "label": "Telegram", "icon": "message", "scope": "both", "description": "Send a Telegram message", "available": True,
|
||||
"variables": ["time", "name", "run_count", "status"]},
|
||||
{"type": "webhook", "label": "Webhook (POST)", "icon": "globe", "description": "Send a POST request to any URL", "available": True,
|
||||
{"type": "webhook", "label": "Webhook (POST)", "icon": "globe", "scope": "both", "description": "Send a POST request to any URL", "available": True,
|
||||
"variables": ["time", "name", "run_count", "status"]},
|
||||
# Signal fire action
|
||||
{"type": "fire_signal", "label": "Fire Signal", "icon": "zap",
|
||||
{"type": "fire_signal", "label": "Fire Signal", "icon": "zap", "scope": "both",
|
||||
"description": "Fire a signal that other automations can listen for", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "signal_name", "type": "signal_input", "label": "Signal Name"}
|
||||
]},
|
||||
# Run script then-action
|
||||
{"type": "run_script", "label": "Run Script", "icon": "terminal",
|
||||
{"type": "run_script", "label": "Run Script", "icon": "terminal", "scope": "both",
|
||||
"description": "Execute a script after the action completes", "available": True,
|
||||
"config_fields": [
|
||||
{"key": "script_name", "type": "script_select", "label": "Script"}
|
||||
]},
|
||||
]
|
||||
|
||||
|
||||
def _in_scope(block: dict, scope: str) -> bool:
|
||||
"""A block belongs to ``scope`` if it's generic (``"both"``) or tagged for
|
||||
that side. Untagged blocks default to ``"music"`` (the original behaviour),
|
||||
so the video builder never picks up music-only blocks by accident."""
|
||||
s = block.get("scope", "music")
|
||||
return s == "both" or s == scope
|
||||
|
||||
|
||||
def blocks_for_scope(scope: str = "music") -> dict:
|
||||
"""Return the trigger/action/notification lists filtered to one side.
|
||||
|
||||
``scope="music"`` reproduces the pre-scope behaviour (everything except
|
||||
video-only blocks); ``scope="video"`` returns generic + video-only blocks.
|
||||
"""
|
||||
return {
|
||||
"triggers": [b for b in TRIGGERS if _in_scope(b, scope)],
|
||||
"actions": [b for b in ACTIONS if _in_scope(b, scope)],
|
||||
"notifications": [b for b in NOTIFICATIONS if _in_scope(b, scope)],
|
||||
}
|
||||
|
|
|
|||
|
|
@ -105,11 +105,10 @@ def auto_update_discovery_pool(config: Dict[str, Any], deps: AutomationDeps) ->
|
|||
_MAX_BACKUPS = 5
|
||||
|
||||
|
||||
def auto_backup_database(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]:
|
||||
"""Create a hot SQLite backup, then prune old backups so only the
|
||||
newest ``_MAX_BACKUPS`` remain."""
|
||||
automation_id = config.get('_automation_id')
|
||||
db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db')
|
||||
def _backup_db_at(db_path: str, deps: AutomationDeps, automation_id) -> Dict[str, Any]:
|
||||
"""Create a hot SQLite backup of ``db_path``, then prune old backups so only
|
||||
the newest ``_MAX_BACKUPS`` remain. Shared by the music and video backup
|
||||
actions — only the DB file differs (they can't share one backup)."""
|
||||
if not os.path.exists(db_path):
|
||||
return {'status': 'error', 'reason': 'Database file not found'}
|
||||
|
||||
|
|
@ -146,6 +145,20 @@ def auto_backup_database(config: Dict[str, Any], deps: AutomationDeps) -> Dict[s
|
|||
return {'status': 'completed', 'backup_path': backup_path, 'size_mb': str(size_mb)}
|
||||
|
||||
|
||||
def auto_backup_database(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]:
|
||||
"""Hot SQLite backup of the MUSIC database (``DATABASE_PATH``)."""
|
||||
db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db')
|
||||
return _backup_db_at(db_path, deps, config.get('_automation_id'))
|
||||
|
||||
|
||||
def auto_backup_video_database(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]:
|
||||
"""Hot SQLite backup of the VIDEO database (``VIDEO_DATABASE_PATH`` /
|
||||
video_library.db). Same logic as the music backup but a different DB file —
|
||||
the two can't share one backup, so this is a video-specific action."""
|
||||
db_path = os.environ.get('VIDEO_DATABASE_PATH', 'database/video_library.db')
|
||||
return _backup_db_at(db_path, deps, config.get('_automation_id'))
|
||||
|
||||
|
||||
# ─── refresh_beatport_cache ──────────────────────────────────────────
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from core.automation.handlers.maintenance import (
|
|||
auto_cleanup_wishlist,
|
||||
auto_update_discovery_pool,
|
||||
auto_backup_database,
|
||||
auto_backup_video_database,
|
||||
auto_refresh_beatport_cache,
|
||||
)
|
||||
from core.automation.handlers.download_cleanup import (
|
||||
|
|
@ -35,6 +36,17 @@ from core.automation.handlers.download_cleanup import (
|
|||
)
|
||||
from core.automation.handlers.run_script import auto_run_script
|
||||
from core.automation.handlers.search_and_download import auto_search_and_download
|
||||
from core.automation.handlers.video_auto_wishlist_airing import auto_video_add_airing_episodes
|
||||
from core.automation.handlers.video_refresh_airing_schedules import auto_video_refresh_airing_schedules
|
||||
from core.automation.handlers.video_clean_youtube import auto_video_clean_youtube_episodes
|
||||
from core.automation.handlers.video_scan_watchlist_people import auto_video_scan_watchlist_people
|
||||
from core.automation.handlers.video_scan_watchlist_channels import auto_video_scan_watchlist_channels
|
||||
from core.automation.handlers.video_process_youtube_wishlist import auto_video_process_youtube_wishlist
|
||||
from core.automation.handlers.video_scan_watchlist_playlists import auto_video_scan_watchlist_playlists
|
||||
from core.automation.handlers.video_process_wishlist import auto_video_process_wishlist, is_running
|
||||
from core.automation.handlers.video_scan_library import (
|
||||
auto_video_scan_library, auto_video_scan_server, auto_video_update_database,
|
||||
)
|
||||
from core.automation.handlers.progress_callbacks import (
|
||||
progress_init,
|
||||
progress_finish,
|
||||
|
|
@ -150,6 +162,25 @@ def register_all(deps: AutomationDeps) -> None:
|
|||
'clean_search_history',
|
||||
lambda config: auto_clean_search_history(config, deps),
|
||||
)
|
||||
# Video twin — same handler, distinct action_type so the system seeder
|
||||
# (keyed on action_type) creates a separate video-owned row.
|
||||
engine.register_action_handler(
|
||||
'video_clean_search_history',
|
||||
lambda config: auto_clean_search_history(config, deps),
|
||||
)
|
||||
engine.register_action_handler(
|
||||
'video_clean_completed_downloads',
|
||||
lambda config: auto_clean_completed_downloads(config, deps),
|
||||
)
|
||||
engine.register_action_handler(
|
||||
'video_full_cleanup',
|
||||
lambda config: auto_full_cleanup(config, deps),
|
||||
)
|
||||
# Video DB backup — its OWN handler (video_library.db, not the music DB).
|
||||
engine.register_action_handler(
|
||||
'video_backup_database',
|
||||
lambda config: auto_backup_video_database(config, deps),
|
||||
)
|
||||
engine.register_action_handler(
|
||||
'clean_completed_downloads',
|
||||
lambda config: auto_clean_completed_downloads(config, deps),
|
||||
|
|
@ -167,6 +198,96 @@ def register_all(deps: AutomationDeps) -> None:
|
|||
lambda config: auto_search_and_download(config, deps),
|
||||
)
|
||||
|
||||
# Video side (isolated app, shared engine). The video twins are tagged
|
||||
# owned_by='video' on their automation rows so they never surface on the
|
||||
# music automations page; the handlers bridge into core.video.
|
||||
engine.register_action_handler(
|
||||
'video_scan_library',
|
||||
lambda config: auto_video_scan_library(config, deps),
|
||||
)
|
||||
# Per-library deep scans (the video twin of music's 'Auto-Deep Scan Library',
|
||||
# split because Movies and TV are independent libraries). A deep scan READS the
|
||||
# server's current state into video.db + prunes what's gone (a full reconcile) —
|
||||
# it does NOT tell Plex to rescan its disk, so it runs through the read-only
|
||||
# update-database handler in 'deep' mode, not the nudge+read scan-library one.
|
||||
# Distinct action types so the seeder (which keys on action_type) sees two
|
||||
# separate automations; both reuse the one handler, scoped via media_type.
|
||||
engine.register_action_handler(
|
||||
'video_deep_scan_movies',
|
||||
lambda config: auto_video_update_database({**config, 'media_type': 'movie', 'mode': config.get('mode') or 'deep'}, deps),
|
||||
)
|
||||
engine.register_action_handler(
|
||||
'video_deep_scan_tv',
|
||||
lambda config: auto_video_update_database({**config, 'media_type': 'show', 'mode': config.get('mode') or 'deep'}, deps),
|
||||
)
|
||||
# Post-download chain: scan the server, then (on the scan-done event) update the DB.
|
||||
engine.register_action_handler(
|
||||
'video_scan_server',
|
||||
lambda config: auto_video_scan_server(config, deps),
|
||||
)
|
||||
engine.register_action_handler(
|
||||
'video_update_database',
|
||||
lambda config: auto_video_update_database(config, deps),
|
||||
)
|
||||
# Same incremental update, but on an hourly SCHEDULE (not just after a SoulSync scan) —
|
||||
# so manual library additions (which Plex auto-scans) show up within the hour instead of
|
||||
# waiting for the weekly deep scan. A distinct action_type because the seeder keys on it.
|
||||
engine.register_action_handler(
|
||||
'video_update_database_hourly',
|
||||
lambda config: auto_video_update_database({'mode': 'incremental', **config}, deps),
|
||||
)
|
||||
# Sonarr-style: wishlist every episode airing today (for followed shows).
|
||||
engine.register_action_handler(
|
||||
'video_add_airing_episodes',
|
||||
lambda config: auto_video_add_airing_episodes(config, deps),
|
||||
)
|
||||
# Keep the calendar honest: re-pull TMDB episode schedules for still-airing watchlist
|
||||
# shows (the airing automation above reads the LOCAL calendar, so it needs this fresh).
|
||||
engine.register_action_handler(
|
||||
'video_refresh_airing_schedules',
|
||||
lambda config: auto_video_refresh_airing_schedules(config, deps),
|
||||
)
|
||||
# YouTube retention: delete channel episodes outside each channel's keep window (opt-in
|
||||
# per channel; default keeps everything). The history row stays so it's not re-downloaded.
|
||||
engine.register_action_handler(
|
||||
'video_clean_youtube_episodes',
|
||||
lambda config: auto_video_clean_youtube_episodes(config, deps),
|
||||
)
|
||||
# ── Watchlist → Wishlist pipeline ─────────────────────────────────────────
|
||||
# Stage 1 — SCANS that fill the wishlist from what you follow.
|
||||
# People: wishlist every un-owned movie followed actors/directors made (catalog + upcoming).
|
||||
engine.register_action_handler(
|
||||
'video_scan_watchlist_people',
|
||||
lambda config: auto_video_scan_watchlist_people(config, deps),
|
||||
)
|
||||
# Channels: new long-form uploads from followed YouTube channels (forward + last-N net).
|
||||
engine.register_action_handler(
|
||||
'video_scan_watchlist_channels',
|
||||
lambda config: auto_video_scan_watchlist_channels(config, deps),
|
||||
)
|
||||
# Playlists: mirror followed YouTube playlists (whole list + new additions; playlist-as-show).
|
||||
engine.register_action_handler(
|
||||
'video_scan_watchlist_playlists',
|
||||
lambda config: auto_video_scan_watchlist_playlists(config, deps),
|
||||
)
|
||||
# Stage 2 — PROCESSORS that drain the wishlist by downloading. Movie/episode go through
|
||||
# slskd (search → pick best → grab); the guard skips an hourly tick while a drain is still
|
||||
# working. YouTube goes through yt-dlp (queue all, a few concurrent).
|
||||
engine.register_action_handler(
|
||||
'video_process_movie_wishlist',
|
||||
lambda config: auto_video_process_wishlist(config, deps, media_type='movie'),
|
||||
lambda: is_running('movie'),
|
||||
)
|
||||
engine.register_action_handler(
|
||||
'video_process_episode_wishlist',
|
||||
lambda config: auto_video_process_wishlist(config, deps, media_type='episode'),
|
||||
lambda: is_running('episode'),
|
||||
)
|
||||
engine.register_action_handler(
|
||||
'video_process_youtube_wishlist',
|
||||
lambda config: auto_video_process_youtube_wishlist(config, deps),
|
||||
)
|
||||
|
||||
# Progress + history callbacks: the engine invokes these around
|
||||
# each handler run. Lift the closures from
|
||||
# `web_server._register_automation_handlers` into thin lambdas
|
||||
|
|
|
|||
216
core/automation/handlers/video_auto_wishlist_airing.py
Normal file
216
core/automation/handlers/video_auto_wishlist_airing.py
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
"""Automation handler: ``video_add_airing_episodes`` action.
|
||||
|
||||
Sonarr-style "monitor airings": add every episode airing TODAY — for the TV shows you
|
||||
follow on the video watchlist — to the video WISHLIST, skipping ones you already own.
|
||||
Runs on a daily schedule so the day's airings queue up to be grabbed automatically.
|
||||
|
||||
Like the other video handlers it lives on the SHARED automation side (so it may import
|
||||
``core.video`` / ``api.video`` — the isolation contract only forbids the reverse) and
|
||||
owns its own progress reporting (``_manages_own_progress``). The calendar read + wishlist
|
||||
write are injected seams, so the handler is a pure function: tests pass fakes and never
|
||||
touch a DB or a media server; production lazily binds the real calls.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from core.automation.deps import AutomationDeps
|
||||
|
||||
|
||||
def _default_fetch_airing(today: str) -> List[Dict[str, Any]]:
|
||||
"""Production wiring: the calendar's episodes airing on ``today`` for followed shows."""
|
||||
from api.video import get_video_db
|
||||
from core.video.sources import resolve_video_server
|
||||
return get_video_db().calendar_upcoming(
|
||||
today, today, server_source=resolve_video_server(), watchlist_only=True)
|
||||
|
||||
|
||||
def _default_add_episodes(show_tmdb_id: Any, show_title: Any, episodes: List[Dict[str, Any]],
|
||||
library_id: Any = None, poster_url: Any = None) -> int:
|
||||
from api.video import get_video_db
|
||||
from core.video.sources import resolve_video_server
|
||||
return get_video_db().add_episodes_to_wishlist(
|
||||
show_tmdb_id, show_title, episodes, poster_url=poster_url, library_id=library_id,
|
||||
server_source=resolve_video_server())
|
||||
|
||||
|
||||
def _show_poster_url(library_id: Any) -> Optional[str]:
|
||||
"""The SAME poster a manual add stores for a library show — the show poster proxy
|
||||
path the wishlist orb renders directly. Mirrors the get-modal's
|
||||
pUrl = '/api/video/poster/show/<library_id>'. Without it the orb falls back to the
|
||||
show's initials, reading as 'not matched'."""
|
||||
return ('/api/video/poster/show/%s' % library_id) if library_id is not None else None
|
||||
|
||||
|
||||
def _default_season_meta(tmdb_id: Any, season_number: Any):
|
||||
"""The SAME TMDB season fetch the show modal uses for a manual add — so auto-added
|
||||
episodes carry identical stills / overviews / season posters, not the patchy values
|
||||
the local DB happens to hold."""
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
return get_video_enrichment_engine().tmdb_season(tmdb_id, season_number)
|
||||
|
||||
|
||||
# ── watchlist hygiene: drop shows that have ended/been canceled ───────────────
|
||||
_TERMINAL = ('ended', 'canceled', 'cancelled', 'completed')
|
||||
|
||||
|
||||
def _is_terminal_status(status) -> bool:
|
||||
"""A show that won't air again — pointless to keep on the (watch-for-new) list."""
|
||||
return str(status or '').strip().lower() in _TERMINAL
|
||||
|
||||
|
||||
def _default_fetch_follows() -> List[Dict[str, Any]]:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().followed_shows()
|
||||
|
||||
|
||||
def _default_show_status(tmdb_id: Any) -> Optional[str]:
|
||||
"""TMDB status for a follow with no local status (a tmdb-only follow). Cached by
|
||||
the engine; returns None on any hiccup so we never prune on uncertainty."""
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
d = get_video_enrichment_engine().tmdb_full_detail('show', tmdb_id) or {}
|
||||
return d.get('status')
|
||||
|
||||
|
||||
def _default_remove_show(tmdb_id: Any) -> None:
|
||||
from api.video import get_video_db
|
||||
get_video_db().remove_from_watchlist('show', tmdb_id)
|
||||
|
||||
|
||||
def prune_ended_show_follows(deps, automation_id=None, *, fetch_follows=None,
|
||||
show_status=None, remove_show=None) -> int:
|
||||
"""Remove explicitly-followed shows that are no longer airing.
|
||||
|
||||
Auto-airing LIBRARY shows are already excluded by status, so this targets explicit
|
||||
eye-button follows (which persist regardless). For follows with no local status (a
|
||||
tmdb-only follow), look the status up on TMDB. Only prunes on a DEFINITIVE terminal
|
||||
status — unknown status is left alone. Pure: all I/O injected. Returns count removed."""
|
||||
fetch_follows = fetch_follows or _default_fetch_follows
|
||||
show_status = show_status or _default_show_status
|
||||
remove_show = remove_show or _default_remove_show
|
||||
removed = 0
|
||||
for f in (fetch_follows() or []):
|
||||
tid = f.get('tmdb_id')
|
||||
if not tid:
|
||||
continue
|
||||
status = f.get('status')
|
||||
if not status: # tmdb-only follow — ask TMDB
|
||||
try:
|
||||
status = show_status(tid)
|
||||
except Exception: # noqa: BLE001 - never prune on a lookup failure
|
||||
status = None
|
||||
if status and _is_terminal_status(status):
|
||||
try:
|
||||
remove_show(tid)
|
||||
removed += 1
|
||||
deps.update_progress(
|
||||
automation_id, log_line="Removed ended show '%s' from the watchlist"
|
||||
% (f.get('title') or tid), log_type='info')
|
||||
except Exception: # noqa: BLE001, S110 - a progress-log failure must not abort pruning
|
||||
pass
|
||||
return removed
|
||||
|
||||
|
||||
def _season_lookup(season_meta, tmdb_id, season_number, cache):
|
||||
"""(season_poster_url, {episode_number: tmdb_episode}) for a show+season, fetched
|
||||
once and cached. A TMDB hiccup degrades to empty (DB values fill in)."""
|
||||
key = (tmdb_id, season_number)
|
||||
if key not in cache:
|
||||
try:
|
||||
sm = season_meta(tmdb_id, season_number) or {}
|
||||
except Exception: # noqa: BLE001 - never let a metadata fetch break the run
|
||||
sm = {}
|
||||
emap = {e.get('episode_number'): e for e in (sm.get('episodes') or []) if isinstance(e, dict)}
|
||||
cache[key] = (sm.get('poster_url'), emap)
|
||||
return cache[key]
|
||||
|
||||
|
||||
def auto_video_add_airing_episodes(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
fetch_airing: Optional[Callable[[str], List[Dict[str, Any]]]] = None,
|
||||
add_episodes: Optional[Callable[[Any, Any, List[Dict[str, Any]]], int]] = None,
|
||||
today_fn: Optional[Callable[[], str]] = None,
|
||||
season_meta: Optional[Callable[[Any, Any], Any]] = None,
|
||||
prune_follows: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||
show_status: Optional[Callable[[Any], Any]] = None,
|
||||
remove_show: Optional[Callable[[Any], None]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Add today's airing (unowned, followed-show) episodes to the video wishlist, and
|
||||
first tidy the watchlist by dropping shows that have ended / been canceled.
|
||||
|
||||
Returns ``{'status': 'completed', 'episodes_added': int, 'shows': int,
|
||||
'shows_pruned': int, ...}``."""
|
||||
fetch_airing = fetch_airing or _default_fetch_airing
|
||||
add_episodes = add_episodes or _default_add_episodes
|
||||
season_meta = season_meta or _default_season_meta
|
||||
today_fn = today_fn or (lambda: date.today().isoformat())
|
||||
automation_id = config.get('_automation_id')
|
||||
prune_ended = config.get('prune_ended', True)
|
||||
try:
|
||||
today = today_fn()
|
||||
# Watchlist hygiene first: a followed show that has since ended/been canceled
|
||||
# won't air again, so drop it (ended LIBRARY shows are already auto-excluded;
|
||||
# this catches explicit eye-button follows).
|
||||
pruned = 0
|
||||
if prune_ended:
|
||||
deps.update_progress(automation_id, phase='Tidying the watchlist…', progress=10,
|
||||
log_line='Removing shows that have ended or been canceled', log_type='info')
|
||||
pruned = prune_ended_show_follows(deps, automation_id, fetch_follows=prune_follows,
|
||||
show_status=show_status, remove_show=remove_show)
|
||||
deps.update_progress(automation_id, phase="Checking today's airings…", progress=25,
|
||||
log_line='Reading the calendar for episodes airing today', log_type='info')
|
||||
rows = fetch_airing(today) or []
|
||||
|
||||
# Group what to wish for by show: airing today, NOT already owned, with a
|
||||
# real season/episode. add_episodes_to_wishlist is idempotent, so re-runs
|
||||
# never duplicate.
|
||||
by_show: Dict[tuple, Dict[str, Any]] = {}
|
||||
season_cache: Dict[tuple, tuple] = {}
|
||||
for r in rows:
|
||||
if r.get('has_file'):
|
||||
continue
|
||||
tid = r.get('show_tmdb_id')
|
||||
sn, en = r.get('season_number'), r.get('episode_number')
|
||||
if not tid or sn is None or en is None:
|
||||
continue
|
||||
# Pull the SAME TMDB metadata a manual add gets (still + overview + season
|
||||
# poster); fall back to the calendar/DB values if TMDB is unavailable.
|
||||
poster, emap = _season_lookup(season_meta, tid, sn, season_cache)
|
||||
tm = emap.get(en) or {}
|
||||
# library_id (the show's library row id, given as show_id) is REQUIRED — the
|
||||
# wishlist resolves a show's synopsis + cast from /detail/show/<library_id>;
|
||||
# without it the show shows as un-matched with no synopsis/actors.
|
||||
grp = by_show.setdefault((tid, r.get('show_title')),
|
||||
{'library_id': r.get('show_id'), 'eps': []})
|
||||
grp['eps'].append({
|
||||
'season_number': sn,
|
||||
'episode_number': en,
|
||||
'title': r.get('title') or tm.get('title'),
|
||||
'air_date': r.get('air_date') or tm.get('air_date'),
|
||||
'overview': tm.get('overview') or r.get('overview'),
|
||||
'still_url': tm.get('still_url') or r.get('still_url'),
|
||||
'season_poster_url': poster,
|
||||
})
|
||||
|
||||
added = 0
|
||||
for (tid, title), grp in by_show.items():
|
||||
added += int(add_episodes(tid, title, grp['eps'], grp['library_id'],
|
||||
_show_poster_url(grp['library_id'])) or 0)
|
||||
shows = len(by_show)
|
||||
|
||||
done = ('Added %d airing episode(s) across %d show(s) to the wishlist'
|
||||
% (added, shows)) if added else 'No new airing episodes to wishlist today'
|
||||
if pruned:
|
||||
done += ' · pruned %d ended show(s)' % pruned
|
||||
deps.update_progress(
|
||||
automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line=done, log_type='success')
|
||||
return {'status': 'completed', 'episodes_added': added, 'shows': shows,
|
||||
'shows_pruned': pruned, '_manages_own_progress': True}
|
||||
except Exception as e: # noqa: BLE001
|
||||
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||
130
core/automation/handlers/video_clean_youtube.py
Normal file
130
core/automation/handlers/video_clean_youtube.py
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
"""Automation handler: ``video_clean_youtube_episodes`` action.
|
||||
|
||||
YouTube retention: for each followed channel that has a per-channel retention policy (set in
|
||||
the channel's cog modal — default 'keep everything'), delete the episodes that fall outside
|
||||
the keep window (video file + its ``-thumb``/``.nfo`` sidecars). The history row is KEPT and
|
||||
flagged pruned, so the scan's download-dedup still excludes it — a cleaned episode is never
|
||||
re-downloaded. Playlists are mirror-the-whole-thing, so they're out of scope (channels only).
|
||||
|
||||
Runs daily. Pure — channel list, per-channel policy, episode list, file deletion + the prune
|
||||
mark are all injected seams; tests drive it with fakes and never touch a disk or DB.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import date
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from core.automation.deps import AutomationDeps
|
||||
from core.video.retention import episodes_to_prune, parse_retention
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _default_fetch_channels() -> List[Any]:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().youtube_channels_with_downloads()
|
||||
|
||||
|
||||
def _default_channel_retention(channel_id: Any) -> Any:
|
||||
from api.video import get_video_db
|
||||
return (get_video_db().get_channel_settings(channel_id) or {}).get("retention")
|
||||
|
||||
|
||||
def _default_fetch_episodes(channel_id: Any) -> List[Dict[str, Any]]:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().youtube_channel_episodes(channel_id)
|
||||
|
||||
|
||||
def _default_mark_pruned(history_id: Any, when: str) -> bool:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().mark_download_pruned(history_id, when)
|
||||
|
||||
|
||||
def _silent_remove(path: str) -> None:
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _default_delete_files(ep: Dict[str, Any]):
|
||||
"""Delete the episode's video + its sidecars. Returns (ok, freed_bytes). Only ever touches
|
||||
the exact recorded ``dest_path`` and its same-stem sidecars — never walks a folder. A file
|
||||
already gone counts as success (mark it pruned); a real delete error does NOT (so it retries)."""
|
||||
path = ep.get("dest_path")
|
||||
if not path:
|
||||
return False, 0
|
||||
size = 0
|
||||
try:
|
||||
if os.path.exists(path):
|
||||
size = os.path.getsize(path)
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
logger.exception("retention: could not delete %s", path)
|
||||
return False, 0
|
||||
stem = os.path.splitext(path)[0]
|
||||
for sc in (stem + ".nfo", stem + "-thumb.jpg", stem + "-thumb.jpeg",
|
||||
stem + "-thumb.png", stem + "-thumb.webp"):
|
||||
if os.path.exists(sc):
|
||||
_silent_remove(sc)
|
||||
return True, size
|
||||
|
||||
|
||||
def _fmt_gb(b: int) -> str:
|
||||
gb = (b or 0) / (1024 ** 3)
|
||||
return ("%.1f GB" % gb) if gb >= 0.1 else "%d MB" % round((b or 0) / (1024 ** 2))
|
||||
|
||||
|
||||
def auto_video_clean_youtube_episodes(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
fetch_channels: Optional[Callable[[], List[Any]]] = None,
|
||||
channel_retention: Optional[Callable[[Any], Any]] = None,
|
||||
fetch_episodes: Optional[Callable[[Any], List[Dict[str, Any]]]] = None,
|
||||
delete_files: Optional[Callable[[Dict[str, Any]], Any]] = None,
|
||||
mark_pruned: Optional[Callable[[Any, str], Any]] = None,
|
||||
today_fn: Optional[Callable[[], str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Delete out-of-window episodes for channels with a retention policy. Returns
|
||||
``{'status': 'completed', 'deleted': int, 'channels': int, 'freed_bytes': int, ...}``."""
|
||||
fetch_channels = fetch_channels or _default_fetch_channels
|
||||
channel_retention = channel_retention or _default_channel_retention
|
||||
fetch_episodes = fetch_episodes or _default_fetch_episodes
|
||||
delete_files = delete_files or _default_delete_files
|
||||
mark_pruned = mark_pruned or _default_mark_pruned
|
||||
today_fn = today_fn or (lambda: date.today().isoformat())
|
||||
automation_id = config.get("_automation_id")
|
||||
try:
|
||||
today = today_fn()
|
||||
deps.update_progress(automation_id, phase="Checking retention…", progress=10,
|
||||
log_line="Looking for channels with a retention policy", log_type="info")
|
||||
deleted = freed = checked = 0
|
||||
for cid in (fetch_channels() or []):
|
||||
policy = channel_retention(cid)
|
||||
if not parse_retention(policy): # 'keep everything' / unset → skip
|
||||
continue
|
||||
checked += 1
|
||||
prune = episodes_to_prune(fetch_episodes(cid) or [], policy, today=today)
|
||||
for ep in prune:
|
||||
ok, size = delete_files(ep)
|
||||
if not ok:
|
||||
continue
|
||||
mark_pruned(ep.get("id"), today) # keep the row → scan won't re-download it
|
||||
deleted += 1
|
||||
freed += size or 0
|
||||
deps.update_progress(automation_id, log_type="info",
|
||||
log_line="Removed '%s'" % (ep.get("title") or ep.get("media_id") or "episode"))
|
||||
|
||||
msg = ("Removed %d old episode(s) · freed %s" % (deleted, _fmt_gb(freed))) if deleted \
|
||||
else "Nothing to clean — every channel within its retention window"
|
||||
deps.update_progress(automation_id, status="finished", progress=100, phase="Complete",
|
||||
log_line=msg, log_type="success")
|
||||
return {"status": "completed", "deleted": deleted, "channels": checked,
|
||||
"freed_bytes": freed, "_manages_own_progress": True}
|
||||
except Exception as e: # noqa: BLE001
|
||||
deps.update_progress(automation_id, status="error", phase="Error", log_line=str(e), log_type="error")
|
||||
return {"status": "error", "error": str(e), "_manages_own_progress": True}
|
||||
293
core/automation/handlers/video_process_wishlist.py
Normal file
293
core/automation/handlers/video_process_wishlist.py
Normal file
|
|
@ -0,0 +1,293 @@
|
|||
"""Automation handlers: ``video_process_movie_wishlist`` / ``video_process_episode_wishlist``.
|
||||
|
||||
The Soulseek counterpart of the YouTube wishlist drain — the piece that finally makes the
|
||||
people/airing scans pay off. For wished, RELEASED movies (and aired episodes) it searches
|
||||
Soulseek, picks the best release per the quality profile, and enqueues the download; the
|
||||
existing ``download_monitor`` finishes + organises + archives it, exactly like a manual grab.
|
||||
|
||||
Shape mirrors the YouTube drain (Boulder: "same standard"): it processes the WHOLE eligible
|
||||
wishlist (no total cap), but the slow part — each item needs a ~20s blocking Soulseek search
|
||||
— runs only a FEW at a time (``max_concurrent``). A ``guard`` keeps the next hourly tick from
|
||||
overlapping a run that's still working, so it can't pile up.
|
||||
|
||||
Movies are gated on ``status='wanted'`` (released; skips 'monitored'/unreleased). Episodes
|
||||
are all-wished (the airing scan only adds aired ones). Items already downloading are skipped
|
||||
so re-runs never double-grab. The pick is the top ACCEPTED release — the ranker already
|
||||
encodes the quality profile's accept/reject/score, so no extra rules here.
|
||||
|
||||
Shared automation side (may import ``core.video`` / ``api.video``); owns its own progress.
|
||||
The search + enqueue are injected seams, so selection/pick/record are pure + unit-tested.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
from core.automation.deps import AutomationDeps
|
||||
|
||||
|
||||
# ── pure helpers ──────────────────────────────────────────────────────────────
|
||||
def pick_best(candidates: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||
"""The best ACCEPTED release from a ranked candidate list. ``_evaluate_hits`` already
|
||||
sorts best-first (accepted, score, availability), so the first accepted is the pick."""
|
||||
for c in candidates or []:
|
||||
if c.get("accepted"):
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def item_key(item: Dict[str, Any], media_type: str) -> tuple:
|
||||
"""Stable identity for de-duping a wished item against active downloads."""
|
||||
if media_type == "movie":
|
||||
return ("movie", str(item.get("tmdb_id")))
|
||||
return ("episode", str(item.get("show_tmdb_id")),
|
||||
int(item.get("season_number") or 0), int(item.get("episode_number") or 0))
|
||||
|
||||
|
||||
def active_download_keys(active: Iterable[Dict[str, Any]]) -> set:
|
||||
"""Identity keys for the movie/episode downloads already in flight, so we don't
|
||||
re-grab them. Episodes read season/episode out of the row's ``search_ctx``."""
|
||||
keys = set()
|
||||
for d in active or []:
|
||||
kind = str(d.get("kind") or "").lower()
|
||||
if kind == "movie":
|
||||
keys.add(("movie", str(d.get("media_id"))))
|
||||
elif kind == "episode":
|
||||
ctx = d.get("search_ctx")
|
||||
if isinstance(ctx, str):
|
||||
try:
|
||||
ctx = json.loads(ctx)
|
||||
except (ValueError, TypeError):
|
||||
ctx = {}
|
||||
ctx = ctx if isinstance(ctx, dict) else {}
|
||||
keys.add(("episode", str(d.get("media_id")),
|
||||
int(ctx.get("season") or 0), int(ctx.get("episode") or 0)))
|
||||
return keys
|
||||
|
||||
|
||||
def search_context(item: Dict[str, Any], media_type: str) -> Dict[str, Any]:
|
||||
"""The ``search_ctx`` the download row carries (drives the monitor's requery)."""
|
||||
if media_type == "movie":
|
||||
return {"scope": "movie", "title": item.get("title"), "year": item.get("year")}
|
||||
return {"scope": "episode", "title": item.get("show_title"),
|
||||
"season": item.get("season_number"), "episode": item.get("episode_number"),
|
||||
"year": (str(item.get("air_date") or "")[:4] or None)}
|
||||
|
||||
|
||||
def build_download_record(item: Dict[str, Any], best: Dict[str, Any], candidates: List[Dict[str, Any]],
|
||||
*, media_type: str, target_dir: str, query: Any) -> Dict[str, Any]:
|
||||
"""The ``add_video_download`` row for a chosen release — identical shape to a manual
|
||||
grab, so the monitor finishes it the same way (other accepted hits become the retry
|
||||
pool)."""
|
||||
ctx = search_context(item, media_type)
|
||||
# stash the chosen source's peer stats so the drawer can show its availability snapshot
|
||||
# (free slot / queue depth / speed at grab time). Retry ignores the extra key.
|
||||
peer = {k: best.get(k) for k in ("slots", "queue", "speed", "availability") if best.get(k) is not None}
|
||||
if peer:
|
||||
ctx = {**ctx, "peer": peer}
|
||||
rest = [c for c in (candidates or []) if c.get("filename") != best.get("filename")]
|
||||
media_id = str(item.get("tmdb_id") if media_type == "movie" else item.get("show_tmdb_id"))
|
||||
return {
|
||||
"kind": media_type, "title": ctx["title"],
|
||||
"release_title": best.get("title") or best.get("filename"),
|
||||
"source": "soulseek", "username": best.get("username"), "filename": best.get("filename"),
|
||||
"size_bytes": int(best.get("size_bytes") or 0), "quality_label": best.get("quality_label"),
|
||||
"target_dir": target_dir, "status": "downloading",
|
||||
"media_id": media_id, "media_source": "tmdb", "year": ctx.get("year"),
|
||||
"poster_url": item.get("poster_url"),
|
||||
"candidates": json.dumps(rest), "search_ctx": json.dumps(ctx),
|
||||
"tried_queries": json.dumps([query] if query else []),
|
||||
"tried_files": json.dumps([best.get("filename")]), "attempts": 0,
|
||||
}
|
||||
|
||||
|
||||
# ── production seams ──────────────────────────────────────────────────────────
|
||||
def _default_fetch_items(media_type: str) -> List[Dict[str, Any]]:
|
||||
from api.video import get_video_db
|
||||
db = get_video_db()
|
||||
return db.movie_wishlist_to_download() if media_type == "movie" else db.episode_wishlist_to_download()
|
||||
|
||||
|
||||
def _default_active_keys(media_type: str) -> set:
|
||||
from api.video import get_video_db
|
||||
return active_download_keys(get_video_db().get_active_video_downloads())
|
||||
|
||||
|
||||
def _default_target_dir(media_type: str) -> str:
|
||||
from api.video import get_video_db
|
||||
db = get_video_db()
|
||||
if media_type == "movie":
|
||||
return db.get_setting("movies_path") or db.get_setting("transfer_path") or ""
|
||||
return db.get_setting("tv_path") or ""
|
||||
|
||||
|
||||
def _default_search(item: Dict[str, Any], media_type: str):
|
||||
"""A bounded blocking Soulseek search → ranked candidates (same path the retry worker +
|
||||
manual search use). Returns [] for a real empty result, or **None** if the search never
|
||||
ran (slskd not configured / errored / rate-limited) so the caller can say so."""
|
||||
from api.video.downloads import _evaluate_hits
|
||||
from core.video.download_monitor import _search_for_retry
|
||||
from core.video.quality_profile import load as load_profile
|
||||
from core.video.slskd_search import build_query
|
||||
from api.video import get_video_db
|
||||
ctx = search_context(item, media_type)
|
||||
query = build_query(ctx["scope"], ctx["title"], year=ctx.get("year"),
|
||||
season=ctx.get("season"), episode=ctx.get("episode"))
|
||||
res = _search_for_retry(query) or {}
|
||||
if res.get("started") is False:
|
||||
return None, res.get("error") # slskd didn't accept the search — pass the reason
|
||||
profile = load_profile(get_video_db())
|
||||
return _evaluate_hits(res.get("hits") or [], profile, ctx["scope"],
|
||||
ctx.get("season"), ctx.get("episode")), None
|
||||
|
||||
|
||||
def _default_enqueue(item: Dict[str, Any], best: Dict[str, Any], candidates: List[Dict[str, Any]],
|
||||
media_type: str, target_dir: str) -> bool:
|
||||
"""Start the slskd transfer + write the download row (exactly like the manual flow),
|
||||
then ensure the monitor is running. Returns True if slskd accepted it."""
|
||||
from api.video import get_video_db
|
||||
from core.video.download_monitor import ensure_started
|
||||
from core.video.slskd_download import start_download
|
||||
from core.video.slskd_search import build_query
|
||||
started = start_download(best.get("username"), best.get("filename"), best.get("size_bytes") or 0)
|
||||
if not started.get("ok"):
|
||||
return False
|
||||
ctx = search_context(item, media_type)
|
||||
query = build_query(ctx["scope"], ctx["title"], year=ctx.get("year"),
|
||||
season=ctx.get("season"), episode=ctx.get("episode"))
|
||||
get_video_db().add_video_download(
|
||||
build_download_record(item, best, candidates, media_type=media_type,
|
||||
target_dir=target_dir, query=query))
|
||||
ensure_started(get_video_db)
|
||||
return True
|
||||
|
||||
|
||||
# ── guard: keep an in-progress drain from overlapping the next tick ───────────
|
||||
_running: Dict[str, bool] = {"movie": False, "episode": False}
|
||||
|
||||
|
||||
def is_running(media_type: str) -> bool:
|
||||
return bool(_running.get(media_type))
|
||||
|
||||
|
||||
def auto_video_process_wishlist(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
media_type: str = "movie",
|
||||
fetch_items: Optional[Callable[[str], List[Dict[str, Any]]]] = None,
|
||||
active_keys: Optional[Callable[[str], Iterable]] = None,
|
||||
target_dir: Optional[Callable[[str], str]] = None,
|
||||
search: Optional[Callable[[Dict[str, Any], str], List[Dict[str, Any]]]] = None,
|
||||
enqueue: Optional[Callable[..., bool]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Auto-grab the wished movies (or episodes): search Soulseek, pick the best release,
|
||||
enqueue. Processes the whole eligible wishlist, a few searches at a time.
|
||||
|
||||
Returns ``{'status': 'completed', 'searched': int, 'grabbed': int, ...}``."""
|
||||
fetch_items = fetch_items or _default_fetch_items
|
||||
active_keys = active_keys or _default_active_keys
|
||||
target_dir = target_dir or _default_target_dir
|
||||
search = search or _default_search
|
||||
enqueue = enqueue or _default_enqueue
|
||||
automation_id = config.get('_automation_id')
|
||||
concurrency = max(1, int(config.get('max_concurrent', 3) or 3))
|
||||
label = 'movie' if media_type == 'movie' else 'episode'
|
||||
|
||||
_running[media_type] = True
|
||||
try:
|
||||
root = target_dir(media_type)
|
||||
if not root:
|
||||
where = 'Movie' if media_type == 'movie' else 'TV'
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line='%s library folder not set — skipping (Settings → Downloads)' % where,
|
||||
log_type='info')
|
||||
return {'status': 'completed', 'searched': 0, 'grabbed': 0,
|
||||
'skipped': 'no_folder', '_manages_own_progress': True}
|
||||
|
||||
deps.update_progress(automation_id, phase='Checking the wishlist…', progress=5,
|
||||
log_line='Looking for wished %ss to grab' % label, log_type='info')
|
||||
items = fetch_items(media_type) or []
|
||||
active = set(active_keys(media_type) or set())
|
||||
todo = [it for it in items if item_key(it, media_type) not in active]
|
||||
if not todo:
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line='Nothing new to grab (%d already in flight)' % len(active),
|
||||
log_type='info')
|
||||
return {'status': 'completed', 'searched': 0, 'grabbed': 0, '_manages_own_progress': True}
|
||||
|
||||
grabbed = [0]
|
||||
searched = [0]
|
||||
noresults = [0] # search came back empty (the source had nothing)
|
||||
rejected = [0] # source had hits, but none passed the quality profile
|
||||
notrun = [0] # the search never ran (slskd didn't accept it)
|
||||
total = len(todo)
|
||||
lock = threading.Lock()
|
||||
|
||||
def _one(it):
|
||||
found = search(it, media_type)
|
||||
# the seam returns (candidates, error); tolerate a bare list/None too (test fakes)
|
||||
cands, err = found if isinstance(found, tuple) else (found, None)
|
||||
didnt_run = cands is None # slskd not configured / errored / rate-limited
|
||||
cands = cands or []
|
||||
best = pick_best(cands)
|
||||
ok = bool(best) and bool(enqueue(it, best, cands, media_type, root))
|
||||
name = it.get('title') or it.get('show_title') or '?'
|
||||
if media_type == 'episode':
|
||||
name = "%s S%02dE%02d" % (name, int(it.get('season_number') or 0),
|
||||
int(it.get('episode_number') or 0))
|
||||
# tell apart: grabbed / search-didn't-run / source-empty / hits-but-all-rejected.
|
||||
if ok:
|
||||
msg, lt = "Grabbed '%s'" % name, 'success'
|
||||
elif didnt_run:
|
||||
msg = ("Search didn't run for '%s' — slskd error: %s" % (name, err)) if err \
|
||||
else ("Search didn't run for '%s' — slskd not responding?" % name)
|
||||
lt = 'warning'
|
||||
elif not cands:
|
||||
msg, lt = "No search results for '%s'" % name, 'info'
|
||||
else:
|
||||
why = (cands[0].get('rejected') or 'none met your quality profile')
|
||||
msg, lt = "%d result(s) for '%s', none accepted — %s" % (len(cands), name, why), 'info'
|
||||
with lock:
|
||||
searched[0] += 1
|
||||
if ok:
|
||||
grabbed[0] += 1
|
||||
elif didnt_run:
|
||||
notrun[0] += 1
|
||||
elif not cands:
|
||||
noresults[0] += 1
|
||||
else:
|
||||
rejected[0] += 1
|
||||
deps.update_progress(
|
||||
automation_id, phase='Searching + grabbing…',
|
||||
progress=10 + int(85 * searched[0] / max(total, 1)),
|
||||
log_line=msg, log_type=lt)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as ex:
|
||||
list(ex.map(_one, todo))
|
||||
|
||||
# Headline with the WHY breakdown: it's the difference between "the source has
|
||||
# nothing" (noresults) and "it has stuff but your quality profile rejects it" (rejected).
|
||||
tail = []
|
||||
if notrun[0]:
|
||||
tail.append("%d search(es) didn't run (slskd?)" % notrun[0])
|
||||
if noresults[0]:
|
||||
tail.append('%d had no results' % noresults[0])
|
||||
if rejected[0]:
|
||||
tail.append('%d rejected on quality' % rejected[0])
|
||||
breakdown = (' · ' + ', '.join(tail)) if tail else ''
|
||||
done = ('Grabbed %d %s(s) of %d searched%s' % (grabbed[0], label, searched[0], breakdown)) if grabbed[0] \
|
||||
else ('Searched %d %s(s), grabbed 0%s' % (searched[0], label, breakdown))
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line=done, log_type='success' if grabbed[0] else 'info')
|
||||
return {'status': 'completed', 'searched': searched[0], 'grabbed': grabbed[0],
|
||||
'noresults': noresults[0], 'rejected': rejected[0], 'notrun': notrun[0],
|
||||
'_manages_own_progress': True}
|
||||
except Exception as e: # noqa: BLE001
|
||||
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||
finally:
|
||||
_running[media_type] = False
|
||||
200
core/automation/handlers/video_process_youtube_wishlist.py
Normal file
200
core/automation/handlers/video_process_youtube_wishlist.py
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
"""Automation handler: ``video_process_youtube_wishlist`` action.
|
||||
|
||||
The drain side of the YouTube fulfillment lane. The watchlist-channels scan keeps the
|
||||
wishlist fed; THIS queues wished videos for download and keeps a few flowing at a time.
|
||||
|
||||
There is NO cap on how much of the wishlist gets processed — it queues the WHOLE thing.
|
||||
The only limit is how many download SIMULTANEOUSLY (``max_concurrent``, default 3): every
|
||||
wished video becomes a ``queued`` row in the shared ``video_downloads`` table, the handler
|
||||
starts up to the limit, and each finished download starts the next (one-out-one-in, in the
|
||||
worker) so the entire queue drains in a controlled stream. This mirrors the music side's
|
||||
download worker — a concurrency cap plus a small inter-download delay (handled in the
|
||||
worker) to avoid yt-dlp 429s — but stays on the isolated video side.
|
||||
|
||||
The worker (``core.video.youtube_download``) downloads → organises (channel/year/date) →
|
||||
archives to history → removes the video from the wishlist, so a completed grab leaves the
|
||||
wishlist and won't be re-queued; videos already queued or downloading are skipped so re-runs
|
||||
never double-grab.
|
||||
|
||||
Shared automation side (may import ``core.video`` / ``api.video``); owns its own progress.
|
||||
All I/O is injected as seams, so selection + the pump are pure unit-testable functions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
from core.automation.deps import AutomationDeps
|
||||
|
||||
|
||||
def videos_to_enqueue(wanted: List[Dict[str, Any]], already_ids: Iterable) -> List[Dict[str, Any]]:
|
||||
"""Wished videos not already queued or downloading. NO cap — the whole backlog is
|
||||
queued; concurrency is bounded at start time, not here. Pure."""
|
||||
already = {str(x) for x in (already_ids or ()) if x}
|
||||
out: List[Dict[str, Any]] = []
|
||||
for v in wanted or []:
|
||||
vid = v.get("video_id")
|
||||
if vid and str(vid) not in already:
|
||||
out.append(v)
|
||||
return out
|
||||
|
||||
|
||||
def slots_free(running: int, max_concurrent: int) -> int:
|
||||
"""How many new downloads may start now given how many are already fetching. Pure."""
|
||||
return max(0, int(max_concurrent) - max(0, int(running)))
|
||||
|
||||
|
||||
def enqueue_ctx(video: Dict[str, Any], channel_settings: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""The download row's ``search_ctx``, applying per-channel overrides: a custom show-name
|
||||
(the ``$channel`` folder token) and/or a quality override. Pure."""
|
||||
cs = channel_settings if isinstance(channel_settings, dict) else {}
|
||||
ctx = {
|
||||
"channel": cs.get("custom_name") or video.get("channel_title"),
|
||||
"channel_id": video.get("channel_id"), # so the drawer can open the in-app channel page
|
||||
"video_title": video.get("video_title"),
|
||||
"published_at": video.get("published_at"),
|
||||
}
|
||||
if cs.get("quality"):
|
||||
ctx["quality"] = cs["quality"]
|
||||
return ctx
|
||||
|
||||
|
||||
# ── production seams ──────────────────────────────────────────────────────────
|
||||
def _default_youtube_root() -> str:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().get_setting("youtube_path") or ""
|
||||
|
||||
|
||||
def _default_fetch_wanted() -> List[Dict[str, Any]]:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().youtube_wishlist_to_download()
|
||||
|
||||
|
||||
def _default_active_ids() -> List[Any]:
|
||||
from api.video import get_video_db
|
||||
return [d.get("media_id") for d in get_video_db().get_active_video_downloads()
|
||||
if d.get("source") == "youtube" and d.get("media_id")]
|
||||
|
||||
|
||||
def _default_running_count() -> int:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().count_active_youtube_downloads()
|
||||
|
||||
|
||||
def _default_enqueue(video: Dict[str, Any], root: str) -> Any:
|
||||
"""Create a QUEUED download row (no thread spawned here — the pump starts it). Returns
|
||||
the row id."""
|
||||
import json
|
||||
from api.video import get_video_db
|
||||
from core.video.sources import resolve_video_server
|
||||
db = get_video_db()
|
||||
ctx = enqueue_ctx(video, db.get_channel_settings(video.get("channel_id")))
|
||||
ctx["server_source"] = resolve_video_server()
|
||||
return db.add_video_download({
|
||||
"kind": "youtube", "source": "youtube", "media_source": "youtube",
|
||||
"title": video.get("video_title") or video.get("channel_title"),
|
||||
"media_id": video.get("video_id"), "target_dir": root, "status": "queued",
|
||||
"year": video.get("published_at"), "poster_url": video.get("thumbnail_url"),
|
||||
"search_ctx": json.dumps(ctx),
|
||||
})
|
||||
|
||||
|
||||
def _default_start_next() -> Any:
|
||||
"""Claim + start the next queued YouTube download (or None). The worker chains the rest."""
|
||||
from api.video import get_video_db
|
||||
from core.video.youtube_download import start_next_queued
|
||||
return start_next_queued(get_video_db)
|
||||
|
||||
|
||||
def _default_reap() -> int:
|
||||
"""Recover downloads orphaned by a restart (stuck 'downloading', no live worker)."""
|
||||
from api.video import get_video_db
|
||||
from core.video.youtube_download import requeue_orphaned_youtube
|
||||
return requeue_orphaned_youtube(get_video_db)
|
||||
|
||||
|
||||
def auto_video_process_youtube_wishlist(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
youtube_root: Optional[Callable[[], str]] = None,
|
||||
fetch_wanted: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||
active_ids: Optional[Callable[[], Iterable]] = None,
|
||||
running_count: Optional[Callable[[], int]] = None,
|
||||
enqueue: Optional[Callable[[Dict[str, Any], str], Any]] = None,
|
||||
start_next: Optional[Callable[[], Any]] = None,
|
||||
reap: Optional[Callable[[], int]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Queue the whole YouTube wishlist for download and start up to ``max_concurrent`` now.
|
||||
|
||||
Returns ``{'status': 'completed', 'queued': int, 'started': int, 'running': int, ...}``."""
|
||||
youtube_root = youtube_root or _default_youtube_root
|
||||
fetch_wanted = fetch_wanted or _default_fetch_wanted
|
||||
active_ids = active_ids or _default_active_ids
|
||||
running_count = running_count or _default_running_count
|
||||
enqueue = enqueue or _default_enqueue
|
||||
start_next = start_next or _default_start_next
|
||||
reap = reap or _default_reap
|
||||
automation_id = config.get('_automation_id')
|
||||
max_concurrent = max(1, int(config.get('max_concurrent', 3) or 3))
|
||||
|
||||
try:
|
||||
root = youtube_root()
|
||||
if not root:
|
||||
# Always-on automation: a missing folder isn't a failure, it's "not set up for
|
||||
# YouTube" — skip quietly so non-YouTube users don't see a recurring error.
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line='YouTube library folder not set — skipping (Settings → Downloads)',
|
||||
log_type='info')
|
||||
return {'status': 'completed', 'queued': 0, 'started': 0, 'running': 0,
|
||||
'skipped': 'no_youtube_folder', '_manages_own_progress': True}
|
||||
|
||||
# Recover any downloads orphaned by a restart (stuck 'downloading', no worker) so
|
||||
# they don't wedge the concurrency count — they go back to 'queued' and re-run.
|
||||
recovered = int(reap() or 0)
|
||||
if recovered:
|
||||
deps.update_progress(automation_id, log_type='info',
|
||||
log_line='Recovered %d stalled download(s) from a restart' % recovered)
|
||||
|
||||
deps.update_progress(automation_id, phase='Checking the YouTube wishlist…', progress=15,
|
||||
log_line='Queueing new videos for download', log_type='info')
|
||||
wanted = fetch_wanted() or []
|
||||
already = list(active_ids() or [])
|
||||
new = videos_to_enqueue(wanted, already)
|
||||
|
||||
queued = 0
|
||||
for v in new:
|
||||
try:
|
||||
if enqueue(v, root) is not None:
|
||||
queued += 1
|
||||
except Exception: # noqa: BLE001 - one bad enqueue shouldn't stop the rest
|
||||
deps.update_progress(automation_id, log_type='warning',
|
||||
log_line="Couldn't queue '%s'" % (v.get('video_title') or v.get('video_id')))
|
||||
|
||||
# Fill the concurrency slots now; each finished download starts the next, so the
|
||||
# whole queue drains on its own from here.
|
||||
deps.update_progress(automation_id, phase='Starting downloads…', progress=70,
|
||||
log_line='Queued %d new video(s)' % queued, log_type='info')
|
||||
started = 0
|
||||
for _ in range(slots_free(running_count() or 0, max_concurrent)):
|
||||
if start_next() is None:
|
||||
break
|
||||
started += 1
|
||||
|
||||
running = (running_count() or 0)
|
||||
if queued or started:
|
||||
done = 'Queued %d new · %d downloading now (the rest drain automatically)' % (queued, running)
|
||||
log_type = 'success'
|
||||
elif running:
|
||||
done = '%d already downloading; nothing new to queue' % running
|
||||
log_type = 'info'
|
||||
else:
|
||||
done = 'No wished YouTube videos to download'
|
||||
log_type = 'info'
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line=done, log_type=log_type)
|
||||
return {'status': 'completed', 'queued': queued, 'started': started, 'running': running,
|
||||
'_manages_own_progress': True}
|
||||
except Exception as e: # noqa: BLE001
|
||||
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||
90
core/automation/handlers/video_refresh_airing_schedules.py
Normal file
90
core/automation/handlers/video_refresh_airing_schedules.py
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
"""Automation handler: ``video_refresh_airing_schedules`` action.
|
||||
|
||||
Keeps the TV calendar honest. The "Wishlist Episodes Airing Today" automation reads the
|
||||
LOCAL ``episodes`` table — which is only refreshed from TMDB on initial enrichment, a manual
|
||||
re-match, or lazily when you open a show's page. So a newly-announced or rescheduled episode
|
||||
can sit unknown indefinitely, and the airing automation misses it.
|
||||
|
||||
This runs daily (a couple hours before the airing run) and re-pulls each still-airing
|
||||
watchlist show's season/episode schedule from TMDB (air dates, stills, overviews) so the
|
||||
calendar is current when the airing automation reads it.
|
||||
|
||||
Scope: the EFFECTIVE watchlist's continuing LIBRARY shows (follows ∪ airing library shows,
|
||||
skipping ended/canceled — they'll never air again, and skipping tmdb-only follows, which have
|
||||
no episodes table to refresh). Like the other video handlers it owns its progress reporting
|
||||
(``_manages_own_progress``); the show fetch + per-show refresh are injected seams, so the
|
||||
handler is a pure function tests drive with fakes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from core.automation.deps import AutomationDeps
|
||||
|
||||
|
||||
def _default_fetch_shows() -> List[Dict[str, Any]]:
|
||||
"""Production wiring: the still-airing watchlist shows that live in the library."""
|
||||
from api.video import get_video_db
|
||||
from core.video.sources import resolve_video_server
|
||||
return get_video_db().watchlist_continuing_shows(resolve_video_server())
|
||||
|
||||
|
||||
def _default_refresh_show(library_id: Any) -> Dict[str, Any]:
|
||||
"""Re-pull a library show's TMDB season/episode schedule (the lazy on-view backfill,
|
||||
invoked deliberately). Re-matches + cascades episodes, so air dates/stills refresh.
|
||||
``with_ratings=False`` — we only need schedules, and the per-show OMDb ratings call
|
||||
would burn the daily quota across a whole watchlist."""
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
return get_video_enrichment_engine().refresh_show_art(library_id, with_ratings=False) or {}
|
||||
|
||||
|
||||
def auto_video_refresh_airing_schedules(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
fetch_shows: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||
refresh_show: Optional[Callable[[Any], Dict[str, Any]]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Refresh the TMDB episode schedule for every still-airing watchlist library show, so
|
||||
the airing automation's calendar read is current.
|
||||
|
||||
Returns ``{'status': 'completed', 'refreshed': int, 'failed': int, 'shows': int, ...}``."""
|
||||
fetch_shows = fetch_shows or _default_fetch_shows
|
||||
refresh_show = refresh_show or _default_refresh_show
|
||||
automation_id = config.get('_automation_id')
|
||||
try:
|
||||
deps.update_progress(automation_id, phase='Finding shows to refresh…', progress=8,
|
||||
log_line='Reading your watchlist for still-airing shows', log_type='info')
|
||||
shows = fetch_shows() or []
|
||||
total = len(shows)
|
||||
if not total:
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line='No airing shows on your watchlist to refresh', log_type='success')
|
||||
return {'status': 'completed', 'refreshed': 0, 'failed': 0, 'shows': 0,
|
||||
'_manages_own_progress': True}
|
||||
|
||||
refreshed = failed = 0
|
||||
for i, s in enumerate(shows):
|
||||
title = s.get('title') or ('show %s' % s.get('library_id'))
|
||||
deps.update_progress(
|
||||
automation_id, phase='Refreshing TV schedules…', progress=10 + int(i / total * 85),
|
||||
log_line="Pulling the latest episodes for '%s' (%d/%d)" % (title, i + 1, total),
|
||||
log_type='info')
|
||||
try:
|
||||
ok = bool((refresh_show(s.get('library_id')) or {}).get('ok'))
|
||||
except Exception: # noqa: BLE001 - one show failing must not stop the rest
|
||||
ok = False
|
||||
if ok:
|
||||
refreshed += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
done = 'Refreshed %d show schedule(s)' % refreshed + (' · %d failed' % failed if failed else '')
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line=done, log_type='success')
|
||||
return {'status': 'completed', 'refreshed': refreshed, 'failed': failed, 'shows': total,
|
||||
'_manages_own_progress': True}
|
||||
except Exception as e: # noqa: BLE001
|
||||
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||
424
core/automation/handlers/video_scan_library.py
Normal file
424
core/automation/handlers/video_scan_library.py
Normal file
|
|
@ -0,0 +1,424 @@
|
|||
"""Automation handler: ``video_scan_library`` action.
|
||||
|
||||
The VIDEO twin of music's ``scan_library``. It does two things, in order:
|
||||
|
||||
1. Tells the active media server (Plex/Jellyfin) to rescan ONLY the
|
||||
user's selected VIDEO sections (the movie + TV libraries chosen in
|
||||
Settings) — never the music library.
|
||||
2. Reads the server's current state into ``video.db`` so freshly-added
|
||||
media shows up as owned on the video side.
|
||||
|
||||
Both run through injected seams (``server_refresh`` / ``run_video_scan``)
|
||||
so the handler stays a pure function: production lazily binds the real
|
||||
``core.video`` functions; tests pass fakes and never spin up Flask, a DB,
|
||||
or a media-server client.
|
||||
|
||||
ISOLATION NOTE: this handler lives on the SHARED automation side, so it
|
||||
is allowed to import ``core.video`` — the isolation contract only forbids
|
||||
``core/video`` and ``api/video`` from importing the MUSIC side, not the
|
||||
other way round. Video core stays import-clean; the bridge lives here.
|
||||
|
||||
Like ``scan_library``, the handler owns its own progress reporting
|
||||
(``_manages_own_progress: True``) so the engine doesn't stomp the live
|
||||
phase string with a generic 'completed' label.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from core.automation.deps import AutomationDeps
|
||||
|
||||
|
||||
def _default_server_refresh(media_type: str = "all") -> Dict[str, Any]:
|
||||
"""Production wiring: nudge the media server to rescan its video sections.
|
||||
``media_type`` scopes it to the Movie or TV section; 'all' nudges both."""
|
||||
from core.video.sources import refresh_video_server_sections
|
||||
return refresh_video_server_sections(media_type)
|
||||
|
||||
|
||||
def _default_run_video_scan(mode: str, media_type: str = "all") -> Dict[str, Any]:
|
||||
"""Production wiring: read the server into video.db (blocking). ``media_type``
|
||||
scopes it to one library ('movie' / 'show'); 'all' does both."""
|
||||
from api.video import get_video_db
|
||||
from core.video.scanner import get_video_scanner
|
||||
from core.video.sources import get_active_video_source
|
||||
return get_video_scanner(get_video_db()).scan_sync(get_active_video_source, mode, media_type)
|
||||
|
||||
|
||||
def auto_video_scan_library(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
server_refresh: Optional[Callable[[], Dict[str, Any]]] = None,
|
||||
run_video_scan: Optional[Callable[..., Dict[str, Any]]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Trigger a server-side video rescan, then mirror the result into video.db.
|
||||
|
||||
``config['media_type']`` scopes the scan to one library — 'movie' or 'show'
|
||||
(TV); 'all' (default) does both. Movies and TV are independent libraries, so
|
||||
the Movie scan never touches TV and vice-versa.
|
||||
|
||||
Returns one of:
|
||||
- ``{'status': 'completed', '_manages_own_progress': True, 'movies': .., 'shows': .., 'episodes': ..}``
|
||||
- ``{'status': 'skipped', 'reason': '...'}`` (another scan was already running)
|
||||
- ``{'status': 'error', 'error': '...', '_manages_own_progress': True}``
|
||||
"""
|
||||
server_refresh = server_refresh or _default_server_refresh
|
||||
run_video_scan = run_video_scan or _default_run_video_scan
|
||||
|
||||
automation_id = config.get('_automation_id')
|
||||
# 'full' is the safe default — upsert everything, never prune. 'deep' prunes
|
||||
# what the server no longer has; only use it when the config asks explicitly.
|
||||
mode = config.get('mode') or 'full'
|
||||
media_type = config.get('media_type') or 'all'
|
||||
lib_label = {'movie': 'Movie', 'show': 'TV'}.get(media_type, 'video')
|
||||
|
||||
try:
|
||||
deps.update_progress(
|
||||
automation_id,
|
||||
phase=f'Asking media server to rescan {lib_label} sections...',
|
||||
progress=10,
|
||||
log_line=f'Triggering server-side {lib_label} scan',
|
||||
log_type='info',
|
||||
)
|
||||
|
||||
# Step 1 — best-effort server nudge, scoped to the same library we're about
|
||||
# to read. A server that can't be triggered (none configured, or an adapter
|
||||
# without refresh support) is surfaced as a warning, NOT a hard failure: the
|
||||
# read below still mirrors whatever the server currently reports.
|
||||
refresh = server_refresh(media_type) or {}
|
||||
if not refresh.get('ok'):
|
||||
deps.update_progress(
|
||||
automation_id,
|
||||
log_line='Server scan trigger unavailable: ' + str(refresh.get('error') or 'unknown'),
|
||||
log_type='warning',
|
||||
)
|
||||
else:
|
||||
sections = refresh.get('sections')
|
||||
detail = str(sections) + ' video section(s)' if sections else 'selected video section(s)'
|
||||
deps.update_progress(
|
||||
automation_id,
|
||||
progress=30,
|
||||
log_line='Server rescanning ' + detail,
|
||||
log_type='success',
|
||||
)
|
||||
|
||||
# Step 2 — read the server into video.db (blocking; mirrors music's
|
||||
# scan handler blocking until the scan resolves).
|
||||
deps.update_progress(
|
||||
automation_id,
|
||||
phase=f'Reading {lib_label} library into SoulSync...',
|
||||
progress=45,
|
||||
)
|
||||
result = run_video_scan(mode, media_type) or {}
|
||||
state = result.get('state')
|
||||
# Singleton scanner already busy with another scan (e.g. the Movie + TV
|
||||
# deep scans firing close together) — skip cleanly, don't error.
|
||||
if state == 'in_progress':
|
||||
deps.update_progress(
|
||||
automation_id, status='finished', phase='Skipped',
|
||||
log_line='Another video scan is already running — skipping this run',
|
||||
log_type='info',
|
||||
)
|
||||
return {'status': 'skipped', 'reason': 'a video scan is already running',
|
||||
'_manages_own_progress': True}
|
||||
if state == 'error':
|
||||
err = result.get('error') or 'Video library scan failed'
|
||||
deps.update_progress(
|
||||
automation_id,
|
||||
status='error',
|
||||
phase='Error',
|
||||
log_line=err,
|
||||
log_type='error',
|
||||
)
|
||||
return {'status': 'error', 'error': err, '_manages_own_progress': True}
|
||||
|
||||
movies = int(result.get('movies', 0) or 0)
|
||||
shows = int(result.get('shows', 0) or 0)
|
||||
episodes = int(result.get('episodes', 0) or 0)
|
||||
# Summary names only the scanned library so a TV scan doesn't read "0 movies".
|
||||
if media_type == 'movie':
|
||||
summary = f'Movie library scanned: {movies} movies'
|
||||
elif media_type == 'show':
|
||||
summary = f'TV library scanned: {shows} shows, {episodes} episodes'
|
||||
else:
|
||||
summary = f'Video library scanned: {movies} movies, {shows} shows, {episodes} episodes'
|
||||
deps.update_progress(
|
||||
automation_id,
|
||||
status='finished',
|
||||
progress=100,
|
||||
phase='Complete',
|
||||
log_line=summary,
|
||||
log_type='success',
|
||||
)
|
||||
return {
|
||||
'status': 'completed',
|
||||
'_manages_own_progress': True,
|
||||
'movies': movies,
|
||||
'shows': shows,
|
||||
'episodes': episodes,
|
||||
}
|
||||
|
||||
except Exception as e: # noqa: BLE001 — automation handlers must never raise into the engine
|
||||
deps.update_progress(
|
||||
automation_id,
|
||||
status='error',
|
||||
phase='Error',
|
||||
log_line=str(e),
|
||||
log_type='error',
|
||||
)
|
||||
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||
|
||||
|
||||
# ── post-download chain (video twin of music's scan_library → start_database_update) ──
|
||||
# Split into two stages so the calendar/library reflect a download promptly, mirroring
|
||||
# the music side: tell the server to rescan, WAIT until its scan queue is actually idle
|
||||
# (a big library can take 10-20 min — a fixed wait would read too early), then fire
|
||||
# 'video_library_scan_completed'; stage 2 listens for that and reads the fresh state.
|
||||
|
||||
def _default_scan_status(media_type: str = "all"):
|
||||
"""Production wiring: True/False if the active server is/ isn't scanning, or None
|
||||
when it can't report (so the caller falls back to a fixed wait)."""
|
||||
from core.video.sources import video_server_scan_in_progress
|
||||
return video_server_scan_in_progress(media_type)
|
||||
|
||||
|
||||
def _default_latest_completed(media_type: str):
|
||||
"""The newest completed grab of a type — the probe target for the smart skip."""
|
||||
from api.video import get_video_db
|
||||
return get_video_db().latest_completed_download(media_type)
|
||||
|
||||
|
||||
def _default_server_has_item(media_type: str, item) -> bool:
|
||||
"""Does the active server already have this specific grab indexed?"""
|
||||
from core.video.sources import video_server_has_item
|
||||
return video_server_has_item(media_type, item)
|
||||
|
||||
|
||||
def wait_for_server_scan(scan_status, sleep, *, grace_seconds: int = 15,
|
||||
interval_seconds: int = 10, cap_seconds: int = 3600,
|
||||
fallback_seconds: int = 120) -> int:
|
||||
"""Block until the media server's scan queue goes idle, then return ~seconds waited.
|
||||
|
||||
``scan_status()`` returns True (scanning), False (idle) or None (can't tell). After
|
||||
a short grace — so the just-triggered scan has time to register as running — poll
|
||||
every ``interval_seconds`` until idle or ``cap_seconds`` (a generous backstop for a
|
||||
huge library). If the server can't report its state, fall back to a fixed
|
||||
``fallback_seconds`` wait — exactly the old behaviour. ``sleep`` is injected and
|
||||
elapsed is counted from the sleeps, so tests never actually block."""
|
||||
grace = max(0, grace_seconds)
|
||||
if grace:
|
||||
sleep(grace)
|
||||
waited = grace
|
||||
try:
|
||||
status = scan_status()
|
||||
except Exception: # noqa: BLE001 - status is best-effort
|
||||
status = None
|
||||
if status is None:
|
||||
extra = max(0, fallback_seconds - waited)
|
||||
if extra:
|
||||
sleep(extra)
|
||||
return waited + extra
|
||||
while status is True and waited < cap_seconds:
|
||||
sleep(interval_seconds)
|
||||
waited += interval_seconds
|
||||
try:
|
||||
status = scan_status()
|
||||
except Exception: # noqa: BLE001
|
||||
status = None
|
||||
if status is None: # lost the ability to tell — stop waiting, don't hang
|
||||
break
|
||||
return waited
|
||||
|
||||
|
||||
def probe_present_libraries(candidates, has_item, sleep, *, grace_seconds: int = 120,
|
||||
interval_seconds: int = 15):
|
||||
"""Which libraries does the server ALREADY have the newest grab for?
|
||||
|
||||
``candidates`` = [(scope, item), …]. A freshly-dropped file takes ~1-2 min to show
|
||||
up even with the server's auto-scan ON, so we POLL: check each candidate, and give
|
||||
the server up to ``grace_seconds`` (re-checking every ``interval_seconds``) to
|
||||
ingest before giving up on it. Returns the set of scopes confirmed present (→ skip
|
||||
their crawl); anything still missing when grace expires stays out (→ gets crawled).
|
||||
A probe that errors is treated as 'not present' (conservative). ``sleep`` injected."""
|
||||
pending = list(candidates or [])
|
||||
present = set()
|
||||
waited = 0
|
||||
while True:
|
||||
still = []
|
||||
for scope, item in pending:
|
||||
try:
|
||||
if has_item(scope, item):
|
||||
present.add(scope)
|
||||
continue
|
||||
except Exception: # noqa: BLE001, S110 - uncertainty → keep probing, then scan
|
||||
pass
|
||||
still.append((scope, item))
|
||||
pending = still
|
||||
if not pending or waited >= grace_seconds:
|
||||
return present
|
||||
sleep(interval_seconds)
|
||||
waited += interval_seconds
|
||||
|
||||
|
||||
_LIB = {'movie': 'Movie', 'show': 'TV'}
|
||||
|
||||
|
||||
def auto_video_scan_server(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
server_refresh: Optional[Callable[..., Dict[str, Any]]] = None,
|
||||
sleep: Optional[Callable[[float], None]] = None,
|
||||
emit: Optional[Callable[[str, Dict[str, Any]], None]] = None,
|
||||
scan_status: Optional[Callable[..., Any]] = None,
|
||||
latest_completed: Optional[Callable[[str], Any]] = None,
|
||||
server_has_item: Optional[Callable[[str, Any], bool]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Stage 1: get the server to index our new downloads, then fire
|
||||
'video_library_scan_completed' so the DB-update twin reads the fresh state.
|
||||
|
||||
SMART SKIP (``skip_if_present``, default on): scanning is expensive, and many
|
||||
servers auto-ingest new files. So per library, we first probe whether the server
|
||||
already has the NEWEST grab of that type (from download history) — if it does, it
|
||||
already picked everything up, and we skip that library's crawl. Only libraries the
|
||||
server is missing get the (expensive) rescan + poll-until-idle. We always emit so
|
||||
stage 2 still READS the new items into video.db. (Video twin of music's scan_library.)"""
|
||||
server_refresh = server_refresh or _default_server_refresh
|
||||
sleep = sleep or time.sleep
|
||||
emit = emit or deps.engine.emit
|
||||
scan_status = scan_status or _default_scan_status
|
||||
latest_completed = latest_completed or _default_latest_completed
|
||||
server_has_item = server_has_item or _default_server_has_item
|
||||
automation_id = config.get('_automation_id')
|
||||
media_type = config.get('media_type') or 'all'
|
||||
skip_if_present = config.get('skip_if_present', True)
|
||||
try:
|
||||
fallback = int(config.get('debounce_seconds') or 120)
|
||||
except (TypeError, ValueError):
|
||||
fallback = 120
|
||||
try:
|
||||
cap = int(config.get('max_wait_minutes') or 60) * 60
|
||||
except (TypeError, ValueError):
|
||||
cap = 3600
|
||||
try:
|
||||
_pg = config.get('probe_grace_minutes') # 0 is valid (probe once, no wait)
|
||||
grace = max(0, int(float(_pg if _pg is not None else 2) * 60))
|
||||
except (TypeError, ValueError):
|
||||
grace = 120
|
||||
try:
|
||||
# Which libraries actually need the expensive crawl? Skip any the server already
|
||||
# has the newest grab for (it auto-ingested → nothing to find). Fresh drops take
|
||||
# ~1-2 min to appear even with auto-scan ON, so POLL the probe over a grace
|
||||
# window instead of checking once immediately (which would always miss).
|
||||
scopes = ['movie', 'show'] if media_type == 'all' else \
|
||||
([media_type] if media_type in ('movie', 'show') else ['movie', 'show'])
|
||||
present = set()
|
||||
if skip_if_present:
|
||||
candidates = []
|
||||
for sc in scopes:
|
||||
try:
|
||||
latest = latest_completed(sc)
|
||||
except Exception: # noqa: BLE001
|
||||
latest = None
|
||||
if latest:
|
||||
candidates.append((sc, latest))
|
||||
if candidates:
|
||||
deps.update_progress(automation_id, progress=15,
|
||||
phase='Checking whether the server already has the new downloads…',
|
||||
log_line='Probing the server (giving its auto-scan time to ingest)…',
|
||||
log_type='info')
|
||||
present = probe_present_libraries(candidates, server_has_item, sleep,
|
||||
grace_seconds=grace)
|
||||
for sc in present:
|
||||
deps.update_progress(
|
||||
automation_id, log_line=f'{_LIB[sc]} library: server already has the newest grab — skipping its scan',
|
||||
log_type='info')
|
||||
to_scan = [sc for sc in scopes if sc not in present]
|
||||
|
||||
waited, server_name = 0, ''
|
||||
if to_scan:
|
||||
scan_scope = 'all' if set(to_scan) == {'movie', 'show'} else to_scan[0]
|
||||
lib_label = _LIB.get(scan_scope, 'video')
|
||||
deps.update_progress(automation_id, phase=f'Asking media server to rescan {lib_label}…', progress=20,
|
||||
log_line=f'Triggering server-side {lib_label} scan', log_type='info')
|
||||
refresh = server_refresh(scan_scope) or {}
|
||||
if not refresh.get('ok'):
|
||||
deps.update_progress(automation_id,
|
||||
log_line='Server scan trigger unavailable: ' + str(refresh.get('error') or 'unknown'),
|
||||
log_type='warning')
|
||||
server_name = refresh.get('server') or ''
|
||||
deps.update_progress(automation_id, phase='Waiting for the server to finish indexing…', progress=55)
|
||||
waited = wait_for_server_scan(lambda: scan_status(scan_scope), sleep,
|
||||
fallback_seconds=fallback, cap_seconds=cap)
|
||||
else:
|
||||
deps.update_progress(automation_id, progress=55,
|
||||
log_line='Server already has all the new downloads — no scan needed',
|
||||
log_type='success')
|
||||
|
||||
# Always emit with the ORIGINAL scope so stage 2 reads everything we grabbed.
|
||||
emit('video_library_scan_completed', {'server': server_name, 'media_type': media_type})
|
||||
done = (f'Server scan finished (~{waited}s) — updating the database' if to_scan
|
||||
else 'Skipped server scan — updating the database')
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line=done, log_type='success')
|
||||
return {'status': 'completed', '_manages_own_progress': True,
|
||||
'scanned': to_scan, 'skipped': [s for s in scopes if s not in to_scan]}
|
||||
except Exception as e: # noqa: BLE001
|
||||
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||
|
||||
|
||||
def auto_video_update_database(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
run_video_scan: Optional[Callable[..., Dict[str, Any]]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Stage 2: read the (now-rescanned) server into video.db — INCREMENTAL by default
|
||||
(newest-first, stop after N consecutive known). Video twin of start_database_update.
|
||||
|
||||
``media_type`` scopes it to the Movie or TV library ('all' does both). When fired by
|
||||
the post-download chain it inherits the scope of the scan that ran (carried on the
|
||||
event), so a TV-only rescan updates only TV."""
|
||||
run_video_scan = run_video_scan or _default_run_video_scan
|
||||
automation_id = config.get('_automation_id')
|
||||
mode = config.get('mode') or 'incremental'
|
||||
media_type = (config.get('media_type')
|
||||
or (config.get('_event_data') or {}).get('media_type')
|
||||
or 'all')
|
||||
lib_label = {'movie': 'Movie', 'show': 'TV'}.get(media_type, 'video')
|
||||
# 'deep'/'full' re-read the whole library; 'incremental' only grabs new items.
|
||||
phase = (f'Re-reading the {lib_label} library from the server…' if mode in ('deep', 'full')
|
||||
else f'Reading new {lib_label} media into SoulSync…')
|
||||
try:
|
||||
deps.update_progress(automation_id, phase=phase, progress=40)
|
||||
result = run_video_scan(mode, media_type) or {}
|
||||
if result.get('state') == 'in_progress':
|
||||
deps.update_progress(automation_id, status='finished', phase='Skipped',
|
||||
log_line='Another video scan is already running — skipping this run',
|
||||
log_type='info')
|
||||
return {'status': 'skipped', 'reason': 'a video scan is already running',
|
||||
'_manages_own_progress': True}
|
||||
if result.get('state') == 'error':
|
||||
err = result.get('error') or 'Video database update failed'
|
||||
deps.update_progress(automation_id, status='error', phase='Error', log_line=err, log_type='error')
|
||||
return {'status': 'error', 'error': err, '_manages_own_progress': True}
|
||||
movies = int(result.get('movies', 0) or 0)
|
||||
shows = int(result.get('shows', 0) or 0)
|
||||
episodes = int(result.get('episodes', 0) or 0)
|
||||
if media_type == 'movie':
|
||||
summary = f'Video database updated: {movies} movies'
|
||||
elif media_type == 'show':
|
||||
summary = f'Video database updated: {shows} shows, {episodes} episodes'
|
||||
else:
|
||||
summary = f'Video database updated: {movies} movies, {shows} shows, {episodes} episodes'
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line=summary, log_type='success')
|
||||
return {'status': 'completed', '_manages_own_progress': True,
|
||||
'movies': movies, 'shows': shows, 'episodes': episodes}
|
||||
except Exception as e: # noqa: BLE001
|
||||
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||
235
core/automation/handlers/video_scan_watchlist_channels.py
Normal file
235
core/automation/handlers/video_scan_watchlist_channels.py
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
"""Automation handler: ``video_scan_watchlist_channels`` action.
|
||||
|
||||
The "what's new from the channels I follow" scan — the piece that lets SoulSync replace
|
||||
ytdl-sub-style YouTube auto-downloaders. For every YouTube channel on the video watchlist,
|
||||
look at its recent uploads and wishlist the new long-form videos the user doesn't already
|
||||
have, so the (future) fulfillment engine can grab them.
|
||||
|
||||
This runs on a SHORT schedule (channels publish at all hours — pair it with a 6-hourly
|
||||
Schedule trigger; 3h is fine too, the scan is cheap). It's forward-looking and dup-proof:
|
||||
|
||||
* **Baseline = follow time.** What the user had before following isn't our concern — only
|
||||
uploads published on/after they followed the channel (the watchlist row's ``date_added``)
|
||||
are "new" and get wishlisted, forever forward.
|
||||
* **Last-N safety net.** A per-channel default (10) reaches a little BEFORE the baseline so
|
||||
the user is always kept current on the most recent videos even right after following / if
|
||||
a scan was missed. Global setting now; per-channel override (the hover settings modal,
|
||||
like watchlist-artist settings) comes later.
|
||||
* **Long-form only.** Shorts are excluded (the channel's Videos tab + a duration floor);
|
||||
livestreams/premieres that haven't aired are skipped (future-dated).
|
||||
* **Never duplicates.** Each candidate is diffed against what's already wishlisted, what's
|
||||
already been downloaded (the permanent download-history ledger — empty for YouTube until
|
||||
the fulfillment engine lands, wired here so it's correct the day it does), and what the
|
||||
user has dismissed.
|
||||
|
||||
Like the other video handlers it lives on the SHARED automation side (may import
|
||||
``core.video`` / ``api.video`` — the isolation contract only forbids the reverse) and owns
|
||||
its own progress (``_manages_own_progress``). All I/O is injected as seams, so the selection
|
||||
logic is a pure, unit-testable function; production lazily binds the real calls.
|
||||
|
||||
NOTE (follow-up): "remove from the YouTube wishlist" currently just deletes the row, so the
|
||||
last-N net could re-add a video the user deliberately removed. The scan already respects a
|
||||
``dismissed_ids`` seam — wiring the wishlist-remove to record a dismissal (once the wishlist
|
||||
is a real download queue) closes that loop. Tracked, out of scope for this scan-only phase.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
from core.automation.deps import AutomationDeps
|
||||
|
||||
|
||||
# ── pure selection ────────────────────────────────────────────────────────────
|
||||
def is_short(video: Dict[str, Any], min_seconds: int) -> bool:
|
||||
"""A YouTube Short — a known duration under the floor. Unknown duration is NOT
|
||||
assumed short (the Videos-tab listing already excludes Shorts; this is a backstop)."""
|
||||
d = video.get("duration_seconds")
|
||||
return isinstance(d, (int, float)) and 0 < d < min_seconds
|
||||
|
||||
|
||||
def long_form_uploads(uploads: Iterable, min_seconds: int) -> List[Dict[str, Any]]:
|
||||
"""The channel's real videos, newest-first order preserved: drop Shorts + entries
|
||||
with no id."""
|
||||
return [v for v in (uploads or [])
|
||||
if isinstance(v, dict) and v.get("youtube_id") and not is_short(v, min_seconds)]
|
||||
|
||||
|
||||
def _day(value: Any) -> Optional[str]:
|
||||
s = str(value or "").strip()
|
||||
return s[:10] or None
|
||||
|
||||
|
||||
def select_channel_video_gaps(
|
||||
uploads: List[Dict[str, Any]],
|
||||
*,
|
||||
baseline_date: Optional[str],
|
||||
backfill_count: int,
|
||||
wishlisted_ids: Iterable = (),
|
||||
dismissed_ids: Iterable = (),
|
||||
downloaded_ids: Iterable = (),
|
||||
today: str,
|
||||
min_seconds: int = 60,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""The pure core: which of a channel's recent uploads to wishlist now.
|
||||
|
||||
``uploads`` is the channel's recent videos newest-first (rich shape: youtube_id, title,
|
||||
published_at, duration_seconds, thumbnail_url, …). Keeps a long-form upload if it's not
|
||||
already wishlisted / downloaded / dismissed AND either it's within the newest
|
||||
``backfill_count`` (the safety net) or it was published on/after ``baseline_date`` (the
|
||||
forward-looking part). Future-dated (unaired premiere/stream) is skipped. No I/O."""
|
||||
longs = long_form_uploads(uploads, min_seconds)
|
||||
n = max(0, int(backfill_count or 0))
|
||||
net_ids = {v["youtube_id"] for v in longs[:n]}
|
||||
excluded = set(wishlisted_ids or ()) | set(dismissed_ids or ()) | set(downloaded_ids or ())
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
for v in longs:
|
||||
vid = v["youtube_id"]
|
||||
if vid in excluded:
|
||||
continue
|
||||
d = _day(v.get("published_at"))
|
||||
if d and today and d > today:
|
||||
continue # scheduled / unaired — not out yet
|
||||
if vid in net_ids or (d and baseline_date and d >= baseline_date):
|
||||
out.append(v)
|
||||
return out
|
||||
|
||||
|
||||
# ── production seams ──────────────────────────────────────────────────────────
|
||||
def _default_fetch_channels() -> List[Dict[str, Any]]:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().list_watchlist_channels()
|
||||
|
||||
|
||||
def _default_fetch_uploads(channel_id: Any, limit: int) -> List[Dict[str, Any]]:
|
||||
"""A channel's recent uploads (Videos tab → long-form, with durations) with upload
|
||||
dates merged on from the cache + a cheap RSS pull — the same composition the channel
|
||||
detail endpoint uses. Caches any dates it learns so later scans get them free."""
|
||||
from api.video import get_video_db
|
||||
from core.video import youtube as yt
|
||||
db = get_video_db()
|
||||
cid = str(channel_id)
|
||||
channel = yt.resolve_channel("https://www.youtube.com/channel/" + cid,
|
||||
limit=max(1, min(90, int(limit)))) or {}
|
||||
vids = channel.get("videos") or []
|
||||
ids = [v.get("youtube_id") for v in vids if v.get("youtube_id")]
|
||||
dates = db.get_video_dates(ids) or {}
|
||||
try:
|
||||
dates.update(yt.channel_recent_dates(cid) or {})
|
||||
except Exception: # noqa: BLE001, S110 - RSS is best-effort; cached dates still apply
|
||||
pass
|
||||
for v in vids:
|
||||
if not v.get("published_at") and dates.get(v.get("youtube_id")):
|
||||
v["published_at"] = dates[v["youtube_id"]]
|
||||
try:
|
||||
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")}
|
||||
for v in vids if v.get("published_at")])
|
||||
except Exception: # noqa: BLE001, S110 - caching is opportunistic
|
||||
pass
|
||||
return vids
|
||||
|
||||
|
||||
def _default_wishlisted_ids(channel_id: Any) -> List[Any]:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().wishlisted_video_ids_for_channel(channel_id)
|
||||
|
||||
|
||||
def _default_dismissed_ids(channel_id: Any) -> List[Any]:
|
||||
# No YouTube "dismissed" store yet (video_ignored is movie/show-level). Returns empty;
|
||||
# wiring wishlist-remove → dismiss is the tracked follow-up (see module docstring).
|
||||
return []
|
||||
|
||||
|
||||
def _default_downloaded_ids(channel_id: Any) -> List[Any]:
|
||||
# Already-downloaded YouTube videos (global; a video id is unique). A completed download
|
||||
# leaves the wishlist, so without this the scan would re-add + re-download it.
|
||||
from api.video import get_video_db
|
||||
return get_video_db().downloaded_youtube_video_ids()
|
||||
|
||||
|
||||
def _default_add_videos(channel: Dict[str, Any], videos: List[Dict[str, Any]]) -> int:
|
||||
from api.video import get_video_db
|
||||
from core.video.sources import resolve_video_server
|
||||
return get_video_db().add_videos_to_wishlist(channel, videos, server_source=resolve_video_server())
|
||||
|
||||
|
||||
def auto_video_scan_watchlist_channels(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
fetch_channels: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||
fetch_uploads: Optional[Callable[[Any, int], List[Dict[str, Any]]]] = None,
|
||||
wishlisted_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||
dismissed_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||
downloaded_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||
add_videos: Optional[Callable[[Dict[str, Any], List[Dict[str, Any]]], int]] = None,
|
||||
today_fn: Optional[Callable[[], str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Scan every followed YouTube channel and wishlist its new long-form uploads.
|
||||
|
||||
Returns ``{'status': 'completed', 'channels': int, 'videos_added': int, ...}``."""
|
||||
fetch_channels = fetch_channels or _default_fetch_channels
|
||||
fetch_uploads = fetch_uploads or _default_fetch_uploads
|
||||
wishlisted_ids = wishlisted_ids or _default_wishlisted_ids
|
||||
dismissed_ids = dismissed_ids or _default_dismissed_ids
|
||||
downloaded_ids = downloaded_ids or _default_downloaded_ids
|
||||
add_videos = add_videos or _default_add_videos
|
||||
today_fn = today_fn or (lambda: date.today().isoformat())
|
||||
automation_id = config.get('_automation_id')
|
||||
backfill = max(0, int(config.get('backfill_count', 10) or 0))
|
||||
min_seconds = max(0, int(config.get('min_seconds', 60) or 0))
|
||||
limit = max(backfill, 30)
|
||||
|
||||
try:
|
||||
today = today_fn()
|
||||
deps.update_progress(automation_id, phase='Reading your watchlist…', progress=5,
|
||||
log_line='Loading the channels you follow', log_type='info')
|
||||
channels = fetch_channels() or []
|
||||
if not channels:
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line='No channels on the watchlist to scan', log_type='info')
|
||||
return {'status': 'completed', 'channels': 0, 'videos_added': 0,
|
||||
'_manages_own_progress': True}
|
||||
|
||||
added = 0
|
||||
total = len(channels)
|
||||
for i, ch in enumerate(channels):
|
||||
cid = ch.get('youtube_id')
|
||||
ctitle = ch.get('title') or cid
|
||||
deps.update_progress(automation_id, phase='Scanning channels…',
|
||||
progress=10 + int(80 * i / max(total, 1)),
|
||||
log_line="Checking %s for new videos" % ctitle, log_type='info')
|
||||
if not cid:
|
||||
continue
|
||||
try:
|
||||
uploads = fetch_uploads(cid, limit) or []
|
||||
except Exception: # noqa: BLE001 - one flaky channel shouldn't abort the scan
|
||||
deps.update_progress(automation_id, log_line="Couldn't reach %s — skipping" % ctitle,
|
||||
log_type='warning')
|
||||
continue
|
||||
|
||||
baseline = _day(ch.get('date_added')) or today
|
||||
gaps = select_channel_video_gaps(
|
||||
uploads, baseline_date=baseline, backfill_count=backfill,
|
||||
wishlisted_ids=wishlisted_ids(cid), dismissed_ids=dismissed_ids(cid),
|
||||
downloaded_ids=downloaded_ids(cid), today=today, min_seconds=min_seconds)
|
||||
if gaps:
|
||||
n = int(add_videos({'youtube_id': cid, 'title': ctitle,
|
||||
'avatar_url': ch.get('poster_url')}, gaps) or 0)
|
||||
added += n
|
||||
if n:
|
||||
deps.update_progress(
|
||||
automation_id, log_type='success',
|
||||
log_line="Wishlisted %d new video(s) from %s" % (n, ctitle))
|
||||
|
||||
done = ('Wishlisted %d new video(s) across %d channel(s)' % (added, total)) if added \
|
||||
else ('Channels are up to date — nothing new across %d channel(s)' % total)
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line=done, log_type='success')
|
||||
return {'status': 'completed', 'channels': total, 'videos_added': added,
|
||||
'_manages_own_progress': True}
|
||||
except Exception as e: # noqa: BLE001
|
||||
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||
296
core/automation/handlers/video_scan_watchlist_people.py
Normal file
296
core/automation/handlers/video_scan_watchlist_people.py
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
"""Automation handler: ``video_scan_watchlist_people`` action.
|
||||
|
||||
The video "watchlist" follows ongoing *things* — shows, channels, and people. For shows
|
||||
the ``video_add_airing_episodes`` automation already keeps the wishlist fed. This handler
|
||||
does the equivalent for the PEOPLE you follow:
|
||||
|
||||
For every person on the watchlist, look up their filmography and wishlist every MOVIE the
|
||||
user doesn't already own — the whole back catalog they acted in or directed, plus anything
|
||||
upcoming. The first run is the meaty one (it backlogs everything); later runs are fast
|
||||
because they skip movies already on the wishlist and only promote ones that have since been
|
||||
released.
|
||||
|
||||
Design decisions (Boulder):
|
||||
* MOVIES ONLY — no TV episodes for a person (shows are handled by their own automation).
|
||||
* Both ACTOR (Acting credits, minus "playing themselves") and DIRECTOR credits count.
|
||||
* UNRELEASED movies are added as ``status='monitored'`` so the wishlist/download engine
|
||||
leaves them alone until they're out; a later scan PROMOTES them to ``'wanted'``.
|
||||
* Best-in-class UX: grab as much data as possible at add time (backdrop, overview, genres,
|
||||
runtime, rating, top cast, director, release date, + provenance "because you follow X")
|
||||
and stash it as a rich ``detail_json`` blob on the wishlist row, so the UI renders a full
|
||||
card later without re-fetching.
|
||||
|
||||
Like the other video handlers this lives on the SHARED automation side (so it may import
|
||||
``core.video`` / ``api.video`` — the isolation contract only forbids the reverse) and owns
|
||||
its own progress (``_manages_own_progress``). All I/O is injected as seams, so the logic is
|
||||
a pure function in tests (no DB, no TMDB); production lazily binds the real calls.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
from core.automation.deps import AutomationDeps
|
||||
from core.video.discovery_gaps import filmography_gaps
|
||||
|
||||
|
||||
# ── pure credit classification ────────────────────────────────────────────────
|
||||
# An actor "playing themselves" in a documentary / talk show / award broadcast isn't a
|
||||
# dramatic role — drop those so the wishlist stays films they actually acted in.
|
||||
_SELF_MARKERS = ('self', 'himself', 'herself', 'themselves', 'archive footage',
|
||||
'archival footage')
|
||||
|
||||
|
||||
def is_self_credit(role) -> bool:
|
||||
"""True for a 'plays themselves' / archive-footage credit (role text only)."""
|
||||
r = str(role or '').strip().lower()
|
||||
return bool(r) and any(m in r for m in _SELF_MARKERS)
|
||||
|
||||
|
||||
def is_actor_movie_credit(c: Dict[str, Any]) -> bool:
|
||||
return (c.get('kind') == 'movie'
|
||||
and str(c.get('department') or '') == 'Acting'
|
||||
and not is_self_credit(c.get('role')))
|
||||
|
||||
|
||||
def is_director_movie_credit(c: Dict[str, Any]) -> bool:
|
||||
return c.get('kind') == 'movie' and str(c.get('role') or '').strip().lower() == 'director'
|
||||
|
||||
|
||||
def is_relevant_movie_credit(c: Any) -> bool:
|
||||
"""A movie this person ACTED in (not as themselves) or DIRECTED."""
|
||||
return isinstance(c, dict) and (is_actor_movie_credit(c) or is_director_movie_credit(c))
|
||||
|
||||
|
||||
def is_released(date_str: Any, today: str) -> bool:
|
||||
"""True if the release date is on/before ``today``. No date → NOT yet released, so we
|
||||
monitor it (a later scan promotes once a real, past date arrives)."""
|
||||
d = str(date_str or '').strip()
|
||||
return bool(d) and d[:10] <= today
|
||||
|
||||
|
||||
def _int_set(values: Iterable) -> set:
|
||||
out = set()
|
||||
for x in values or []:
|
||||
try:
|
||||
out.add(int(x))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return out
|
||||
|
||||
|
||||
def select_person_movie_gaps(credits: List[Dict[str, Any]], owned_ids: Iterable,
|
||||
ignored_ids: Iterable, *, today: str) -> List[Dict[str, Any]]:
|
||||
"""The pure core: a followed person's un-owned actor/director MOVIE credits.
|
||||
|
||||
Keeps only relevant movie credits, drops owned + ignored + duplicates, ranks by
|
||||
popularity (via ``filmography_gaps``), and tags each with the wishlist ``_status`` it
|
||||
should get — ``'wanted'`` if released, else ``'monitored'``. No I/O."""
|
||||
relevant = [c for c in (credits or []) if is_relevant_movie_credit(c)]
|
||||
ignored = _int_set(ignored_ids)
|
||||
out: List[Dict[str, Any]] = []
|
||||
for g in filmography_gaps(owned_ids, relevant, kinds=("movie",)):
|
||||
if int(g['tmdb_id']) in ignored:
|
||||
continue
|
||||
g = dict(g)
|
||||
g['_status'] = 'wanted' if is_released(g.get('date'), today) else 'monitored'
|
||||
out.append(g)
|
||||
return out
|
||||
|
||||
|
||||
def build_detail_blob(detail: Optional[Dict[str, Any]], credit: Dict[str, Any],
|
||||
person: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Trim TMDB full-detail down to a rich-but-lean card blob + provenance.
|
||||
|
||||
Drops the heavy ``_extras`` (similar/recommendations/keywords/reviews/providers) and
|
||||
keeps what a wishlist card / detail view wants. Degrades to the credit's own fields when
|
||||
``detail`` is missing or a library redirect."""
|
||||
via = {
|
||||
'person_tmdb_id': person.get('tmdb_id'),
|
||||
'person_name': person.get('title') or person.get('name'),
|
||||
'role': credit.get('role'),
|
||||
'as': 'director' if is_director_movie_credit(credit) else 'actor',
|
||||
}
|
||||
if not detail or detail.get('redirect'):
|
||||
return {
|
||||
'title': credit.get('title'), 'year': credit.get('year'),
|
||||
'release_date': credit.get('date'), 'poster_url': credit.get('poster'),
|
||||
'added_via': via,
|
||||
}
|
||||
director = next((p.get('name') for p in (detail.get('crew') or [])
|
||||
if str(p.get('job')) == 'Director'), None)
|
||||
return {
|
||||
'title': detail.get('title'),
|
||||
'overview': detail.get('overview'),
|
||||
'tagline': detail.get('tagline'),
|
||||
'status': detail.get('status'),
|
||||
'rating': detail.get('rating'),
|
||||
'imdb_id': detail.get('imdb_id'),
|
||||
'poster_url': detail.get('poster_url') or credit.get('poster'),
|
||||
'backdrop_url': detail.get('backdrop_url'),
|
||||
'logo': detail.get('logo'),
|
||||
'genres': detail.get('genres') or [],
|
||||
'runtime_minutes': detail.get('runtime_minutes'),
|
||||
'studio': detail.get('studio'),
|
||||
'year': detail.get('year') or credit.get('year'),
|
||||
'release_date': detail.get('release_date') or credit.get('date'),
|
||||
'cast': (detail.get('cast') or [])[:15],
|
||||
'director': director,
|
||||
'added_via': via,
|
||||
}
|
||||
|
||||
|
||||
# ── production seams ──────────────────────────────────────────────────────────
|
||||
def _default_fetch_people() -> List[Dict[str, Any]]:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().list_watchlist('person')
|
||||
|
||||
|
||||
def _default_fetch_credits(tmdb_id: Any) -> List[Dict[str, Any]]:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
d = get_video_enrichment_engine().person_detail(tmdb_id) or {}
|
||||
return d.get('credits') or []
|
||||
|
||||
|
||||
def _default_fetch_detail(tmdb_id: Any) -> Optional[Dict[str, Any]]:
|
||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||
return get_video_enrichment_engine().tmdb_detail('movie', tmdb_id)
|
||||
|
||||
|
||||
def _default_owned_ids() -> Iterable:
|
||||
from api.video import get_video_db
|
||||
from core.video.sources import resolve_video_server
|
||||
return get_video_db().owned_movie_tmdb_ids(resolve_video_server())
|
||||
|
||||
|
||||
def _default_ignored_ids() -> List[Any]:
|
||||
from api.video import get_video_db
|
||||
return [r.get('tmdb_id') for r in (get_video_db().list_ignored() or [])
|
||||
if r.get('kind') == 'movie']
|
||||
|
||||
|
||||
def _default_wishlisted_status() -> Dict[int, str]:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().wishlisted_movie_status()
|
||||
|
||||
|
||||
def _default_add_movie(tmdb_id, title, *, year, poster_url, status, detail_json) -> bool:
|
||||
from api.video import get_video_db
|
||||
from core.video.sources import resolve_video_server
|
||||
return get_video_db().add_movie_to_wishlist(
|
||||
tmdb_id, title, year=year, poster_url=poster_url, status=status,
|
||||
detail_json=detail_json, server_source=resolve_video_server())
|
||||
|
||||
|
||||
def auto_video_scan_watchlist_people(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
fetch_people: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||
fetch_credits: Optional[Callable[[Any], List[Dict[str, Any]]]] = None,
|
||||
fetch_detail: Optional[Callable[[Any], Optional[Dict[str, Any]]]] = None,
|
||||
owned_ids: Optional[Callable[[], Iterable]] = None,
|
||||
ignored_ids: Optional[Callable[[], List[Any]]] = None,
|
||||
wishlisted_status: Optional[Callable[[], Dict[int, str]]] = None,
|
||||
add_movie: Optional[Callable[..., bool]] = None,
|
||||
today_fn: Optional[Callable[[], str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Scan every followed person's filmography and wishlist the movies the user is missing.
|
||||
|
||||
Returns ``{'status': 'completed', 'people': int, 'movies_added': int, 'upcoming': int,
|
||||
'promoted': int, ...}`` — ``movies_added`` = released wishlisted, ``upcoming`` = monitored
|
||||
(unreleased) wishlisted, ``promoted`` = monitored rows flipped to wanted now they're out."""
|
||||
fetch_people = fetch_people or _default_fetch_people
|
||||
fetch_credits = fetch_credits or _default_fetch_credits
|
||||
fetch_detail = fetch_detail or _default_fetch_detail
|
||||
owned_ids = owned_ids or _default_owned_ids
|
||||
ignored_ids = ignored_ids or _default_ignored_ids
|
||||
wishlisted_status = wishlisted_status or _default_wishlisted_status
|
||||
add_movie = add_movie or _default_add_movie
|
||||
today_fn = today_fn or (lambda: date.today().isoformat())
|
||||
automation_id = config.get('_automation_id')
|
||||
|
||||
try:
|
||||
today = today_fn()
|
||||
deps.update_progress(automation_id, phase='Reading your watchlist…', progress=5,
|
||||
log_line='Loading the people you follow', log_type='info')
|
||||
people = fetch_people() or []
|
||||
if not people:
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line='No people on the watchlist to scan', log_type='info')
|
||||
return {'status': 'completed', 'people': 0, 'movies_added': 0, 'upcoming': 0,
|
||||
'promoted': 0, '_manages_own_progress': True}
|
||||
|
||||
owned = owned_ids() or set()
|
||||
ignored = ignored_ids() or []
|
||||
# Snapshot of what's already wishlisted {tmdb_id: status}; updated as we add so a
|
||||
# movie credited to two followed people is handled once.
|
||||
wished: Dict[int, str] = dict(wishlisted_status() or {})
|
||||
|
||||
added = upcoming = promoted = 0
|
||||
total = len(people)
|
||||
for i, person in enumerate(people):
|
||||
pid = person.get('tmdb_id')
|
||||
pname = person.get('title') or person.get('name') or pid
|
||||
deps.update_progress(automation_id, phase='Scanning filmographies…',
|
||||
progress=10 + int(80 * i / max(total, 1)),
|
||||
log_line="Looking up %s's movies" % pname, log_type='info')
|
||||
if not pid:
|
||||
continue
|
||||
try:
|
||||
credits = fetch_credits(pid) or []
|
||||
except Exception: # noqa: BLE001 - one bad lookup shouldn't abort the whole scan
|
||||
deps.update_progress(automation_id, log_line="Couldn't fetch %s — skipping" % pname,
|
||||
log_type='warning')
|
||||
continue
|
||||
|
||||
for g in select_person_movie_gaps(credits, owned, ignored, today=today):
|
||||
tid = int(g['tmdb_id'])
|
||||
want = g['_status'] # 'wanted' | 'monitored'
|
||||
existing = wished.get(tid)
|
||||
if existing is not None:
|
||||
# Already wishlisted — the only action left is promoting a monitored
|
||||
# row that has since been released (don't re-fetch detail for it).
|
||||
if existing == 'monitored' and want == 'wanted':
|
||||
if add_movie(tid, g.get('title'), year=g.get('year'),
|
||||
poster_url=g.get('poster'), status='wanted', detail_json=None):
|
||||
wished[tid] = 'wanted'
|
||||
promoted += 1
|
||||
deps.update_progress(
|
||||
automation_id, log_type='success',
|
||||
log_line="'%s' is out now — promoted to wanted"
|
||||
% (g.get('title') or tid))
|
||||
continue
|
||||
|
||||
# New gap — grab the rich detail (cached), build the blob, add it.
|
||||
try:
|
||||
detail = fetch_detail(tid)
|
||||
except Exception: # noqa: BLE001 - degrade to the cheap credit fields
|
||||
detail = None
|
||||
blob = build_detail_blob(detail, g, person)
|
||||
if add_movie(tid, g.get('title') or blob.get('title'), year=g.get('year'),
|
||||
poster_url=blob.get('poster_url') or g.get('poster'),
|
||||
status=want, detail_json=blob):
|
||||
wished[tid] = want
|
||||
if want == 'wanted':
|
||||
added += 1
|
||||
else:
|
||||
upcoming += 1
|
||||
|
||||
parts = []
|
||||
if added:
|
||||
parts.append('%d new movie(s)' % added)
|
||||
if upcoming:
|
||||
parts.append('%d upcoming' % upcoming)
|
||||
if promoted:
|
||||
parts.append('%d promoted' % promoted)
|
||||
done = ('Wishlisted ' + ', '.join(parts) + ' across %d followed person(s)' % total) \
|
||||
if parts else ('Watchlist is up to date — nothing new across %d person(s)' % total)
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line=done, log_type='success')
|
||||
return {'status': 'completed', 'people': total, 'movies_added': added,
|
||||
'upcoming': upcoming, 'promoted': promoted, '_manages_own_progress': True}
|
||||
except Exception as e: # noqa: BLE001
|
||||
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||
179
core/automation/handlers/video_scan_watchlist_playlists.py
Normal file
179
core/automation/handlers/video_scan_watchlist_playlists.py
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
"""Automation handler: ``video_scan_watchlist_playlists`` action.
|
||||
|
||||
Sibling of the watchlist-CHANNELS scan, with a deliberately different rule. A channel is a
|
||||
creator posting new stuff over time, so that scan is forward-looking (new uploads + a
|
||||
last-N net). A **playlist** is a curated, finite set someone assembled — the reason you
|
||||
follow it is "give me this whole list and keep it complete." So this scan MIRRORS the
|
||||
playlist: it wishlists every long-form video in it you don't already have, plus anything
|
||||
later added. No follow-date baseline, no last-N net.
|
||||
|
||||
Organisation: the playlist becomes its own "show" (playlist-as-show) — its videos are
|
||||
wishlisted under the playlist's title, so the download worker files them as
|
||||
``Playlist Name / Season YEAR / Playlist Name - DATE - Title`` (matches the ytdl-sub
|
||||
``tv_show_name``-on-a-playlist convention). All other plumbing is shared with channels: the
|
||||
same wishlist rows, the same "Process YouTube Wishlist" drain, quality + org template.
|
||||
|
||||
Edge: a video in both a followed channel AND a followed playlist is wishlisted once (dedup
|
||||
by video id); whichever scan touched it last sets its show-name. Accepted.
|
||||
|
||||
Pure selection with all I/O injected; shared automation side; owns its own progress.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||
|
||||
from core.automation.deps import AutomationDeps
|
||||
from core.automation.handlers.video_scan_watchlist_channels import _day, long_form_uploads
|
||||
|
||||
# how many playlist entries to read per scan (yt-dlp flat). Big enough for almost any real
|
||||
# playlist; genuinely huge ones truncate (logged) rather than hammering YouTube each scan.
|
||||
_PLAYLIST_FETCH_LIMIT = 1000
|
||||
|
||||
|
||||
def select_playlist_video_gaps(
|
||||
videos: List[Dict[str, Any]],
|
||||
*,
|
||||
wishlisted_ids: Iterable = (),
|
||||
downloaded_ids: Iterable = (),
|
||||
dismissed_ids: Iterable = (),
|
||||
today: str,
|
||||
min_seconds: int = 60,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""The pure core: every long-form video in a playlist not already wishlisted /
|
||||
downloaded / dismissed (mirror the whole list). Future-dated (unaired) entries are
|
||||
skipped. No baseline, no last-N — a curated playlist is wanted in full. No I/O."""
|
||||
excluded = set(wishlisted_ids or ()) | set(downloaded_ids or ()) | set(dismissed_ids or ())
|
||||
out: List[Dict[str, Any]] = []
|
||||
for v in long_form_uploads(videos, min_seconds):
|
||||
vid = v["youtube_id"]
|
||||
if vid in excluded:
|
||||
continue
|
||||
d = _day(v.get("published_at"))
|
||||
if d and today and d > today:
|
||||
continue # unaired premiere — not out yet
|
||||
out.append(v)
|
||||
return out
|
||||
|
||||
|
||||
# ── production seams ──────────────────────────────────────────────────────────
|
||||
def _default_fetch_playlists() -> List[Dict[str, Any]]:
|
||||
from api.video import get_video_db
|
||||
return get_video_db().list_watchlist_playlists()
|
||||
|
||||
|
||||
def _default_fetch_videos(playlist_id: Any) -> List[Dict[str, Any]]:
|
||||
"""A playlist's videos (flat, with durations) with upload dates merged from the cache.
|
||||
Playlists have no per-channel RSS, so dates come from whatever's cached (filled over
|
||||
time by the channel/InnerTube enrichers); undated entries organise cleanly."""
|
||||
from api.video import get_video_db
|
||||
from core.video import youtube as yt
|
||||
db = get_video_db()
|
||||
vids = yt.playlist_videos(str(playlist_id), limit=_PLAYLIST_FETCH_LIMIT) or []
|
||||
ids = [v.get("youtube_id") for v in vids if v.get("youtube_id")]
|
||||
dates = db.get_video_dates(ids) or {}
|
||||
for v in vids:
|
||||
if not v.get("published_at") and dates.get(v.get("youtube_id")):
|
||||
v["published_at"] = dates[v["youtube_id"]]
|
||||
return vids
|
||||
|
||||
|
||||
def _default_wishlisted_ids(playlist_id: Any) -> List[Any]:
|
||||
# parent_source_id holds the playlist id for playlist-sourced wishlist videos.
|
||||
from api.video import get_video_db
|
||||
return get_video_db().wishlisted_video_ids_for_channel(playlist_id)
|
||||
|
||||
|
||||
def _default_downloaded_ids(playlist_id: Any) -> List[Any]:
|
||||
# Already-downloaded YouTube videos (global). A completed download leaves the wishlist,
|
||||
# so without this the scan would re-add + re-download it.
|
||||
from api.video import get_video_db
|
||||
return get_video_db().downloaded_youtube_video_ids()
|
||||
|
||||
|
||||
def _default_dismissed_ids(playlist_id: Any) -> List[Any]:
|
||||
return [] # no youtube "dismissed" store yet (see channels-scan note)
|
||||
|
||||
|
||||
def _default_add_videos(playlist: Dict[str, Any], videos: List[Dict[str, Any]]) -> int:
|
||||
"""Wishlist under the PLAYLIST as the show (title = playlist name → playlist-as-show)."""
|
||||
from api.video import get_video_db
|
||||
from core.video.sources import resolve_video_server
|
||||
return get_video_db().add_videos_to_wishlist(playlist, videos, server_source=resolve_video_server())
|
||||
|
||||
|
||||
def auto_video_scan_watchlist_playlists(
|
||||
config: Dict[str, Any],
|
||||
deps: AutomationDeps,
|
||||
*,
|
||||
fetch_playlists: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||
fetch_videos: Optional[Callable[[Any], List[Dict[str, Any]]]] = None,
|
||||
wishlisted_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||
downloaded_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||
dismissed_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||
add_videos: Optional[Callable[[Dict[str, Any], List[Dict[str, Any]]], int]] = None,
|
||||
today_fn: Optional[Callable[[], str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Mirror every followed YouTube playlist into the wishlist (whole list + new additions).
|
||||
|
||||
Returns ``{'status': 'completed', 'playlists': int, 'videos_added': int, ...}``."""
|
||||
fetch_playlists = fetch_playlists or _default_fetch_playlists
|
||||
fetch_videos = fetch_videos or _default_fetch_videos
|
||||
wishlisted_ids = wishlisted_ids or _default_wishlisted_ids
|
||||
downloaded_ids = downloaded_ids or _default_downloaded_ids
|
||||
dismissed_ids = dismissed_ids or _default_dismissed_ids
|
||||
add_videos = add_videos or _default_add_videos
|
||||
today_fn = today_fn or (lambda: date.today().isoformat())
|
||||
automation_id = config.get('_automation_id')
|
||||
min_seconds = max(0, int(config.get('min_seconds', 60) or 0))
|
||||
|
||||
try:
|
||||
today = today_fn()
|
||||
deps.update_progress(automation_id, phase='Reading your watchlist…', progress=5,
|
||||
log_line='Loading the playlists you follow', log_type='info')
|
||||
playlists = fetch_playlists() or []
|
||||
if not playlists:
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line='No playlists on the watchlist to scan', log_type='info')
|
||||
return {'status': 'completed', 'playlists': 0, 'videos_added': 0,
|
||||
'_manages_own_progress': True}
|
||||
|
||||
added = 0
|
||||
total = len(playlists)
|
||||
for i, pl in enumerate(playlists):
|
||||
pid = pl.get('playlist_id')
|
||||
ptitle = pl.get('title') or pid
|
||||
deps.update_progress(automation_id, phase='Scanning playlists…',
|
||||
progress=10 + int(80 * i / max(total, 1)),
|
||||
log_line="Mirroring '%s'" % ptitle, log_type='info')
|
||||
if not pid:
|
||||
continue
|
||||
try:
|
||||
videos = fetch_videos(pid) or []
|
||||
except Exception: # noqa: BLE001 - one flaky playlist shouldn't abort the scan
|
||||
deps.update_progress(automation_id, log_line="Couldn't reach '%s' — skipping" % ptitle,
|
||||
log_type='warning')
|
||||
continue
|
||||
|
||||
gaps = select_playlist_video_gaps(
|
||||
videos, wishlisted_ids=wishlisted_ids(pid), downloaded_ids=downloaded_ids(pid),
|
||||
dismissed_ids=dismissed_ids(pid), today=today, min_seconds=min_seconds)
|
||||
if gaps:
|
||||
n = int(add_videos({'youtube_id': pid, 'title': ptitle,
|
||||
'avatar_url': pl.get('poster_url')}, gaps) or 0)
|
||||
added += n
|
||||
if n:
|
||||
deps.update_progress(
|
||||
automation_id, log_type='success',
|
||||
log_line="Wishlisted %d new video(s) from '%s'" % (n, ptitle))
|
||||
|
||||
done = ('Wishlisted %d new video(s) across %d playlist(s)' % (added, total)) if added \
|
||||
else ('Playlists are up to date — nothing new across %d playlist(s)' % total)
|
||||
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||
log_line=done, log_type='success')
|
||||
return {'status': 'completed', 'playlists': total, 'videos_added': added,
|
||||
'_manages_own_progress': True}
|
||||
except Exception as e: # noqa: BLE001
|
||||
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||
|
|
@ -145,6 +145,186 @@ SYSTEM_AUTOMATIONS = [
|
|||
'action_type': 'full_cleanup',
|
||||
'initial_delay': 900, # 15 min after startup
|
||||
},
|
||||
# ── Video side (isolated app, shared engine) ──────────────────────────
|
||||
# owned_by='video' keeps these OFF the music automations page (it filters
|
||||
# them out) and ON the video Automations page (it shows only these).
|
||||
# Schedule-based for now; a video-download-complete event trigger can
|
||||
# replace the schedule once that event is wired into the engine.
|
||||
# (No standalone scheduled scan — the post-download chain below keeps the library
|
||||
# fresh. The 'video_scan_library' action/block still exist for a custom automation.)
|
||||
# Post-download chain (video twin of music's batch_complete → scan_library →
|
||||
# library_scan_completed → start_database_update). Event-based, so a finished
|
||||
# video download refreshes the server then pulls the new media into video.db.
|
||||
{
|
||||
'name': 'Auto-Scan Video After Downloads',
|
||||
'trigger_type': 'video_batch_complete',
|
||||
'trigger_config': {},
|
||||
'action_type': 'video_scan_server',
|
||||
'owned_by': 'video',
|
||||
},
|
||||
{
|
||||
'name': 'Auto-Update Video Database After Scan',
|
||||
'trigger_type': 'video_library_scan_completed',
|
||||
'trigger_config': {},
|
||||
'action_type': 'video_update_database',
|
||||
'action_config': {'mode': 'incremental'},
|
||||
'owned_by': 'video',
|
||||
},
|
||||
# Safety net: re-read the server hourly too, so MANUAL library additions (which Plex
|
||||
# auto-scans) appear within the hour instead of waiting for the weekly deep scan. Same
|
||||
# cheap incremental read as the after-scan one.
|
||||
{
|
||||
'name': 'Auto-Update Video Database (Hourly)',
|
||||
'trigger_type': 'schedule',
|
||||
'trigger_config': {'interval': 1, 'unit': 'hours'},
|
||||
'action_type': 'video_update_database_hourly',
|
||||
'action_config': {'mode': 'incremental'},
|
||||
'initial_delay': 900,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
# Sonarr-style: once a day at 1am (server-local), wishlist every episode airing
|
||||
# today for the shows you follow (skipping ones already owned) so the day's episodes
|
||||
# are queued overnight. A fixed wall-clock 'daily_time' (not a rolling 24h interval
|
||||
# that drifts with restarts) — the seeder now arms timed system triggers, and
|
||||
# _fix_airing_automation_schedule migrates the old 24h-interval row.
|
||||
# Runs before the airing automation so the calendar it reads is current — re-pulls
|
||||
# TMDB episode schedules for still-airing watchlist shows (the airing read is LOCAL).
|
||||
{
|
||||
'name': 'Refresh Airing TV Schedules',
|
||||
'trigger_type': 'daily_time',
|
||||
'trigger_config': {'time': '23:00'},
|
||||
'action_type': 'video_refresh_airing_schedules',
|
||||
'owned_by': 'video',
|
||||
},
|
||||
{
|
||||
'name': 'Auto-Wishlist Episodes Airing Today',
|
||||
'trigger_type': 'daily_time',
|
||||
'trigger_config': {'time': '01:00'},
|
||||
'action_type': 'video_add_airing_episodes',
|
||||
'owned_by': 'video',
|
||||
},
|
||||
# Video twins of the music maintenance jobs — same schedule + shared handler,
|
||||
# distinct action_type + owned_by='video' so they seed as separate rows and
|
||||
# show on the video Automations page (music's copies are untouched).
|
||||
{
|
||||
'name': 'Clean Search History',
|
||||
'trigger_type': 'schedule',
|
||||
'trigger_config': {'interval': 1, 'unit': 'hours'},
|
||||
'action_type': 'video_clean_search_history',
|
||||
'initial_delay': 600,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
{
|
||||
'name': 'Clean Completed Downloads',
|
||||
'trigger_type': 'schedule',
|
||||
'trigger_config': {'interval': 5, 'unit': 'minutes'},
|
||||
'action_type': 'video_clean_completed_downloads',
|
||||
'initial_delay': 300,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
{
|
||||
'name': 'Full Cleanup',
|
||||
'trigger_type': 'schedule',
|
||||
'trigger_config': {'interval': 12, 'unit': 'hours'},
|
||||
'action_type': 'video_full_cleanup',
|
||||
'initial_delay': 900,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
{
|
||||
'name': 'Auto-Backup Database',
|
||||
'trigger_type': 'schedule',
|
||||
'trigger_config': {'interval': 3, 'unit': 'days'},
|
||||
'action_type': 'video_backup_database',
|
||||
'initial_delay': 600,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
# Video twin of music's 'Auto-Deep Scan Library', split into TWO because Movies
|
||||
# and TV are independent libraries — a TV scan never pulls in new movies and
|
||||
# vice-versa. Fixed weekly deep scan (re-read + prune removed) at 02:00 server-
|
||||
# local: TV Mondays, Movies Tuesdays — different days so they never overlap. The
|
||||
# seeder arms timed system triggers; _fix_deep_scan_schedules migrates the
|
||||
# original rolling-7-day rows. (Busy guard in the scanner is still a safety net.)
|
||||
{
|
||||
'name': 'Auto-Deep Scan TV Library',
|
||||
'trigger_type': 'weekly_time',
|
||||
'trigger_config': {'time': '02:00', 'days': ['mon']},
|
||||
'action_type': 'video_deep_scan_tv',
|
||||
'action_config': {'mode': 'deep', 'media_type': 'show'},
|
||||
'owned_by': 'video',
|
||||
},
|
||||
{
|
||||
'name': 'Auto-Deep Scan Movie Library',
|
||||
'trigger_type': 'weekly_time',
|
||||
'trigger_config': {'time': '02:00', 'days': ['tue']},
|
||||
'action_type': 'video_deep_scan_movies',
|
||||
'action_config': {'mode': 'deep', 'media_type': 'movie'},
|
||||
'owned_by': 'video',
|
||||
},
|
||||
# ── Watchlist → Wishlist pipeline ─────────────────────────────────────────
|
||||
# Stage 1: SCANS that FILL the wishlist from what you follow. (The airing-episodes
|
||||
# scan above is the show equivalent.) All no-op cleanly if you follow nobody.
|
||||
{
|
||||
'name': 'Auto-Scan Watchlist People', # followed actors/directors → wished movies
|
||||
'trigger_type': 'daily_time', # daily; filmographies change slowly
|
||||
'trigger_config': {'time': '03:00'}, # after airing/deep-scan jobs, no overlap
|
||||
'action_type': 'video_scan_watchlist_people',
|
||||
'owned_by': 'video',
|
||||
},
|
||||
{
|
||||
'name': 'Auto-Scan Watchlist Channels', # followed YouTube channels → wished videos
|
||||
'trigger_type': 'schedule',
|
||||
'trigger_config': {'interval': 6, 'unit': 'hours'}, # YouTube posts at all hours
|
||||
'action_type': 'video_scan_watchlist_channels',
|
||||
'action_config': {'backfill_count': 10},
|
||||
'initial_delay': 1200,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
{
|
||||
'name': 'Auto-Scan Watchlist Playlists', # followed playlists (mirror-all) → wished videos
|
||||
'trigger_type': 'schedule',
|
||||
'trigger_config': {'interval': 6, 'unit': 'hours'},
|
||||
'action_type': 'video_scan_watchlist_playlists',
|
||||
'initial_delay': 1320,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
# Stage 2: PROCESSORS that DRAIN the wishlist by downloading. Hourly; each skips
|
||||
# quietly until its library folder (+ slskd, for movie/episode) is configured.
|
||||
{
|
||||
'name': 'Auto-Process Movie Wishlist', # wished movies → slskd search/pick/grab
|
||||
'trigger_type': 'schedule',
|
||||
'trigger_config': {'interval': 1, 'unit': 'hours'},
|
||||
'action_type': 'video_process_movie_wishlist',
|
||||
'action_config': {'max_concurrent': 3},
|
||||
'initial_delay': 1620,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
{
|
||||
'name': 'Auto-Process Episode Wishlist', # wished episodes → slskd search/pick/grab
|
||||
'trigger_type': 'schedule',
|
||||
'trigger_config': {'interval': 1, 'unit': 'hours'},
|
||||
'action_type': 'video_process_episode_wishlist',
|
||||
'action_config': {'max_concurrent': 3},
|
||||
'initial_delay': 1740,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
{
|
||||
'name': 'Auto-Process YouTube Wishlist', # wished YouTube videos → yt-dlp download
|
||||
'trigger_type': 'schedule',
|
||||
'trigger_config': {'interval': 1, 'unit': 'hours'},
|
||||
'action_type': 'video_process_youtube_wishlist',
|
||||
'action_config': {'max_concurrent': 3},
|
||||
'initial_delay': 1500,
|
||||
'owned_by': 'video',
|
||||
},
|
||||
# YouTube retention: delete channel episodes outside each channel's keep window. No-op
|
||||
# unless a channel opts in (cog modal → Keep); default keeps everything, so safe to seed.
|
||||
{
|
||||
'name': 'Auto-Clean Old YouTube Episodes',
|
||||
'trigger_type': 'daily_time',
|
||||
'trigger_config': {'time': '04:00'},
|
||||
'action_type': 'video_clean_youtube_episodes',
|
||||
'owned_by': 'video',
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -237,8 +417,11 @@ class AutomationEngine:
|
|||
trigger_type=spec['trigger_type'],
|
||||
trigger_config=json.dumps(spec['trigger_config']),
|
||||
action_type=spec['action_type'],
|
||||
action_config='{}',
|
||||
action_config=json.dumps(spec.get('action_config', {})),
|
||||
profile_id=1,
|
||||
# owned_by tags the side that owns this automation (e.g.
|
||||
# 'video'), so the music page can exclude another side's rows.
|
||||
owned_by=spec.get('owned_by'),
|
||||
)
|
||||
if aid:
|
||||
self.db.update_automation(aid, is_system=1)
|
||||
|
|
@ -284,7 +467,117 @@ class AutomationEngine:
|
|||
self.db.update_automation(existing['id'], next_run=nr)
|
||||
logger.info(f"System automation '{spec['name']}' next_run set to {initial_delay}s from now")
|
||||
else:
|
||||
logger.info(f"System automation '{spec['name']}' ready (event-based)")
|
||||
# No initial_delay. A timer-based timed trigger (daily/weekly/
|
||||
# monthly at a fixed wall-clock time) still needs its next_run
|
||||
# armed — compute it from the schedule. Genuinely event-based
|
||||
# triggers (batch_complete, scan_done) have no next_run, so
|
||||
# next_run_at returns None and we leave them alone. Don't clobber
|
||||
# an existing future next_run (manual edit / restart-resume).
|
||||
nr_dt = next_run_at(spec['trigger_type'], spec['trigger_config'],
|
||||
now_utc=_utcnow(), default_tz=self._default_tz)
|
||||
if nr_dt is not None and not existing.get('next_run'):
|
||||
self.db.update_automation(existing['id'], next_run=_dt_to_db_str(nr_dt))
|
||||
logger.info(f"System automation '{spec['name']}' next_run armed for {_dt_to_db_str(nr_dt)} (timed)")
|
||||
else:
|
||||
logger.info(f"System automation '{spec['name']}' ready (event-based)")
|
||||
self._fix_video_scan_default()
|
||||
self._fix_airing_automation_schedule()
|
||||
self._fix_deep_scan_schedules()
|
||||
self._fix_wishlist_processor_rename()
|
||||
|
||||
def _fix_video_scan_default(self):
|
||||
"""Remove the obsolete standalone 'Scan Video Library' SYSTEM automation — it's
|
||||
superseded by the post-download chain (Auto-Scan Video After Downloads →
|
||||
Auto-Update Video Database After Scan).
|
||||
|
||||
``get_system_automation_by_action`` matches ONLY a system-seeded row
|
||||
(is_system=1), so a user's own scan automation is never touched. Idempotent —
|
||||
safe to run on every startup; once the row is gone the lookup returns None and
|
||||
it no-ops. (No flag guard: the old one could latch True without ever deleting,
|
||||
which is exactly why the row survived earlier 'cleanups'.)"""
|
||||
try:
|
||||
auto = self.db.get_system_automation_by_action('video_scan_library')
|
||||
if auto:
|
||||
self.db.delete_automation(auto['id'])
|
||||
logger.info("Removed superseded 'Scan Video Library' system automation (id=%s)",
|
||||
auto.get('id'))
|
||||
except Exception:
|
||||
logger.exception("video scan cleanup failed")
|
||||
|
||||
def _fix_wishlist_processor_rename(self):
|
||||
"""Migrate the wishlist processors' 'Download' → 'Process' rename so a DB seeded
|
||||
under the old names doesn't show stale duplicates.
|
||||
|
||||
The movie/episode ACTIONS were renamed (``video_download_*`` → ``video_process_*``),
|
||||
so the old seeded rows are now orphaned (dead action_type) while the new ones reseed
|
||||
alongside them — delete the orphans. ``delete_automation`` refuses system rows, so
|
||||
clear ``is_system`` first. The YouTube action kept its type but its label changed, so
|
||||
rename that row in place. Idempotent — no-ops once the DB is clean."""
|
||||
try:
|
||||
for dead in ('video_download_movie_wishlist', 'video_download_episode_wishlist'):
|
||||
auto = self.db.get_system_automation_by_action(dead)
|
||||
if auto:
|
||||
self.db.update_automation(auto['id'], is_system=0) # lift the delete guard
|
||||
self.db.delete_automation(auto['id'])
|
||||
logger.info("Removed orphaned '%s' system automation (renamed to process, id=%s)",
|
||||
auto.get('name'), auto.get('id'))
|
||||
yt = self.db.get_system_automation_by_action('video_process_youtube_wishlist')
|
||||
if yt and yt.get('name') == 'Auto-Download YouTube Wishlist':
|
||||
self.db.update_automation(yt['id'], name='Auto-Process YouTube Wishlist')
|
||||
logger.info("Renamed YouTube wishlist automation → 'Auto-Process YouTube Wishlist' (id=%s)",
|
||||
yt.get('id'))
|
||||
except Exception:
|
||||
logger.exception("wishlist processor rename migration failed")
|
||||
|
||||
def _fix_airing_automation_schedule(self):
|
||||
"""Migrate 'Auto-Wishlist Episodes Airing Today' from the old rolling 24h
|
||||
interval to a fixed daily 1am run.
|
||||
|
||||
It originally shipped as a 'schedule'/24h interval because the seeder only
|
||||
armed interval specs — a 'daily_time' spec sat idle and never fired. The 24h
|
||||
interval fires reliably but at a time that drifts with every restart (5 min
|
||||
after startup, then +24h). Now that the seeder arms timed triggers, rewrite
|
||||
the live row to run at a fixed 1am (better for 'today's airings' — queues the
|
||||
day overnight). Matches only the is_system row; idempotent (no-op once the row
|
||||
is already daily_time)."""
|
||||
try:
|
||||
auto = self.db.get_system_automation_by_action('video_add_airing_episodes')
|
||||
if not auto or auto.get('trigger_type') == 'daily_time':
|
||||
return
|
||||
cfg = {'time': '01:00'}
|
||||
nr_dt = next_run_at('daily_time', cfg, now_utc=_utcnow(), default_tz=self._default_tz)
|
||||
self.db.update_automation(
|
||||
auto['id'], trigger_type='daily_time', trigger_config=json.dumps(cfg),
|
||||
next_run=_dt_to_db_str(nr_dt) if nr_dt is not None else None)
|
||||
logger.info("Migrated 'Auto-Wishlist Episodes Airing Today' to a fixed daily 01:00 (id=%s)",
|
||||
auto.get('id'))
|
||||
except Exception:
|
||||
logger.exception("airing automation schedule migration failed")
|
||||
|
||||
def _fix_deep_scan_schedules(self):
|
||||
"""Migrate the two video deep-scan system automations from the original
|
||||
rolling 7-day interval to fixed weekly times — TV Mondays 02:00, Movies
|
||||
Tuesdays 02:00 (different days so they never overlap). The seeder only
|
||||
creates rows, never updates a drifted trigger; this rewrites the live rows.
|
||||
Only converts the original interval rows (skips once trigger_type is already
|
||||
weekly_time, so a hand-tuned day/time sticks). Idempotent."""
|
||||
targets = {
|
||||
'video_deep_scan_tv': {'time': '02:00', 'days': ['mon']},
|
||||
'video_deep_scan_movies': {'time': '02:00', 'days': ['tue']},
|
||||
}
|
||||
for action_type, cfg in targets.items():
|
||||
try:
|
||||
auto = self.db.get_system_automation_by_action(action_type)
|
||||
if not auto or auto.get('trigger_type') == 'weekly_time':
|
||||
continue
|
||||
nr_dt = next_run_at('weekly_time', cfg, now_utc=_utcnow(), default_tz=self._default_tz)
|
||||
self.db.update_automation(
|
||||
auto['id'], trigger_type='weekly_time', trigger_config=json.dumps(cfg),
|
||||
next_run=_dt_to_db_str(nr_dt) if nr_dt is not None else None)
|
||||
logger.info("Set '%s' to weekly %s %s (id=%s)", auto.get('name'),
|
||||
cfg['days'], cfg['time'], auto.get('id'))
|
||||
except Exception:
|
||||
logger.exception("deep-scan schedule migration failed for %s", action_type)
|
||||
|
||||
def get_system_automation_next_run_seconds(self, action_type):
|
||||
"""Get seconds until next run for a system automation. Returns 0 if not found or disabled."""
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
"""Boot-phase guard for non-blocking container startup.
|
||||
|
||||
While the gunicorn worker is importing ``web_server`` (module-level client and
|
||||
worker initialization), external provider API probes must not block startup.
|
||||
Network validation is deferred until ``mark_boot_complete()`` runs at the end
|
||||
of that import pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
_boot_lock = threading.Lock()
|
||||
_boot_active = True
|
||||
|
||||
|
||||
def is_boot_phase() -> bool:
|
||||
"""Return True while module import must avoid blocking provider API calls."""
|
||||
with _boot_lock:
|
||||
return _boot_active
|
||||
|
||||
|
||||
def mark_boot_complete() -> None:
|
||||
"""End the boot phase — provider clients may perform network probes again."""
|
||||
global _boot_active
|
||||
with _boot_lock:
|
||||
_boot_active = False
|
||||
|
|
@ -117,48 +117,6 @@ def _is_full_track_payload(payload: Optional[Dict[str, Any]]) -> bool:
|
|||
return 'track_position' in payload and 'contributors' in payload
|
||||
|
||||
|
||||
def resolve_album_track_positions(session, base_url, album_ids, cache=None, sleep_s=0.2):
|
||||
"""Build ``{str(track_id): track_position}`` for a set of Deezer album ids.
|
||||
|
||||
Deezer PLAYLIST and SEARCH track objects (and even the album object's embedded
|
||||
``tracks.data``) omit ``track_position`` — only ``/album/<id>/tracks`` and
|
||||
``/track/<id>`` carry it. So numbering playlist tracks by their playlist index
|
||||
silently poisons the real album track number, which then rides onto the
|
||||
downloaded file's tag. This resolves the authoritative position per album
|
||||
(cache-first, best-effort — a failed album just isn't in the map)."""
|
||||
import time as _time
|
||||
positions: Dict[str, int] = {}
|
||||
for aid in album_ids:
|
||||
aid = str(aid)
|
||||
at_list = None
|
||||
if cache:
|
||||
try:
|
||||
ct = cache.get_entity('deezer', 'album_tracks', aid)
|
||||
if ct and ct.get('data'):
|
||||
at_list = ct['data']
|
||||
except Exception: # noqa: BLE001 - cache is best-effort
|
||||
at_list = None
|
||||
if at_list is None:
|
||||
try:
|
||||
if sleep_s:
|
||||
_time.sleep(sleep_s) # respect Deezer rate limits
|
||||
r = session.get(f"{base_url}/album/{aid}/tracks", params={'limit': 500}, timeout=10)
|
||||
if getattr(r, 'ok', False):
|
||||
at_list = (r.json() or {}).get('data', [])
|
||||
if cache and at_list is not None:
|
||||
try:
|
||||
cache.store_entity('deezer', 'album_tracks', aid, {'data': at_list})
|
||||
except Exception as _cache_err: # noqa: BLE001
|
||||
logger.debug("album_tracks cache store failed for %s: %s", aid, _cache_err)
|
||||
except Exception: # noqa: BLE001 - never let metadata resolution break the fetch
|
||||
at_list = None
|
||||
for at in (at_list or []):
|
||||
tp = at.get('track_position')
|
||||
if at.get('id') and tp:
|
||||
positions[str(at['id'])] = tp
|
||||
return positions
|
||||
|
||||
|
||||
# ==================== Dataclasses (match iTunesClient / SpotifyClient format) ====================
|
||||
|
||||
@dataclass
|
||||
|
|
@ -1399,16 +1357,6 @@ class DeezerClient:
|
|||
|
||||
raw_tracks.extend(page_tracks)
|
||||
|
||||
# Real album track positions — playlist tracks don't carry track_position,
|
||||
# so numbering by playlist index would poison the downloaded file's tag.
|
||||
album_ids = {str(t.get('album', {}).get('id')) for t in raw_tracks if t.get('album', {}).get('id')}
|
||||
try:
|
||||
from core.metadata.cache import get_metadata_cache
|
||||
_cache = get_metadata_cache()
|
||||
except Exception:
|
||||
_cache = None
|
||||
track_positions = resolve_album_track_positions(self.session, self.BASE_URL, album_ids, _cache)
|
||||
|
||||
# Normalize tracks
|
||||
tracks: List[Dict[str, Any]] = []
|
||||
for i, t in enumerate(raw_tracks, start=1):
|
||||
|
|
@ -1420,8 +1368,7 @@ class DeezerClient:
|
|||
'artists': [artist_name],
|
||||
'album': t.get('album', {}).get('title', ''),
|
||||
'duration_ms': t.get('duration', 0) * 1000,
|
||||
# REAL album position; the playlist index is a last resort only.
|
||||
'track_number': track_positions.get(str(t.get('id'))) or i,
|
||||
'track_number': i,
|
||||
})
|
||||
|
||||
result = {
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ from typing import Any, Dict, List, Optional, Tuple
|
|||
import requests
|
||||
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
from core.quality.source_map import quality_from_deezer, quality_tier_for_source
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("deezer_download")
|
||||
|
|
@ -93,10 +92,7 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
self.download_path = Path(download_path)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Engine reference is populated by set_engine() at registration
|
||||
# time. None until orchestrator wires the registry.
|
||||
|
|
@ -121,20 +117,14 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
self._license_token = None
|
||||
self._user_data = None
|
||||
self._authenticated = False
|
||||
self._pending_arl: Optional[str] = None
|
||||
|
||||
# Quality preference
|
||||
self._quality = quality_tier_for_source('deezer', default='flac')
|
||||
self._quality = config_manager.get('deezer_download.quality', 'flac')
|
||||
|
||||
# Try to authenticate on init if ARL is configured
|
||||
arl = config_manager.get('deezer_download.arl', '')
|
||||
if arl:
|
||||
from core.boot_phase import is_boot_phase
|
||||
if is_boot_phase():
|
||||
self._pending_arl = arl
|
||||
logger.debug("Deezer ARL present — authentication deferred until after boot")
|
||||
else:
|
||||
self._authenticate(arl)
|
||||
self._authenticate(arl)
|
||||
|
||||
logger.info(f"Deezer download client initialized (download path: {self.download_path})")
|
||||
|
||||
|
|
@ -233,66 +223,12 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
return self._authenticated
|
||||
|
||||
def is_authenticated(self) -> bool:
|
||||
if self._pending_arl and not self._authenticated:
|
||||
from core.boot_phase import is_boot_phase
|
||||
if not is_boot_phase():
|
||||
self._authenticate(self._pending_arl)
|
||||
self._pending_arl = None
|
||||
return self._authenticated
|
||||
|
||||
async def check_connection(self) -> bool:
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(None, self.is_available)
|
||||
|
||||
# ─── Playlist export (#945) ──────────────────────────────────
|
||||
#
|
||||
# UNOFFICIAL: rides the private gw-light gateway with the ARL session already used
|
||||
# for downloads. Deezer shut their public developer API, so this is the only write
|
||||
# path — and it's fragile by nature (breaks when Deezer changes internals).
|
||||
|
||||
def create_or_update_playlist(self, name, track_ids, *, existing_id=None,
|
||||
public=False, description=""):
|
||||
"""Create a Deezer playlist (or append to an existing one) from a mirrored
|
||||
playlist's tracks. ``track_ids`` are stored ``deezer_id`` values per library track.
|
||||
``existing_id`` set → add to that playlist (idempotent re-export reuses the stored
|
||||
target); unset → create a new one. Returns
|
||||
``{success, playlist_id, url, added, error}``."""
|
||||
if not self._authenticated:
|
||||
return {"success": False, "error": "Deezer is not connected (ARL)"}
|
||||
song_ids = [str(t) for t in (track_ids or []) if t]
|
||||
if not song_ids:
|
||||
return {"success": False, "error": "No matching Deezer tracks to export"}
|
||||
try:
|
||||
songs = [[sid, i] for i, sid in enumerate(song_ids)]
|
||||
if existing_id:
|
||||
res = self._gw_call("playlist.addSongs",
|
||||
{"playlist_id": int(existing_id), "songs": songs})
|
||||
if res is None:
|
||||
return {"success": False, "error": "Deezer rejected the playlist update"}
|
||||
playlist_id = existing_id
|
||||
else:
|
||||
res = self._gw_call("playlist.create", {
|
||||
"title": name, "description": description,
|
||||
"is_public": bool(public), "songs": songs,
|
||||
})
|
||||
if res is None:
|
||||
return {"success": False, "error": "Deezer rejected the playlist create"}
|
||||
# gw 'playlist.create' returns the new playlist id (int) as `results`.
|
||||
if isinstance(res, dict):
|
||||
playlist_id = res.get("PLAYLIST_ID") or res.get("id")
|
||||
else:
|
||||
playlist_id = res
|
||||
if not playlist_id:
|
||||
return {"success": False, "error": "Deezer did not return a playlist id"}
|
||||
return {
|
||||
"success": True,
|
||||
"playlist_id": str(playlist_id),
|
||||
"url": f"https://www.deezer.com/playlist/{playlist_id}",
|
||||
"added": len(song_ids),
|
||||
}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def reconnect(self, arl: str = None) -> bool:
|
||||
"""Re-authenticate with a new or existing ARL."""
|
||||
if arl is None:
|
||||
|
|
@ -481,12 +417,6 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
if aid:
|
||||
album_ids.add(str(aid))
|
||||
album_release_dates = {}
|
||||
# Deezer PLAYLIST tracks do NOT carry `track_position` (only `/track/<id>`
|
||||
# and `/album/<id>/tracks` do), so numbering them by their playlist index
|
||||
# poisons the real album track number — which then rides into the wishlist
|
||||
# and onto the downloaded file's tag (e.g. 'Apologize' tagged track 1 instead
|
||||
# of 16). Resolve the REAL position from each album's track list (cache-first).
|
||||
track_positions: Dict[str, int] = {} # str(track_id) -> album track_position
|
||||
try:
|
||||
from core.metadata.cache import get_metadata_cache
|
||||
cache = get_metadata_cache()
|
||||
|
|
@ -499,32 +429,24 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
cached = cache.get_entity('deezer', 'album', aid)
|
||||
if cached and cached.get('release_date'):
|
||||
album_release_dates[aid] = cached['release_date']
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("cache get_entity album release_date: %s", e)
|
||||
# Cache miss — fetch from API
|
||||
if aid not in album_release_dates:
|
||||
try:
|
||||
time.sleep(0.3) # Respect rate limits
|
||||
a_resp = self._session.get(f'https://api.deezer.com/album/{aid}', timeout=10)
|
||||
if a_resp.ok:
|
||||
a_data = a_resp.json()
|
||||
album_release_dates[aid] = a_data.get('release_date', '')
|
||||
# Store in metadata cache for future use
|
||||
if cache:
|
||||
try:
|
||||
cache.store_entity('deezer', 'album', aid, a_data)
|
||||
except Exception as e:
|
||||
logger.debug("cache store_entity album release_date: %s", e)
|
||||
except Exception as e:
|
||||
logger.debug("fetch deezer album release_date %s: %s", aid, e)
|
||||
# Real album track positions (separate endpoint — playlist tracks AND the
|
||||
# album object's embedded tracks both omit track_position). Cache-first.
|
||||
try:
|
||||
from core.deezer_client import resolve_album_track_positions
|
||||
track_positions = resolve_album_track_positions(
|
||||
self._session, 'https://api.deezer.com', album_ids, cache)
|
||||
except Exception as e:
|
||||
logger.debug("resolve deezer album track positions: %s", e)
|
||||
try:
|
||||
time.sleep(0.3) # Respect rate limits
|
||||
a_resp = self._session.get(f'https://api.deezer.com/album/{aid}', timeout=10)
|
||||
if a_resp.ok:
|
||||
a_data = a_resp.json()
|
||||
album_release_dates[aid] = a_data.get('release_date', '')
|
||||
# Store in metadata cache for future use
|
||||
if cache:
|
||||
try:
|
||||
cache.store_entity('deezer', 'album', aid, a_data)
|
||||
except Exception as e:
|
||||
logger.debug("cache store_entity album release_date: %s", e)
|
||||
except Exception as e:
|
||||
logger.debug("fetch deezer album release_date %s: %s", aid, e)
|
||||
|
||||
tracks = []
|
||||
for i, t in enumerate(raw_tracks, start=1):
|
||||
|
|
@ -545,9 +467,7 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
'id': album_id,
|
||||
},
|
||||
'duration_ms': t.get('duration', 0) * 1000,
|
||||
# REAL album position (resolved above); the playlist index is a last
|
||||
# resort only when the album lookup failed, never the default.
|
||||
'track_number': track_positions.get(str(t.get('id'))) or t.get('track_position') or i,
|
||||
'track_number': i,
|
||||
})
|
||||
|
||||
return {
|
||||
|
|
@ -661,7 +581,7 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
bitrate = 128
|
||||
quality = 'mp3'
|
||||
|
||||
tr = TrackResult(
|
||||
results.append(TrackResult(
|
||||
username='deezer_dl',
|
||||
filename=f"{track_id}||{artist} - {title}",
|
||||
size=est_size,
|
||||
|
|
@ -675,10 +595,7 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
title=title,
|
||||
album=album,
|
||||
track_number=item.get('track_position'),
|
||||
)
|
||||
# Stamp CD-quality FLAC (16/44.1) so lossless ranks correctly.
|
||||
tr.set_quality(quality_from_deezer(self._quality))
|
||||
results.append(tr)
|
||||
))
|
||||
|
||||
logger.info(f"Deezer search for '{query}' returned {len(results)} results")
|
||||
return results, []
|
||||
|
|
|
|||
|
|
@ -1,340 +0,0 @@
|
|||
"""Listening-driven recommendation core (#913).
|
||||
|
||||
PURE, side-effect-free ranking that turns "the artists you listen to most" plus
|
||||
"who's similar to each" into:
|
||||
|
||||
1. a consensus-ranked list of artists you'd probably love but don't own, and
|
||||
2. an aggregated candidate-track list for a generated playlist.
|
||||
|
||||
No DB / network / config here. The caller (the watchlist scanner) supplies the
|
||||
seeds (top-played artists), the ``similar_artists`` rows per seed, and the
|
||||
owned-artist set, then fetches top tracks for the winners. Keeping the decision
|
||||
logic in one pure place makes it fully unit-testable without the live stack and
|
||||
keeps the scan wiring thin — and additive, so it can't disturb existing flows.
|
||||
|
||||
Scoring rationale (the "best in class" bit): a recommended artist's score is
|
||||
``Σ over the seeds that recommend it of (seed_weight × similarity)``. That single
|
||||
sum rewards all three signals at once — **consensus** (an artist endorsed by many
|
||||
of your seeds accumulates more terms), your **play weight** (heavier seeds push
|
||||
harder), and **similarity strength** — instead of a flat "appears in N lists".
|
||||
``seed_count`` is exposed separately for display ("because you like A, B, C") and
|
||||
as the adventurousness dial's lever (``min_seed_count``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Sequence, Set
|
||||
|
||||
|
||||
def _norm(name: object) -> str:
|
||||
return str(name or "").strip().lower()
|
||||
|
||||
|
||||
def _positive_float(value: object, default: float = 1.0) -> float:
|
||||
try:
|
||||
f = float(value) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return f if f > 0 else default
|
||||
|
||||
|
||||
def _get(row: object, attr: str):
|
||||
"""Read a field from a dataclass row or a dict row."""
|
||||
if isinstance(row, dict):
|
||||
return row.get(attr)
|
||||
return getattr(row, attr, None)
|
||||
|
||||
|
||||
def choose_mix_fetch_source(active_source: object, active_can_fetch: bool) -> str:
|
||||
"""Pick which source to fetch the "Listening Mix" top tracks from.
|
||||
|
||||
The mix is a list of (artist, title) pairs acquired via Soulseek, so the fetch source need
|
||||
NOT match the user's active metadata source. Use the active source when it can fetch top
|
||||
tracks itself (Spotify/Deezer); otherwise fall back to Deezer, whose public ``artist/{id}/top``
|
||||
needs no auth and is available to every user — so iTunes / Discogs / MusicBrainz users still
|
||||
get a full mix without switching sources. Pure.
|
||||
"""
|
||||
if str(active_source or "").lower() in ("spotify", "deezer") and active_can_fetch:
|
||||
return str(active_source).lower()
|
||||
return "deezer"
|
||||
|
||||
|
||||
def names_match(a: object, b: object) -> bool:
|
||||
"""Strict artist-name equality after stripping case + non-alphanumerics.
|
||||
|
||||
Used to verify a name-search result before fetching that artist's top tracks, so the
|
||||
"Listening Mix" can never pull the WRONG artist's songs (e.g. a same-name act). Exact
|
||||
alphanumeric match: "Tyler, The Creator" == "Tyler The Creator", but "Drake" != "Drake Bell".
|
||||
Pure.
|
||||
"""
|
||||
def _alnum(x: object) -> str:
|
||||
return "".join(ch for ch in str(x or "").lower() if ch.isalnum())
|
||||
na, nb = _alnum(a), _alnum(b)
|
||||
return bool(na) and na == nb
|
||||
|
||||
|
||||
def similarity_from_rank(rank: object, max_rank: int = 10) -> float:
|
||||
"""Turn a stored ``similarity_rank`` (1 = most similar … 10 = least) into a 0–1 weight.
|
||||
|
||||
SoulSync stores each ``(seed → similar)`` edge with a 1–10 rank (``1`` is the closest
|
||||
match). The ranker multiplies this into the score so a seed's *closest* matches count
|
||||
for more than its long-tail ones. Linear decay over the documented range: rank 1 → 1.0,
|
||||
rank 5 → 0.6, rank 10 → 0.1, with a 0.1 floor so a far match still contributes. A
|
||||
missing/garbage rank falls back to 1.0 (treat as "no rank info, full weight"). Pure.
|
||||
"""
|
||||
try:
|
||||
r = int(rank)
|
||||
except (TypeError, ValueError):
|
||||
return 1.0
|
||||
floor = round(1.0 / max_rank, 4)
|
||||
if r <= 1:
|
||||
return 1.0
|
||||
if r >= max_rank:
|
||||
return floor
|
||||
return round((max_rank - r + 1) / max_rank, 4)
|
||||
|
||||
|
||||
def build_recency_weighted_seeds(
|
||||
top_artists: Sequence[dict],
|
||||
recent_play_counts: Optional[Dict[str, float]] = None,
|
||||
*,
|
||||
recency_factor: float = 1.5,
|
||||
) -> List[dict]:
|
||||
"""Blend lifetime + recent play counts into seed weights — "what you're into NOW".
|
||||
|
||||
``weight = lifetime_plays + recency_factor × recent_plays``. An artist you've played a
|
||||
lot *recently* outranks one you played a lot years ago, so the recommendations track
|
||||
your current taste instead of your all-time history. ``recency_factor`` is the dial
|
||||
(0 = pure lifetime). Returns ``[{'name', 'weight'}]`` for :func:`rank_recommended_artists`.
|
||||
Pure — the caller supplies both play-count maps from the listening history.
|
||||
"""
|
||||
recent = {_norm(k): _positive_float(v, 0.0) for k, v in (recent_play_counts or {}).items()}
|
||||
out: List[dict] = []
|
||||
for a in top_artists or ():
|
||||
name = str(a.get("name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
lifetime = _positive_float(a.get("play_count", a.get("weight", 1.0)))
|
||||
boost = recency_factor * recent.get(_norm(name), 0.0)
|
||||
out.append({"name": name, "weight": lifetime + boost})
|
||||
return out
|
||||
|
||||
|
||||
def group_similars_by_seed(
|
||||
seeds: Sequence[dict],
|
||||
similar_rows: Sequence,
|
||||
id_to_name: Dict[str, str],
|
||||
*,
|
||||
source_id_attr: str = "source_artist_id",
|
||||
similar_name_attr: str = "similar_artist_name",
|
||||
rank_attr: Optional[str] = None,
|
||||
) -> Dict[str, List[dict]]:
|
||||
"""Reshape flat ``similar_artists`` rows into ``{seed_name_lower: [{'name', 'score'?}]}``.
|
||||
|
||||
The stored rows key the similar artist by the SEED's source id (``source_artist_id``),
|
||||
not its name, so :func:`rank_recommended_artists` can't consume them directly. This
|
||||
resolves each row's source id to a name via ``id_to_name`` (``{source_artist_id:
|
||||
artist_name}`` for the library, built by the caller) and keeps only rows that resolve
|
||||
to one of the ``seeds``. Rows may be dataclass objects or dicts. Pure — no I/O.
|
||||
|
||||
``id_to_name`` MUST be keyed by whatever id the edges actually store — for SoulSync that
|
||||
is the artist's SOURCE id (Spotify/iTunes/Deezer/MusicBrainz), NOT the internal row id.
|
||||
When ``rank_attr`` is given, each row's rank is converted via :func:`similarity_from_rank`
|
||||
and carried as ``score`` so closer matches weigh more; without it every similar comes out
|
||||
score-less (the ranker then treats similarity as 1.0 — original behavior).
|
||||
"""
|
||||
seed_names = {_norm(s.get("name")) for s in seeds}
|
||||
seed_names.discard("")
|
||||
id_to_norm = {str(k): _norm(v) for k, v in (id_to_name or {}).items()}
|
||||
|
||||
out: Dict[str, List[dict]] = {}
|
||||
for row in similar_rows or ():
|
||||
seed_name = id_to_norm.get(str(_get(row, source_id_attr) or ""), "")
|
||||
if not seed_name or seed_name not in seed_names:
|
||||
continue
|
||||
sim_name = str(_get(row, similar_name_attr) or "").strip()
|
||||
if not sim_name:
|
||||
continue
|
||||
entry = {"name": sim_name}
|
||||
if rank_attr is not None:
|
||||
entry["score"] = similarity_from_rank(_get(row, rank_attr))
|
||||
out.setdefault(seed_name, []).append(entry)
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecommendedArtist:
|
||||
"""One artist recommended from your listening, with the why."""
|
||||
name: str # display name (first-seen casing)
|
||||
score: float # Σ seed_weight × similarity
|
||||
seed_count: int # distinct seeds endorsing it (consensus)
|
||||
seeds: List[str] = field(default_factory=list) # display names of those seeds
|
||||
|
||||
|
||||
def rank_recommended_artists(
|
||||
seeds: Sequence[dict],
|
||||
similars_by_seed: Dict[str, Sequence[dict]],
|
||||
owned_artist_names: Optional[Set[str]] = None,
|
||||
*,
|
||||
limit: int = 30,
|
||||
min_seed_count: int = 1,
|
||||
) -> List[RecommendedArtist]:
|
||||
"""Rank artists similar to your most-played by consensus + play weight + similarity.
|
||||
|
||||
Args:
|
||||
seeds: ``[{'name': str, 'weight': float}]`` — your top-played artists.
|
||||
``weight`` (play count or any positive number) defaults to 1.0.
|
||||
similars_by_seed: ``{seed_name_lower: [{'name': str, 'score': float}]}`` — the
|
||||
similar-artist rows for each seed. ``score`` is optional (defaults 1.0).
|
||||
owned_artist_names: lowercased names already in the library — excluded so the
|
||||
result is artists you DON'T have. The seeds themselves are always excluded.
|
||||
limit: max results.
|
||||
min_seed_count: drop recommendations endorsed by fewer than N seeds — the
|
||||
adventurousness dial's "Safer" end raises this for higher-confidence picks.
|
||||
|
||||
Returns up to ``limit`` :class:`RecommendedArtist`, highest score first.
|
||||
"""
|
||||
owned = {_norm(a) for a in (owned_artist_names or set())}
|
||||
seed_norms = {_norm(s.get("name")) for s in seeds}
|
||||
seed_norms.discard("")
|
||||
exclude = owned | seed_norms
|
||||
|
||||
acc: Dict[str, dict] = {}
|
||||
for seed in seeds:
|
||||
s_name = _norm(seed.get("name"))
|
||||
if not s_name:
|
||||
continue
|
||||
s_display = str(seed.get("name") or "").strip()
|
||||
weight = _positive_float(seed.get("weight", 1.0))
|
||||
for sim in similars_by_seed.get(s_name, ()) or ():
|
||||
a_norm = _norm(sim.get("name"))
|
||||
if not a_norm or a_norm in exclude:
|
||||
continue
|
||||
sim_score = _positive_float(sim.get("score", 1.0))
|
||||
row = acc.setdefault(
|
||||
a_norm, {"name": str(sim.get("name") or "").strip(), "score": 0.0, "seeds": {}}
|
||||
)
|
||||
row["score"] += weight * sim_score
|
||||
row["seeds"].setdefault(s_name, s_display) # one seed counts once
|
||||
|
||||
out: List[RecommendedArtist] = []
|
||||
floor = max(1, int(min_seed_count))
|
||||
for row in acc.values():
|
||||
seed_count = len(row["seeds"])
|
||||
if seed_count < floor:
|
||||
continue
|
||||
out.append(RecommendedArtist(
|
||||
name=row["name"],
|
||||
score=round(row["score"], 6),
|
||||
seed_count=seed_count,
|
||||
seeds=list(row["seeds"].values()),
|
||||
))
|
||||
out.sort(key=lambda r: (-r.score, -r.seed_count, r.name.lower()))
|
||||
return out[:limit]
|
||||
|
||||
|
||||
def aggregate_candidate_tracks(
|
||||
recommended_artists: Sequence[RecommendedArtist],
|
||||
top_tracks_by_artist: Dict[str, Sequence[dict]],
|
||||
owned_track_keys: Optional[Set] = None,
|
||||
*,
|
||||
per_artist: int = 3,
|
||||
limit: int = 50,
|
||||
exclude_owned: bool = True,
|
||||
) -> List[dict]:
|
||||
"""Build the candidate track list for the generated playlist.
|
||||
|
||||
Takes the top ``per_artist`` tracks from each recommended artist **in artist-rank
|
||||
order**, dedups by ``(artist, title)``, optionally drops owned tracks (the
|
||||
"discovery" flavor) and caps at ``limit``. Each returned track dict is the source
|
||||
track plus ``_seed_artist`` (which recommended artist it came from).
|
||||
|
||||
Args:
|
||||
recommended_artists: ranked output of :func:`rank_recommended_artists`.
|
||||
top_tracks_by_artist: ``{artist_name_lower: [track_dict, ...]}`` — fetched by
|
||||
the caller (Last.fm / source top tracks), NOT limited to a curated pool.
|
||||
owned_track_keys: set of ``(artist_lower, title_lower)`` already in the library.
|
||||
exclude_owned: drop tracks in ``owned_track_keys`` (discovery flavor). Set False
|
||||
for a "replay" playlist of tracks you already own.
|
||||
"""
|
||||
owned = owned_track_keys or set()
|
||||
seen: Set = set()
|
||||
out: List[dict] = []
|
||||
for art in recommended_artists:
|
||||
tracks = top_tracks_by_artist.get(_norm(art.name), ()) or ()
|
||||
taken = 0
|
||||
for t in tracks:
|
||||
if taken >= per_artist:
|
||||
break
|
||||
title = str(t.get("name") or t.get("title") or "").strip()
|
||||
if not title:
|
||||
continue
|
||||
key = (_norm(art.name), _norm(title))
|
||||
if key in seen:
|
||||
continue
|
||||
if exclude_owned and key in owned:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append({**t, "_seed_artist": art.name})
|
||||
taken += 1
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out[:limit]
|
||||
|
||||
|
||||
def to_mix_track(track: object, source: str) -> Optional[dict]:
|
||||
"""Shape one source "top tracks" API dict into the flat dict the Discover compact
|
||||
playlist row renders + syncs (the "Listening Mix" #913 playlist).
|
||||
|
||||
Spotify's ``artist_top_tracks`` and Deezer's ``get_artist_top_tracks`` both return the
|
||||
same Spotify-shape object (``id, name, artists[], album{name,images[]}, duration_ms``).
|
||||
This flattens that into the renderer's field names (``track_name/artist_name/album_name/
|
||||
album_cover_url/duration_ms``), keeps the original under ``track_data_json`` for sync, and
|
||||
sets the source-specific id field. Returns None for anything without a usable id/title so
|
||||
the caller can filter. A ``name`` key is kept so :func:`aggregate_candidate_tracks` can
|
||||
dedup by title. Pure — no I/O.
|
||||
"""
|
||||
if not isinstance(track, dict):
|
||||
return None
|
||||
tid = track.get("id")
|
||||
name = str(track.get("name") or "").strip()
|
||||
if not tid or not name:
|
||||
return None
|
||||
artists = track.get("artists") or []
|
||||
artist_name = ""
|
||||
if artists and isinstance(artists[0], dict):
|
||||
artist_name = str(artists[0].get("name") or "").strip()
|
||||
album = track.get("album") if isinstance(track.get("album"), dict) else {}
|
||||
album_name = str(album.get("name") or "").strip()
|
||||
images = album.get("images") or []
|
||||
cover = images[0].get("url") if images and isinstance(images[0], dict) else None
|
||||
out = {
|
||||
"track_id": str(tid),
|
||||
"name": name, # for aggregate_candidate_tracks dedup
|
||||
"track_name": name, # for the renderer
|
||||
"artist_name": artist_name,
|
||||
"album_name": album_name,
|
||||
"album_cover_url": cover,
|
||||
"duration_ms": track.get("duration_ms") or 0,
|
||||
"track_data_json": track, # full payload for sync/download
|
||||
"source": source,
|
||||
}
|
||||
id_field = {"spotify": "spotify_track_id", "deezer": "deezer_track_id",
|
||||
"itunes": "itunes_track_id"}.get(source)
|
||||
if id_field:
|
||||
out[id_field] = str(tid)
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RecommendedArtist",
|
||||
"choose_mix_fetch_source",
|
||||
"names_match",
|
||||
"similarity_from_rank",
|
||||
"build_recency_weighted_seeds",
|
||||
"to_mix_track",
|
||||
"group_similars_by_seed",
|
||||
"rank_recommended_artists",
|
||||
"aggregate_candidate_tracks",
|
||||
]
|
||||
|
|
@ -21,11 +21,7 @@ from core.metadata.registry import get_client_for_source
|
|||
from core.metadata.types import Album
|
||||
from core.wishlist.payloads import ensure_wishlist_track_format
|
||||
|
||||
# Use the project logger namespace ("soulsync.*") so the scanner's progress and
|
||||
# diagnostics actually surface in the app log — plain getLogger(__name__) lands
|
||||
# under "core.discovery.quality_scanner", which the app log view doesn't show.
|
||||
from utils.logging_config import get_logger
|
||||
logger = get_logger("discovery.quality_scanner")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Per-source typed converter dispatch — same registry pattern as
|
||||
|
|
|
|||
|
|
@ -173,32 +173,6 @@ async def _database_only_find_track(spotify_track, candidate_pool=None):
|
|||
logger.debug("sync match cache fast-path failed: %s", e)
|
||||
# --- End cache fast-path ---
|
||||
|
||||
# Durable manual library match (#787) — survives a library rescan (the
|
||||
# sync_match_cache above does not), so a user's Find & Add pairing keeps
|
||||
# sticking across auto-syncs instead of being re-matched from scratch (#895
|
||||
# follow-up). Self-heals a stale library id via the stored file path.
|
||||
if spotify_id:
|
||||
try:
|
||||
from core.artists.map import get_current_profile_id
|
||||
m = db.find_manual_library_match_by_source_track_id(
|
||||
get_current_profile_id(), str(spotify_id), active_server)
|
||||
if m:
|
||||
lib_id = m.get('library_track_id')
|
||||
dt = db.get_track_by_id(lib_id) if lib_id is not None else None
|
||||
if not dt and m.get('library_file_path'):
|
||||
new_id = db.find_track_id_by_file_path(m['library_file_path'])
|
||||
dt = db.get_track_by_id(new_id) if new_id else None
|
||||
if dt:
|
||||
class DatabaseTrackDurable:
|
||||
def __init__(self, db_t):
|
||||
self.ratingKey = db_t.id
|
||||
self.title = db_t.title
|
||||
self.id = db_t.id
|
||||
logger.debug(f"Durable manual match hit: '{original_title}' → {lib_id}")
|
||||
return DatabaseTrackDurable(dt), 1.0
|
||||
except Exception as e:
|
||||
logger.debug("durable manual match fast-path failed: %s", e)
|
||||
|
||||
# Try each artist
|
||||
for artist in spotify_track.artists:
|
||||
if isinstance(artist, str):
|
||||
|
|
@ -307,11 +281,35 @@ def run_sync_task(
|
|||
# This avoids needing to re-fetch it from Spotify
|
||||
logger.info("Converting JSON tracks to SpotifyTrack objects...")
|
||||
|
||||
# Store original track data with full album objects (for wishlist with cover art).
|
||||
# Shared with the sync-detail "re-add to wishlist" action so both build the
|
||||
# IDENTICAL payload (album→dict + images, artists→dicts). Copy-safe.
|
||||
from core.sync.wishlist_readd import build_original_tracks_map
|
||||
original_tracks_map = build_original_tracks_map(tracks_json)
|
||||
# Store original track data with full album objects (for wishlist with cover art)
|
||||
# Normalize formats for wishlist: album must be dict {'name': ...}, artists must be [{'name': ...}]
|
||||
# Important: copy data — don't mutate tracks_json since SpotifyTrack expects List[str] artists
|
||||
original_tracks_map = {}
|
||||
for t in tracks_json:
|
||||
track_id = t.get('id', '')
|
||||
if track_id:
|
||||
normalized = dict(t)
|
||||
# Normalize album to dict format, preserving images and metadata
|
||||
raw_album = normalized.get('album', '')
|
||||
if isinstance(raw_album, str):
|
||||
normalized['album'] = {
|
||||
'name': raw_album or normalized.get('name', 'Unknown Album'),
|
||||
'images': [], 'album_type': 'single', 'total_tracks': 1, 'release_date': ''
|
||||
}
|
||||
elif not isinstance(raw_album, dict):
|
||||
normalized['album'] = {
|
||||
'name': str(raw_album) if raw_album else normalized.get('name', 'Unknown Album'),
|
||||
'images': [], 'album_type': 'single', 'total_tracks': 1, 'release_date': ''
|
||||
}
|
||||
else:
|
||||
# Dict — ensure required keys exist
|
||||
raw_album.setdefault('name', 'Unknown Album')
|
||||
raw_album.setdefault('images', [])
|
||||
# Normalize artists to list of dicts
|
||||
raw_artists = normalized.get('artists', [])
|
||||
if raw_artists and isinstance(raw_artists[0], str):
|
||||
normalized['artists'] = [{'name': a} for a in raw_artists]
|
||||
original_tracks_map[track_id] = normalized
|
||||
|
||||
tracks = []
|
||||
for i, t in enumerate(tracks_json):
|
||||
|
|
|
|||
|
|
@ -32,26 +32,6 @@ from typing import Any, Callable
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_UNKNOWN_ARTIST = 'Unknown Artist'
|
||||
|
||||
|
||||
def resolve_display_artist(yt_artist: str, matched_artist: str) -> str:
|
||||
"""The artist to show in the 'YT Artist' column (#909).
|
||||
|
||||
YouTube's flat playlist data carries no artist, so a track starts as
|
||||
"Unknown Artist" and only gains a real name if per-video recovery succeeds.
|
||||
When recovery comes up empty but the track still matched confidently, show
|
||||
the matched artist instead of a misleading "Unknown Artist". Returns the
|
||||
original ``yt_artist`` whenever it's already a real name (recovery worked) or
|
||||
when there's no matched artist to fall back to — purely a display choice, the
|
||||
match itself is unaffected.
|
||||
"""
|
||||
current = (yt_artist or '').strip()
|
||||
if current and current != _UNKNOWN_ARTIST:
|
||||
return current # recovery already gave a real name — keep it
|
||||
fallback = (matched_artist or '').strip()
|
||||
return fallback or _UNKNOWN_ARTIST # backfill from the match, else honest Unknown
|
||||
|
||||
|
||||
@dataclass
|
||||
class YoutubeDiscoveryDeps:
|
||||
|
|
@ -151,15 +131,14 @@ def run_youtube_discovery_worker(url_hash, deps: YoutubeDiscoveryDeps):
|
|||
cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source)
|
||||
if cached_match and deps.validate_discovery_cache_artist(cleaned_artist, cached_match):
|
||||
logger.debug(f"CACHE HIT [{i+1}/{len(tracks)}]: {cleaned_artist} - {cleaned_title}")
|
||||
_match_artist = deps.extract_artist_name(cached_match.get('artists', [''])[0]) if cached_match.get('artists') else ''
|
||||
result = {
|
||||
'index': i,
|
||||
'yt_track': cleaned_title,
|
||||
'yt_artist': resolve_display_artist(cleaned_artist, _match_artist),
|
||||
'yt_artist': cleaned_artist,
|
||||
'status': 'Found',
|
||||
'status_class': 'found',
|
||||
'spotify_track': cached_match.get('name', ''),
|
||||
'spotify_artist': _match_artist,
|
||||
'spotify_artist': deps.extract_artist_name(cached_match.get('artists', [''])[0]) if cached_match.get('artists') else '',
|
||||
'spotify_album': cached_match.get('album', {}).get('name', '') if isinstance(cached_match.get('album'), dict) else cached_match.get('album', ''),
|
||||
'duration': f"{int(track['duration_ms']) // 60000}:{(int(track['duration_ms']) % 60000) // 1000:02d}" if track['duration_ms'] else '0:00',
|
||||
'discovery_source': discovery_source,
|
||||
|
|
@ -286,17 +265,15 @@ def run_youtube_discovery_worker(url_hash, deps: YoutubeDiscoveryDeps):
|
|||
best_confidence = confidence
|
||||
logger.info(f"Strategy 4 YouTube match (extended): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")
|
||||
|
||||
# Create result entry. yt_artist falls back to the matched artist when
|
||||
# YouTube/recovery left it "Unknown Artist" but we matched confidently (#909).
|
||||
_match_artist = deps.extract_artist_name(matched_track.artists[0]) if matched_track else ''
|
||||
# Create result entry
|
||||
result = {
|
||||
'index': i,
|
||||
'yt_track': cleaned_title,
|
||||
'yt_artist': resolve_display_artist(cleaned_artist, _match_artist),
|
||||
'yt_artist': cleaned_artist,
|
||||
'status': 'Found' if matched_track else 'Not Found',
|
||||
'status_class': 'found' if matched_track else 'not-found',
|
||||
'spotify_track': matched_track.name if matched_track else '',
|
||||
'spotify_artist': _match_artist,
|
||||
'spotify_artist': deps.extract_artist_name(matched_track.artists[0]) if matched_track else '',
|
||||
'spotify_album': matched_track.album if matched_track else '',
|
||||
'duration': f"{int(track['duration_ms']) // 60000}:{(int(track['duration_ms']) % 60000) // 1000:02d}" if track['duration_ms'] else '0:00',
|
||||
'discovery_source': discovery_source,
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ big-bang switchover.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
||||
|
||||
|
|
@ -392,16 +391,6 @@ class DownloadEngine:
|
|||
(tracks, albums) tuple, or ``([], [])`` when every source
|
||||
in the chain is exhausted.
|
||||
|
||||
Priority mode is deliberately quality-AGNOSTIC at search time — source
|
||||
order is king and the first source that returns any tracks wins, exactly
|
||||
matching pre-quality-system behaviour byte-for-byte (#896 review #3).
|
||||
Quality-gating the priority path would deprioritise e.g. a soulseek
|
||||
mp3 whose bitrate slskd omitted (``bitrate=None`` → "unsatisfied"),
|
||||
changing which source wins and adding latency for users who never opted
|
||||
in. Cross-source quality pooling is the job of best_quality mode
|
||||
(``search_all_sources``); final per-result ranking still happens in the
|
||||
orchestrator's match/quality filter. RAW tracks are returned.
|
||||
|
||||
Replaces orchestrator's hand-rolled hybrid search loop. The
|
||||
chain is ordered (most-preferred first).
|
||||
"""
|
||||
|
|
@ -417,10 +406,9 @@ class DownloadEngine:
|
|||
try:
|
||||
logger.info(f"Trying {source_name} (priority {i+1}): {query}")
|
||||
tracks, albums = await plugin.search(query, timeout, progress_callback)
|
||||
if not tracks:
|
||||
continue
|
||||
logger.info(f"{source_name} found {len(tracks)} tracks")
|
||||
return (tracks, albums)
|
||||
if tracks:
|
||||
logger.info(f"{source_name} found {len(tracks)} tracks")
|
||||
return (tracks, albums)
|
||||
except Exception as e:
|
||||
logger.warning(f"{source_name} search failed: {e}")
|
||||
|
||||
|
|
@ -430,75 +418,6 @@ class DownloadEngine:
|
|||
)
|
||||
return ([], [])
|
||||
|
||||
async def search_all_sources(self, query: str, source_chain,
|
||||
timeout=None, progress_callback=None,
|
||||
exclude_sources=None):
|
||||
"""Best-quality mode: pool RAW tracks from EVERY configured source in
|
||||
``source_chain`` instead of stopping at the first satisfying one.
|
||||
|
||||
Unlike :meth:`search_with_fallback`, no source short-circuits the
|
||||
search — the caller (orchestrator/worker) ranks the combined pool
|
||||
best→worst by actual audio quality. ``exclude_sources`` drops sources
|
||||
whose per-source retry budget is already spent (so their candidates
|
||||
never re-enter the pool). Unconfigured / unregistered / raising sources
|
||||
are skipped exactly like the fallback path. Returns
|
||||
``(combined_tracks, combined_albums)``.
|
||||
"""
|
||||
excluded = {s.lower() for s in (exclude_sources or []) if s}
|
||||
pooled_tracks = []
|
||||
pooled_albums = []
|
||||
# Per-source contribution for an honest pool log — e.g. a release-level
|
||||
# source like usenet/torrent that returns nothing for a track-title
|
||||
# query should read "usenet=0", not silently hide behind the chain name.
|
||||
contributions = []
|
||||
|
||||
# Decide which sources to actually query, recording why the rest were
|
||||
# skipped. Searches then run CONCURRENTLY so the pool waits only for the
|
||||
# slowest source (e.g. usenet/Prowlarr, which can be slow) rather than
|
||||
# the sum of every source's latency.
|
||||
to_search = [] # (source_name, plugin)
|
||||
for source_name in source_chain:
|
||||
if source_name.lower() in excluded:
|
||||
contributions.append(f"{source_name}=excluded")
|
||||
continue
|
||||
plugin = self._plugins.get(source_name)
|
||||
if plugin is None:
|
||||
logger.info(f"Skipping {source_name} (not available)")
|
||||
contributions.append(f"{source_name}=unavailable")
|
||||
continue
|
||||
if hasattr(plugin, 'is_configured') and not plugin.is_configured():
|
||||
logger.info(f"Skipping {source_name} (not configured)")
|
||||
contributions.append(f"{source_name}=unconfigured")
|
||||
continue
|
||||
to_search.append((source_name, plugin))
|
||||
|
||||
async def _one(plugin):
|
||||
return await plugin.search(query, timeout, progress_callback)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*[_one(plugin) for _, plugin in to_search],
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
for (source_name, _), result in zip(to_search, results, strict=True):
|
||||
if isinstance(result, Exception):
|
||||
logger.warning(f"{source_name} search failed: {result}")
|
||||
contributions.append(f"{source_name}=error")
|
||||
continue
|
||||
tracks, albums = result
|
||||
n = len(tracks) if tracks else 0
|
||||
if tracks:
|
||||
pooled_tracks.extend(tracks)
|
||||
if albums:
|
||||
pooled_albums.extend(albums)
|
||||
contributions.append(f"{source_name}={n}")
|
||||
|
||||
logger.info(
|
||||
"Best-quality pool: %d candidates [%s] for: %s",
|
||||
len(pooled_tracks), ', '.join(contributions), query,
|
||||
)
|
||||
return (pooled_tracks, pooled_albums)
|
||||
|
||||
async def download_with_fallback(self, username: str, filename: str,
|
||||
file_size: int, source_chain) -> Optional[str]:
|
||||
"""Try each source in ``source_chain`` until one accepts the
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ from config.settings import config_manager
|
|||
from core.download_engine import DownloadEngine
|
||||
from core.download_plugins.registry import DownloadPluginRegistry, build_default_registry
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.quality.selection import load_search_mode
|
||||
|
||||
logger = get_logger("download_orchestrator")
|
||||
|
||||
|
|
@ -103,14 +102,12 @@ class DownloadOrchestrator:
|
|||
deezer_dl = self.client('deezer_dl')
|
||||
if deezer_arl and deezer_dl:
|
||||
deezer_dl.reconnect(deezer_arl)
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
deezer_dl._quality = quality_tier_for_source('deezer', default='flac')
|
||||
deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac')
|
||||
|
||||
# Reload Amazon quality preference (T2Tunes needs no reconnect — public proxy)
|
||||
amazon = self.client('amazon')
|
||||
if amazon:
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
quality = quality_tier_for_source('amazon', default='flac')
|
||||
quality = config_manager.get('amazon_download.quality', 'flac')
|
||||
amazon._quality = quality
|
||||
amazon._allow_fallback = config_manager.get('amazon_download.allow_fallback', True)
|
||||
if hasattr(amazon, '_client') and amazon._client:
|
||||
|
|
@ -143,10 +140,7 @@ class DownloadOrchestrator:
|
|||
continue
|
||||
if hasattr(client, 'download_path') and client.download_path != new_path:
|
||||
client.download_path = new_path
|
||||
try:
|
||||
client.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {new_path}: {e}")
|
||||
client.download_path.mkdir(parents=True, exist_ok=True)
|
||||
# YouTube also caches path in yt-dlp opts
|
||||
if hasattr(client, 'download_opts') and 'outtmpl' in client.download_opts:
|
||||
client.download_opts['outtmpl'] = str(new_path / '%(title)s.%(ext)s')
|
||||
|
|
@ -348,11 +342,6 @@ class DownloadOrchestrator:
|
|||
if not chain:
|
||||
logger.warning("Hybrid search exhausted: no eligible sources after exclusion filter")
|
||||
return [], []
|
||||
if load_search_mode() == 'best_quality':
|
||||
logger.info(f"Best-quality search ({' → '.join(chain)}): {query}")
|
||||
return await self.engine.search_all_sources(
|
||||
query, chain, timeout, progress_callback,
|
||||
)
|
||||
logger.info(f"Hybrid search ({' → '.join(chain)}): {query}")
|
||||
return await self.engine.search_with_fallback(query, chain, timeout, progress_callback)
|
||||
|
||||
|
|
@ -423,17 +412,9 @@ class DownloadOrchestrator:
|
|||
|
||||
if scored:
|
||||
scored.sort(key=lambda x: x._match_confidence, reverse=True)
|
||||
# Match filter done (right track); now prefer the best quality
|
||||
# among the confidence-passing survivors so streaming isn't
|
||||
# quality-blind like Soulseek already isn't. Stable ranking
|
||||
# keeps confidence order within an equal quality tier; the
|
||||
# `or scored` fail-safe never leaves us with nothing to try.
|
||||
from core.quality.selection import rank_for_profile
|
||||
ranked, _ = rank_for_profile(scored)
|
||||
filtered_results = ranked or scored
|
||||
filtered_results = scored
|
||||
logger.info(f"Streaming validation: {len(scored)}/{len(tracks)} passed "
|
||||
f"(best: {scored[0]._match_confidence:.2f}, "
|
||||
f"quality pick: {filtered_results[0].audio_quality.label()})")
|
||||
f"(best: {scored[0]._match_confidence:.2f})")
|
||||
else:
|
||||
logger.warning(f"No streaming results passed validation for: {query}")
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -13,11 +13,10 @@ import from a neutral package per Cin's contract-first standard.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from core.imports.filename import parse_filename_metadata
|
||||
from core.quality.model import AudioQuality
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -33,36 +32,6 @@ class SearchResult:
|
|||
upload_speed: int
|
||||
queue_length: int
|
||||
result_type: str = "track" # "track" or "album"
|
||||
# Rich quality metadata — populated by sources that provide it.
|
||||
# None means "unknown", not "absent".
|
||||
sample_rate: Optional[int] = None # Hz (e.g. 44100, 96000, 192000)
|
||||
bit_depth: Optional[int] = None # bits per sample (16, 24)
|
||||
|
||||
@property
|
||||
def audio_quality(self) -> AudioQuality:
|
||||
"""Unified quality descriptor derived from this result's fields."""
|
||||
return AudioQuality(
|
||||
format=self.quality.lower() if self.quality else 'unknown',
|
||||
bitrate=self.bitrate,
|
||||
sample_rate=self.sample_rate,
|
||||
bit_depth=self.bit_depth,
|
||||
)
|
||||
|
||||
def set_quality(self, aq: AudioQuality) -> None:
|
||||
"""Merge a mapped :class:`AudioQuality` onto this result's fields.
|
||||
|
||||
Used by streaming sources to stamp their claimed tier (Tidal/HiFi
|
||||
tier strings, Qobuz API values, …) so ``audio_quality`` ranks
|
||||
correctly. Mapper-provided fields win; a ``None`` from the mapper
|
||||
leaves any already-reported value (e.g. a probed bitrate) intact.
|
||||
"""
|
||||
self.quality = aq.format
|
||||
if aq.bitrate is not None:
|
||||
self.bitrate = aq.bitrate
|
||||
if aq.sample_rate is not None:
|
||||
self.sample_rate = aq.sample_rate
|
||||
if aq.bit_depth is not None:
|
||||
self.bit_depth = aq.bit_depth
|
||||
|
||||
@property
|
||||
def quality_score(self) -> float:
|
||||
|
|
@ -158,19 +127,6 @@ class AlbumResult:
|
|||
queue_length: int = 0
|
||||
result_type: str = "album"
|
||||
|
||||
@property
|
||||
def audio_quality(self) -> AudioQuality:
|
||||
"""Unified quality descriptor derived from dominant track quality."""
|
||||
sample_rates = [t.sample_rate for t in self.tracks if t.sample_rate]
|
||||
bit_depths = [t.bit_depth for t in self.tracks if t.bit_depth]
|
||||
bitrates = [t.bitrate for t in self.tracks if t.bitrate]
|
||||
return AudioQuality(
|
||||
format=self.dominant_quality.lower() if self.dominant_quality else 'unknown',
|
||||
bitrate=max(bitrates) if bitrates else None,
|
||||
sample_rate=max(sample_rates) if sample_rates else None,
|
||||
bit_depth=max(bit_depths) if bit_depths else None,
|
||||
)
|
||||
|
||||
@property
|
||||
def quality_score(self) -> float:
|
||||
"""Calculate album quality score based on dominant quality and track count"""
|
||||
|
|
|
|||
|
|
@ -83,26 +83,9 @@ def clear_completed_local() -> int:
|
|||
"""
|
||||
cleared = 0
|
||||
with tasks_lock:
|
||||
# Protect tasks belonging to a still-active batch. A batch is "active"
|
||||
# while any of its queued tasks is non-terminal (still searching /
|
||||
# downloading / queued / post-processing). Pruning a batch's completed
|
||||
# or failed tasks mid-run would yank them out of the Downloads page —
|
||||
# and failed/cancelled rows aren't recoverable from library_history —
|
||||
# so the user would never see them until the batch ended. Keep the whole
|
||||
# active batch intact; it gets cleaned by a later run once it finishes.
|
||||
protected_task_ids: set = set()
|
||||
for batch in download_batches.values():
|
||||
queue = batch.get('queue', []) if isinstance(batch, dict) else []
|
||||
batch_active = any(
|
||||
download_tasks.get(tid, {}).get('status') not in _TERMINAL_STATUSES
|
||||
for tid in queue if tid in download_tasks
|
||||
)
|
||||
if batch_active:
|
||||
protected_task_ids.update(queue)
|
||||
|
||||
task_ids_to_remove = [
|
||||
tid for tid, task in download_tasks.items()
|
||||
if task.get('status') in _TERMINAL_STATUSES and tid not in protected_task_ids
|
||||
if task.get('status') in _TERMINAL_STATUSES
|
||||
]
|
||||
for tid in task_ids_to_remove:
|
||||
del download_tasks[tid]
|
||||
|
|
|
|||
|
|
@ -47,55 +47,6 @@ from core.runtime_state import (
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _priority_sort_key(r):
|
||||
"""Today's confidence-first key: never download a high-quality WRONG file."""
|
||||
return (
|
||||
getattr(r, 'confidence', 0) or 0,
|
||||
getattr(r, 'quality_score', 0) or 0,
|
||||
getattr(r, 'upload_speed', 0) or 0,
|
||||
-(getattr(r, 'queue_length', 0) or 0),
|
||||
getattr(r, 'free_upload_slots', 0) or 0,
|
||||
getattr(r, 'size', 0) or 0,
|
||||
)
|
||||
|
||||
|
||||
def _quality_first_sort_key(r, targets):
|
||||
"""Best-quality key: the user's profile quality rank dominates; all the
|
||||
priority-mode signals (confidence, speed, …) become tiebreakers.
|
||||
|
||||
Every candidate reaching this point already passed match filtering, so it
|
||||
is "correct enough" — ordering by quality among correct candidates is safe.
|
||||
Candidates with no usable quality info, or that match no target, sort last
|
||||
(never dropped). Lower target index = better target, so it's negated to fit
|
||||
the descending (reverse=True) sort.
|
||||
"""
|
||||
from core.quality.model import rank_candidate
|
||||
|
||||
aq = getattr(r, 'audio_quality', None)
|
||||
if aq is None or not targets:
|
||||
target_idx, tier = (len(targets) if targets else 0), 0.0
|
||||
else:
|
||||
try:
|
||||
target_idx, tier = rank_candidate(aq, targets)
|
||||
except Exception:
|
||||
target_idx, tier = len(targets), 0.0
|
||||
return (-target_idx, tier) + _priority_sort_key(r)
|
||||
|
||||
|
||||
def order_candidates(candidates, *, quality_first=False, targets=None):
|
||||
"""Return *candidates* ordered best-first for the download walk.
|
||||
|
||||
``quality_first=False`` (priority mode) → confidence-first, byte-for-byte
|
||||
today's behaviour. ``quality_first=True`` (best-quality mode) → the user's
|
||||
profile quality rank dominates, confidence/peer signals break ties.
|
||||
"""
|
||||
if quality_first:
|
||||
key = lambda r: _quality_first_sort_key(r, targets or [])
|
||||
else:
|
||||
key = _priority_sort_key
|
||||
return sorted(candidates, key=key, reverse=True)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CandidatesDeps:
|
||||
"""Bundle of cross-cutting deps the candidate-fallback logic needs."""
|
||||
|
|
@ -108,25 +59,25 @@ class CandidatesDeps:
|
|||
on_download_completed: Callable
|
||||
|
||||
|
||||
def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
|
||||
deps: CandidatesDeps = None, *,
|
||||
quality_first=False, quality_targets=None):
|
||||
def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, deps: CandidatesDeps = None):
|
||||
"""
|
||||
Attempts to download with fallback candidate logic (matches GUI's retry_parallel_download_with_fallback).
|
||||
Returns True if successful, False if all candidates fail.
|
||||
|
||||
``quality_first`` (best-quality search mode) orders the walk by the user's
|
||||
profile quality rank instead of confidence-first; ``quality_targets`` is the
|
||||
profile target list used for that ranking. Defaults preserve priority-mode
|
||||
behaviour exactly.
|
||||
"""
|
||||
# Sort candidates. Priority mode: confidence-first, then peer quality —
|
||||
# upstream Soulseek validation already considers peer speed/slots/queue when
|
||||
# scores are close; preserve that signal instead of flattening ties back to
|
||||
# arbitrary slskd response order. Best-quality mode: profile quality rank
|
||||
# dominates (all candidates here already passed match filtering).
|
||||
candidates = order_candidates(
|
||||
candidates, quality_first=quality_first, targets=quality_targets,
|
||||
# Sort candidates by match confidence first, then peer quality. Upstream
|
||||
# Soulseek validation already considers peer speed/slots/queue when scores
|
||||
# are close; preserve that signal here instead of flattening ties back to
|
||||
# arbitrary slskd response order.
|
||||
candidates.sort(
|
||||
key=lambda r: (
|
||||
getattr(r, 'confidence', 0) or 0,
|
||||
getattr(r, 'quality_score', 0) or 0,
|
||||
getattr(r, 'upload_speed', 0) or 0,
|
||||
-(getattr(r, 'queue_length', 0) or 0),
|
||||
getattr(r, 'free_upload_slots', 0) or 0,
|
||||
getattr(r, 'size', 0) or 0,
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
with tasks_lock:
|
||||
|
|
@ -254,21 +205,6 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
|
|||
'artists': _fallback_album_artists
|
||||
}
|
||||
|
||||
# #915: parity with Reorganize / manual Enrich. If the album context is lean
|
||||
# (no release_date) and the user's PRIMARY metadata source isn't Spotify, hydrate
|
||||
# it from that source — the same place a reorganize reads — so the download's
|
||||
# $year folder, release_date and album_type match instead of dropping the year /
|
||||
# defaulting to YYYY-01-01 and forcing a manual reorganize afterwards.
|
||||
try:
|
||||
from core.downloads.track_metadata_backfill import backfill_album_context_from_source
|
||||
from core.metadata import registry as _meta_registry
|
||||
from core.metadata.album_tracks import get_album_for_source as _get_album_for_source
|
||||
backfill_album_context_from_source(
|
||||
spotify_album_context, _meta_registry.get_primary_source(), _get_album_for_source,
|
||||
)
|
||||
except Exception as _bf_err: # noqa: BLE001 — never let backfill break a download
|
||||
logger.debug("[Context] primary-source album backfill skipped: %s", _bf_err)
|
||||
|
||||
download_payload = candidate.__dict__
|
||||
|
||||
username = download_payload.get('username')
|
||||
|
|
|
|||
|
|
@ -77,16 +77,7 @@ def _normalize_for_finding(text: str) -> str:
|
|||
return ""
|
||||
text = unidecode(text).lower()
|
||||
text = re.sub(r'[._/]', ' ', text)
|
||||
# Strip ONLY balanced bracket pairs (tags like "[FLAC]", "(Remastered 2016)").
|
||||
# The old combined pattern r'[\[\(].*?[\]\)]' allowed MISMATCHED delimiters, so a
|
||||
# lone unbalanced '[' — slskd reports "[34 - You & Me (Flume Remix)" but saves the
|
||||
# file as "34 - You & Me (Flume Remix)" — matched from that '[' all the way to the
|
||||
# next ')', eating the entire title and collapsing the search target to "flac". The
|
||||
# file then scored 0.40 against the real on-disk name and was reported "not found"
|
||||
# despite sitting right there. Per-delimiter pairs can't over-consume; a stray
|
||||
# unbalanced bracket simply survives to the alphanumeric strip below.
|
||||
text = re.sub(r'\[[^\]]*\]', '', text)
|
||||
text = re.sub(r'\([^)]*\)', '', text)
|
||||
text = re.sub(r'[\[\(].*?[\]\)]', '', text)
|
||||
text = re.sub(r'[^a-z0-9\s-]', '', text)
|
||||
return ' '.join(text.split()).strip()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
"""Match a file back to its download-history row when its path has drifted (#934).
|
||||
|
||||
``library_history.file_path`` is frozen at import time, but the file moves afterward
|
||||
(media-server import, library reorganize) and ``tracks.file_path`` — what the AcoustID
|
||||
scanner reads — no longer equals it. Matching on the exact path alone then fails twice:
|
||||
the verification status never reaches the history row (verified tracks read "unverified"),
|
||||
and a fresh ``acoustid_scan`` row gets inserted every run (thousands of duplicates).
|
||||
|
||||
This module picks the canonical history row by exact path first, then by FILENAME guarded
|
||||
by a title check — so a shared filename ("01 - Intro.flac") can never heal the wrong song.
|
||||
Pure (no DB) so the matching rules are unit-testable; the caller does the SQL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Iterable, Optional, Sequence, Tuple
|
||||
|
||||
|
||||
def _norm_title(value) -> str:
|
||||
"""Alphanumeric-only lowercase form, so "Song (Remaster)" vs "song remaster"
|
||||
style drift between the download tag and the media-server tag still agrees."""
|
||||
return ''.join(ch for ch in str(value or '').lower() if ch.isalnum())
|
||||
|
||||
|
||||
def like_filename_filter(basename: str) -> str:
|
||||
"""A ``LIKE ... ESCAPE '\\'`` pattern that coarsely matches rows whose path ends
|
||||
in ``basename``. Escapes the LIKE metacharacters (``%`` ``_`` ``\\``) — filenames
|
||||
routinely contain underscores. Callers MUST still confirm with an exact basename
|
||||
compare (``pick_history_row`` does), since ``'%name'`` also matches ``'xname'``."""
|
||||
esc = basename.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_')
|
||||
return '%' + esc
|
||||
|
||||
|
||||
def pick_history_row(candidates: Sequence[Tuple], *, current_paths: Iterable[str],
|
||||
basename: str, title: str) -> Optional[int]:
|
||||
"""Return the id of the history row to update for this file, or None.
|
||||
|
||||
``candidates``: ``(id, file_path, title, download_source)`` rows the DB pre-filtered
|
||||
(exact path or filename LIKE). A row matches when its path equals the current path OR
|
||||
its filename matches AND its title agrees — the title guard prevents a shared filename
|
||||
("01 - Intro.flac") from healing a different song's row. Among matches a REAL download
|
||||
row is preferred over a synthetic ``acoustid_scan`` row, so the scanner heals the
|
||||
genuine record and the caller can delete the synthetic duplicate. None when nothing
|
||||
matches safely (caller then inserts a fresh row — the "file SoulSync never downloaded"
|
||||
intent)."""
|
||||
paths = {p for p in current_paths if p}
|
||||
want = _norm_title(title)
|
||||
matches: list = [] # (id, is_exact, is_real)
|
||||
for cid, cpath, ctitle, csource in candidates:
|
||||
is_real = csource != 'acoustid_scan'
|
||||
if cpath and cpath in paths:
|
||||
matches.append((cid, True, is_real))
|
||||
elif (basename and cpath and os.path.basename(cpath) == basename
|
||||
and (not want or not _norm_title(ctitle) or _norm_title(ctitle) == want)):
|
||||
matches.append((cid, False, is_real))
|
||||
if not matches:
|
||||
return None
|
||||
# Prefer a REAL download row over a synthetic acoustid_scan row; within that, prefer an
|
||||
# exact-path match over a filename match. Stable, so ties keep DB order (first/oldest id).
|
||||
matches.sort(key=lambda m: (m[2], m[1]), reverse=True)
|
||||
return matches[0][0]
|
||||
|
|
@ -26,7 +26,6 @@ Lifted verbatim from web_server.py. Dependencies injected via
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import traceback
|
||||
|
|
@ -45,27 +44,6 @@ from core.runtime_state import (
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# A task that has been in 'post_processing' longer than this is treated as stuck.
|
||||
# Post-processing (AcoustID + quality + import) is serialized, so a large batch
|
||||
# legitimately backs up — keep this generous so genuinely-slow imports aren't
|
||||
# cut off mid-flight (the old 5-min cutoff falsely "completed" queued tasks).
|
||||
_POST_PROCESSING_STUCK_TIMEOUT = 1800 # 30 minutes
|
||||
|
||||
|
||||
def _resolve_stuck_post_processing_status(task: dict) -> str:
|
||||
"""Decide the terminal status for a task stuck in post_processing.
|
||||
|
||||
Only call it 'completed' if the import actually produced a file on disk
|
||||
(``final_file_path`` is set at the end of successful post-processing). Without
|
||||
a real file, force-completing is a lie — the task shows as a downloaded track
|
||||
that isn't anywhere. Mark those 'failed' so they're retryable and honest.
|
||||
"""
|
||||
final_path = task.get('final_file_path')
|
||||
if final_path and os.path.exists(final_path):
|
||||
return 'completed'
|
||||
return 'failed'
|
||||
|
||||
|
||||
def _safe_batch_dirname(batch_id: str) -> str:
|
||||
safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch'))
|
||||
return safe or 'batch'
|
||||
|
|
@ -457,15 +435,9 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life
|
|||
retrying_count += 1
|
||||
elif task_status == 'post_processing':
|
||||
task_age = current_time - task.get('status_change_time', current_time)
|
||||
if task_age > _POST_PROCESSING_STUCK_TIMEOUT:
|
||||
new_status = _resolve_stuck_post_processing_status(task)
|
||||
if new_status == 'completed':
|
||||
logger.info(f"⏰ [Stuck Detection] Task {queue_task_id} stuck in post_processing for {task_age:.0f}s but file exists — completing")
|
||||
task['status'] = 'completed'
|
||||
else:
|
||||
logger.warning(f"⏰ [Stuck Detection] Task {queue_task_id} stuck in post_processing for {task_age:.0f}s with no output file — marking failed")
|
||||
task['status'] = 'failed'
|
||||
task['error_message'] = 'Post-processing timed out without producing a file'
|
||||
if task_age > 300: # 5 minutes (post-processing should be fast)
|
||||
logger.info(f"⏰ [Stuck Detection] Task {queue_task_id} stuck in post_processing for {task_age:.0f}s - forcing completion")
|
||||
task['status'] = 'completed' # Assume it worked if file verification is taking too long
|
||||
finished_count += 1
|
||||
else:
|
||||
retrying_count += 1
|
||||
|
|
@ -695,15 +667,9 @@ def check_batch_completion_v2(batch_id: str, deps: LifecycleDeps) -> Optional[bo
|
|||
retrying_count += 1
|
||||
elif task_status == 'post_processing':
|
||||
task_age = current_time - task.get('status_change_time', current_time)
|
||||
if task_age > _POST_PROCESSING_STUCK_TIMEOUT:
|
||||
new_status = _resolve_stuck_post_processing_status(task)
|
||||
if new_status == 'completed':
|
||||
logger.info(f"⏰ [Stuck Detection V2] Task {task_id} stuck in post_processing for {task_age:.0f}s but file exists — completing")
|
||||
task['status'] = 'completed'
|
||||
else:
|
||||
logger.warning(f"⏰ [Stuck Detection V2] Task {task_id} stuck in post_processing for {task_age:.0f}s with no output file — marking failed")
|
||||
task['status'] = 'failed'
|
||||
task['error_message'] = 'Post-processing timed out without producing a file'
|
||||
if task_age > 300: # 5 minutes (post-processing should be fast)
|
||||
logger.info(f"⏰ [Stuck Detection V2] Task {task_id} stuck in post_processing for {task_age:.0f}s - forcing completion")
|
||||
task['status'] = 'completed' # Assume it worked if file verification is taking too long
|
||||
finished_count += 1
|
||||
else:
|
||||
retrying_count += 1
|
||||
|
|
|
|||
|
|
@ -760,8 +760,7 @@ class WebUIDownloadMonitor:
|
|||
# used_sources keys are formatted as "{username}_{filename}", so startswith is exact.
|
||||
is_tidal = any(s.startswith('tidal_') for s in tried_sources)
|
||||
if is_tidal:
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
tidal_quality = quality_tier_for_source('tidal', default='lossless')
|
||||
tidal_quality = config_manager.get('tidal_download.quality', 'lossless')
|
||||
allow_fb = config_manager.get('tidal_download.allow_fallback', True)
|
||||
if tidal_quality == 'hires' and not allow_fb:
|
||||
task['error_message'] = (
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
"""Identify dead review-queue history rows whose file is gone (#934 follow-up).
|
||||
|
||||
The Unverified/Quarantine review queue is fed from ``library_history`` — an
|
||||
append-only log that is never pruned. When a file is deleted, replaced, or
|
||||
re-downloaded elsewhere, its old ``unverified`` row lingers forever and can
|
||||
never be healed (there's no file left to confirm). Those are *orphans*.
|
||||
|
||||
This decides which rows are orphans, given a ``resolve(row) -> path | None``
|
||||
the caller wires to the real filesystem lookup. Pure (no DB, no filesystem) so
|
||||
the rules — including the safety gate — are unit-testable.
|
||||
|
||||
Safety gate: a filesystem check mass-false-positives when the library mount is
|
||||
down (every file looks missing). So if EVERY reviewed file is unreachable and
|
||||
there are enough rows to judge, we flag it ``suspicious`` and the caller refuses
|
||||
to delete — better to clean nothing than to wipe a healthy log during an outage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Sequence
|
||||
|
||||
|
||||
def find_orphan_history_ids(
|
||||
rows: Sequence[dict],
|
||||
resolve: Callable[[dict], Any],
|
||||
*,
|
||||
min_for_safety: int = 5,
|
||||
deletable: Callable[[dict], bool] | None = None,
|
||||
) -> dict:
|
||||
"""Return ``{'orphan_ids', 'checked', 'suspicious'}``.
|
||||
|
||||
A row is an orphan when it has a non-empty ``file_path`` but ``resolve`` can
|
||||
find no file for it. ``suspicious`` is True when every checked row is
|
||||
missing and there are at least ``min_for_safety`` of them — the mount-down
|
||||
signature; the caller should refuse to delete in that case.
|
||||
|
||||
``deletable`` (optional) protects rows from removal WITHOUT weakening the
|
||||
safety gate: a protected row still counts toward ``checked`` and the
|
||||
all-missing signal (so e.g. a few unverified orphans can't be swept during a
|
||||
mount outage just because protected rows were filtered out first), but it
|
||||
never appears in ``orphan_ids``. Default: every missing row is deletable.
|
||||
"""
|
||||
orphan_ids = []
|
||||
checked = 0
|
||||
missing = 0
|
||||
for row in rows:
|
||||
if not str((row.get('file_path') or '')).strip():
|
||||
continue
|
||||
checked += 1
|
||||
if resolve(row) is None:
|
||||
missing += 1
|
||||
if deletable is None or deletable(row):
|
||||
orphan_ids.append(row.get('id'))
|
||||
suspicious = checked >= min_for_safety and missing == checked
|
||||
return {'orphan_ids': orphan_ids, 'checked': checked, 'suspicious': suspicious}
|
||||
|
|
@ -171,21 +171,6 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
|
|||
logger.info(f"[Post-Processing] Task {task_id} already completed by stream processor, skipping verification")
|
||||
return
|
||||
|
||||
# RACE GUARD: the monitor sets status -> 'post_processing' immediately
|
||||
# before submitting this worker. If the status is now anything else, the
|
||||
# browser-poll post-processor already took ownership of this task — e.g.
|
||||
# it quarantined the file and requeued the next-best candidate (status
|
||||
# -> 'searching', source identity cleared). Bail WITHOUT marking failed
|
||||
# or notifying batch completion: otherwise we clobber that in-flight
|
||||
# retry with a bogus "missing file or source information" failure while a
|
||||
# parallel attempt is importing the song.
|
||||
if task['status'] != 'post_processing':
|
||||
logger.info(
|
||||
f"[Post-Processing] Task {task_id} no longer in 'post_processing' "
|
||||
f"(now '{task['status']}') — another path took over, skipping"
|
||||
)
|
||||
return
|
||||
|
||||
# Extract file information for verification
|
||||
track_info = task.get('track_info', {})
|
||||
task_filename = task.get('filename') or track_info.get('filename')
|
||||
|
|
@ -453,19 +438,7 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
|
|||
logger.error(f"[Post-Processing] Task {task_id} was completed by stream processor - not marking as failed")
|
||||
return
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
# slskd reported the transfer complete, but the finder never located
|
||||
# the file under the configured download folder. Name the folder we
|
||||
# searched and the two real causes — "still being written" (timing)
|
||||
# or "SoulSync's download path doesn't match slskd's" (the classic
|
||||
# standalone config mismatch) — so the user can self-diagnose instead
|
||||
# of getting an opaque "not found". (Discord: Shdjfgatdif.)
|
||||
_searched_name = os.path.basename((task_filename or '').replace('\\', '/')) or task_filename
|
||||
download_tasks[task_id]['error_message'] = (
|
||||
f"slskd reported '{_searched_name}' downloaded, but it never appeared "
|
||||
f"under the download folder ({download_dir}) after {_file_search_max_retries} "
|
||||
f"checks. Either it's still being written, or SoulSync's download path "
|
||||
f"doesn't match slskd's download directory — they must point at the same folder."
|
||||
)
|
||||
download_tasks[task_id]['error_message'] = f'File not found on disk after {_file_search_max_retries} search attempts. Expected: {os.path.basename(task_filename)}'
|
||||
deps.on_download_completed(batch_id, task_id, False)
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -94,10 +94,6 @@ class StatusDeps:
|
|||
run_async: Optional[Callable] = None
|
||||
on_download_completed: Optional[Callable[[str, str, bool], None]] = None
|
||||
get_persistent_download_history: Optional[Callable[[int], list[dict]]] = None
|
||||
# Returns ALL library_history rows with verification_status in
|
||||
# ('unverified', 'force_imported') — no recency limit, so historical
|
||||
# entries are never buried by the general history tail cap.
|
||||
get_unverified_download_history: Optional[Callable[[], list[dict]]] = None
|
||||
|
||||
|
||||
# Streaming sources the engine fallback applies to. Soulseek goes through
|
||||
|
|
@ -800,13 +796,6 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
'progress': progress,
|
||||
'error': task.get('error_message'),
|
||||
'verification_status': task.get('verification_status'),
|
||||
# library_history row id (set at import) so the Unverified review
|
||||
# queue can act on a still-live completed task before it becomes
|
||||
# a persistent-history row.
|
||||
'history_id': task.get('history_id'),
|
||||
# Real probed audio quality (mutagen-read from the actual file),
|
||||
# surfaced so the Downloads page can show what was downloaded.
|
||||
'quality': task.get('quality') or '',
|
||||
'retry_info': task.get('retry_info'),
|
||||
'retry_trigger': task.get('retry_trigger'),
|
||||
'batch_id': batch_id,
|
||||
|
|
@ -823,32 +812,6 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
'is_persistent_history': False,
|
||||
})
|
||||
|
||||
# --- Unverified history (unconditional, no limit) ---
|
||||
# Always load every library_history row that still needs human confirmation
|
||||
# (verification_status IN ('unverified', 'force_imported')). This is NOT
|
||||
# gated on len(items) < limit so that historical entries from past batches
|
||||
# are visible even during a large active batch that would otherwise exhaust
|
||||
# the limit before the history tail is read. Dedup against live tasks by
|
||||
# identity so a track currently in post-processing isn't shown twice.
|
||||
if deps.get_unverified_download_history is not None:
|
||||
try:
|
||||
unverified_entries = deps.get_unverified_download_history() or []
|
||||
except Exception as exc:
|
||||
logger.debug("[Downloads] unverified history lookup failed: %s", exc)
|
||||
unverified_entries = []
|
||||
|
||||
for entry in unverified_entries:
|
||||
item = _build_history_download_item(entry)
|
||||
identity = _download_identity(item.get('title'), item.get('artist'), item.get('album'))
|
||||
if identity in live_identities:
|
||||
continue
|
||||
items.append(item)
|
||||
live_identities.add(identity)
|
||||
|
||||
# --- General recent-history tail (capped, recency-ordered) ---
|
||||
# Fills in the completed/verified tail so the full Downloads list looks
|
||||
# populated. Gated on len(items) < limit so a busy batch doesn't trigger
|
||||
# an extra DB round-trip when we're already at capacity.
|
||||
if deps.get_persistent_download_history is not None and len(items) < limit:
|
||||
history_limit = min(limit - len(items), _PERSISTENT_HISTORY_TAIL_LIMIT)
|
||||
try:
|
||||
|
|
@ -869,14 +832,7 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
live_identities.add(identity)
|
||||
appended_history += 1
|
||||
|
||||
# Sort: active first (by priority), then by timestamp desc within each group.
|
||||
# NOTE: the array order is presentation-only — the Downloads page filters
|
||||
# client-side per tab. What matters is that EVERY live task is present: an
|
||||
# earlier `items[:limit]` truncation (active-first) starved completed/failed/
|
||||
# unverified rows off the end during a busy batch, so those tabs stayed empty
|
||||
# until the batch drained. `limit` now bounds only the persistent-history
|
||||
# tail (handled above); live in-memory tasks are always returned in full
|
||||
# (they're already bounded by the 5-min cleanup automation).
|
||||
# Sort: active first (by priority), then by timestamp desc within each group
|
||||
items.sort(key=lambda x: (x['priority'], -x['timestamp']))
|
||||
|
||||
# Build batch summaries for the batch context panel
|
||||
|
|
@ -904,7 +860,7 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
|
||||
return {
|
||||
'success': True,
|
||||
'downloads': items,
|
||||
'downloads': items[:limit],
|
||||
'total': len(items),
|
||||
'batches': batch_summaries,
|
||||
'timestamp': time.time(),
|
||||
|
|
|
|||
|
|
@ -54,37 +54,6 @@ def _cand_user_file(candidate):
|
|||
return getattr(candidate, 'username', None), getattr(candidate, 'filename', None)
|
||||
|
||||
|
||||
def _candidate_ordering():
|
||||
"""Return ``(quality_first, targets)`` for the active search mode + toggle.
|
||||
|
||||
The candidate walk is ordered by the user's profile quality rank
|
||||
(best→worst) instead of confidence-first when EITHER:
|
||||
- best-quality search mode is active (always quality-first), OR
|
||||
- priority mode and the ``rank_candidates_by_quality`` toggle is on
|
||||
(opt-in; default off keeps the byte-for-byte confidence-first walk).
|
||||
|
||||
Quality-first ordering also makes the version-mismatch force-import pick
|
||||
the highest-quality candidate, because that fallback accepts the
|
||||
first-tried (= best-ordered) quarantined entry.
|
||||
|
||||
Fails closed to confidence-first ordering on any error so a profile/DB
|
||||
hiccup never blocks a download. See
|
||||
docs/superpowers/specs/2026-06-14-best-quality-search-mode-design.md.
|
||||
"""
|
||||
try:
|
||||
from core.quality.selection import (
|
||||
load_search_mode,
|
||||
load_profile_targets,
|
||||
load_rank_candidates_by_quality,
|
||||
)
|
||||
if load_search_mode() == 'best_quality' or load_rank_candidates_by_quality():
|
||||
targets, _ = load_profile_targets()
|
||||
return True, targets
|
||||
except Exception as exc:
|
||||
logger.debug("[Modal Worker] quality ordering unavailable: %s", exc)
|
||||
return False, None
|
||||
|
||||
|
||||
def _try_cached_candidates(task_id, batch_id, track, deps):
|
||||
"""Quarantine-retry fast path: attempt the already-found candidates before
|
||||
re-searching anything.
|
||||
|
|
@ -122,10 +91,7 @@ def _try_cached_candidates(task_id, batch_id, track, deps):
|
|||
f"[Modal Worker] Quarantine retry: trying {len(remaining)} cached "
|
||||
f"candidate(s) before re-searching (task {task_id})"
|
||||
)
|
||||
_qf, _qt = _candidate_ordering()
|
||||
return deps.attempt_download_with_candidates(
|
||||
task_id, remaining, track, batch_id, quality_first=_qf, quality_targets=_qt,
|
||||
)
|
||||
return deps.attempt_download_with_candidates(task_id, remaining, track, batch_id)
|
||||
|
||||
|
||||
def _private_album_bundle_staging_miss_reason(batch_id: Optional[str], deps: Any) -> Optional[str]:
|
||||
|
|
@ -403,11 +369,6 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
logger.info(f"[Modal Worker] Generated {len(search_queries)} smart search queries for '{track.name}': {search_queries}")
|
||||
logger.info(f"[Modal Worker] About to start search loop for task {task_id} (track: '{track.name}')")
|
||||
|
||||
# Best-quality search mode: the orchestrator already pooled candidates
|
||||
# across every source for each query, so order the candidate walk by the
|
||||
# user's profile quality rank (best→worst). Computed once per task.
|
||||
_best_quality, _quality_targets = _candidate_ordering()
|
||||
|
||||
# 2. Sequential Query Search (matches GUI's start_search_worker_parallel logic)
|
||||
search_diagnostics = [] # Track what happened per query for detailed error messages
|
||||
all_raw_results = [] # Collect raw results across queries for candidate review modal
|
||||
|
|
@ -534,10 +495,7 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
download_tasks[task_id]['cached_candidates'] = candidates
|
||||
|
||||
# Try to download with these candidates
|
||||
success = deps.attempt_download_with_candidates(
|
||||
task_id, candidates, track, batch_id,
|
||||
quality_first=_best_quality, quality_targets=_quality_targets,
|
||||
)
|
||||
success = deps.attempt_download_with_candidates(task_id, candidates, track, batch_id)
|
||||
if success:
|
||||
# Download initiated successfully - let the download monitoring system handle completion
|
||||
if batch_id:
|
||||
|
|
@ -571,10 +529,7 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
# === HYBRID FALLBACK: If primary source failed, try remaining sources directly ===
|
||||
# The orchestrator's hybrid search stops at the first source with results, even if
|
||||
# those results all fail quality filtering. Try remaining sources individually.
|
||||
#
|
||||
# Best-quality mode already searched EVERY source per query (the pool), so this
|
||||
# block would only re-search the same sources — skip it there.
|
||||
if not _best_quality and getattr(deps.download_orchestrator, 'mode', '') == 'hybrid':
|
||||
if getattr(deps.download_orchestrator, 'mode', '') == 'hybrid':
|
||||
try:
|
||||
orch = deps.download_orchestrator
|
||||
hybrid_order = getattr(orch, 'hybrid_order', None) or []
|
||||
|
|
|
|||
|
|
@ -13,48 +13,9 @@ Pure + import-safe: parsing only, no network.
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, List, Optional, Tuple
|
||||
from typing import Any, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
def linked_track_id(track: Any) -> str:
|
||||
"""The source track id stamped on a search result, read from
|
||||
``_source_metadata['track_id']`` — the field every ID-downloadable source
|
||||
(Tidal, Qobuz) records. Empty string when absent. ``TrackResult`` has no
|
||||
top-level ``id``, so callers must NOT use ``getattr(t, 'id')`` (that always
|
||||
missed and left the pasted-link bubble a silent no-op — #932)."""
|
||||
meta = getattr(track, '_source_metadata', None)
|
||||
if not isinstance(meta, dict):
|
||||
return ''
|
||||
return str(meta.get('track_id') or '')
|
||||
|
||||
|
||||
def bubble_linked_track_first(tracks: List[Any], link_track_id: str) -> List[Any]:
|
||||
"""Float the result whose source id matches a pasted link to the top so the
|
||||
user sees the EXACT track they linked, not a fuzzy text-search lookalike
|
||||
(#813/#932). Stable + a graceful no-op when no result carries the id."""
|
||||
if not link_track_id or not tracks:
|
||||
return tracks
|
||||
target = str(link_track_id)
|
||||
return sorted(tracks, key=lambda t: linked_track_id(t) != target)
|
||||
|
||||
|
||||
def inject_linked_track_first(
|
||||
tracks: List[Any], linked_result: Any, link_track_id: str
|
||||
) -> List[Any]:
|
||||
"""Put the EXACT linked track first.
|
||||
|
||||
When ``linked_result`` is the track fetched directly by id, prepend it and
|
||||
drop any search duplicate of it — so an obscure track a text search never
|
||||
surfaced is still present and downloadable (#932). When it's None (the source
|
||||
can't fetch one), fall back to bubbling a matching search result. Pure."""
|
||||
if not link_track_id:
|
||||
return tracks
|
||||
target = str(link_track_id)
|
||||
if linked_result is not None:
|
||||
return [linked_result] + [t for t in tracks if linked_track_id(t) != target]
|
||||
return bubble_linked_track_first(tracks, target)
|
||||
|
||||
# host substring → download source id. Only ID-downloadable streaming sources.
|
||||
_HOSTS = (
|
||||
('tidal.com', 'tidal'),
|
||||
|
|
|
|||
|
|
@ -93,56 +93,6 @@ def _backfill_album_context(
|
|||
album_context['image_url'] = first['url']
|
||||
|
||||
|
||||
# Placeholder album ids used when no real source album id is known — never queryable.
|
||||
_SENTINEL_ALBUM_IDS = {'explicit_album', 'from_sync_modal', ''}
|
||||
|
||||
|
||||
def backfill_album_context_from_source(
|
||||
album_context: Dict[str, Any],
|
||||
primary_source: Optional[str],
|
||||
get_album_for_source_fn: Any,
|
||||
) -> bool:
|
||||
"""Hydrate a lean album context from the user's PRIMARY metadata source (#915).
|
||||
|
||||
Post-processing's only album backfill (:func:`hydrate_download_metadata`) goes through
|
||||
``spotify_client.get_track_details`` — Spotify-only. An iTunes/Deezer-primary user's
|
||||
download therefore kept a lean context (no ``release_date``), so the path dropped the
|
||||
``$year`` and the date defaulted to ``YYYY-01-01`` — until they ran a Reorganize, which
|
||||
reads the full album from the PRIMARY source. This closes that gap by doing the same:
|
||||
fetch the full album from the primary source and backfill, so a download's pathing/tags
|
||||
match what a later reorganize would produce.
|
||||
|
||||
``get_album_for_source_fn(source, album_id)`` is injected (the real one is
|
||||
``core.metadata.album_tracks.get_album_for_source``) so this stays pure + testable.
|
||||
No-op when: the context is already complete; the primary source is spotify (the existing
|
||||
track-details path covers it); or no real source album id is present. Returns True when
|
||||
it filled anything. Never raises — a backfill failure must not break a download.
|
||||
"""
|
||||
if not isinstance(album_context, dict) or not _album_is_lean(album_context):
|
||||
return False
|
||||
if not primary_source or primary_source == 'spotify':
|
||||
return False
|
||||
album_id = album_context.get('id')
|
||||
if not album_id or str(album_id) in _SENTINEL_ALBUM_IDS:
|
||||
return False
|
||||
try:
|
||||
album = get_album_for_source_fn(primary_source, str(album_id))
|
||||
except Exception as e: # noqa: BLE001 — defensive: never let backfill break a download
|
||||
logger.warning("[Context] primary-source (%s) album backfill failed: %s", primary_source, e)
|
||||
return False
|
||||
if not isinstance(album, dict):
|
||||
return False
|
||||
before = album_context.get('release_date')
|
||||
_backfill_album_context(album_context, {'album': album})
|
||||
if album_context.get('release_date') and album_context.get('release_date') != before:
|
||||
logger.info(
|
||||
"[Context] Hydrated lean album context from primary source %s "
|
||||
"(release_date=%r, total_tracks=%r)",
|
||||
primary_source, album_context.get('release_date'), album_context.get('total_tracks'),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def hydrate_download_metadata(
|
||||
track: Any,
|
||||
track_info: Any,
|
||||
|
|
@ -222,5 +172,4 @@ def hydrate_download_metadata(
|
|||
__all__ = [
|
||||
'ResolvedTrackMetadata',
|
||||
'hydrate_download_metadata',
|
||||
'backfill_album_context_from_source',
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,364 +0,0 @@
|
|||
"""Wire the real cheapest-first sources for the export MBID waterfall (#903).
|
||||
|
||||
``mbid_resolver`` is the pure waterfall; this module supplies the real I/O behind each
|
||||
source and assembles the ``resolve_fn`` the export job uses:
|
||||
|
||||
1. **cache** — ``recording_mbid_cache`` (persistent (artist,title)->mbid).
|
||||
2. **DB** — a text-matched library track's ``tracks.musicbrainz_recording_id``.
|
||||
3. **file** — ``MUSICBRAINZ_RECORDING_ID`` tag of that track's file (when the DB row had
|
||||
no recording id but the file was tagged on import).
|
||||
4. **MusicBrainz** — live ``match_recording(track, artist)`` (rate-limited tail).
|
||||
|
||||
Every source is wrapped so any failure (missing table, unreadable file, MB timeout) returns
|
||||
None — the waterfall just falls through, the export never breaks. ``build_resolve_fn`` also
|
||||
writes a fresh non-cache hit back to the cache so the next export of the same song is free.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
from core.exports.mbid_resolver import (
|
||||
SRC_CACHE,
|
||||
SRC_DB,
|
||||
SRC_FILE,
|
||||
SRC_MUSICBRAINZ,
|
||||
normalize_key,
|
||||
resolve_recording_mbid,
|
||||
)
|
||||
|
||||
logger = get_logger("exports.export_sources")
|
||||
|
||||
|
||||
def _db_match(artist: str, title: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Text-match a library track by (artist, title); return (recording_mbid, file_path).
|
||||
Either may be None. Fail-safe — any DB error returns (None, None)."""
|
||||
if not title:
|
||||
return (None, None)
|
||||
try:
|
||||
from database.music_database import get_database
|
||||
db = get_database()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"SELECT t.musicbrainz_recording_id, t.file_path "
|
||||
"FROM tracks t JOIN artists a ON t.artist_id = a.id "
|
||||
"WHERE LOWER(t.title) = LOWER(?) AND LOWER(a.name) = LOWER(?) "
|
||||
"LIMIT 1",
|
||||
(title, artist),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return (None, None)
|
||||
mbid = row[0] if not hasattr(row, "keys") else row["musicbrainz_recording_id"]
|
||||
fpath = row[1] if not hasattr(row, "keys") else row["file_path"]
|
||||
return ((mbid or None), (fpath or None))
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug(f"export db_match failed for '{artist} - {title}': {exc}")
|
||||
return (None, None)
|
||||
|
||||
|
||||
def db_recording_mbid(artist: str, title: str) -> Optional[str]:
|
||||
"""Recording MBID stored on a matched library track (``musicbrainz_recording_id``)."""
|
||||
return _db_match(artist, title)[0]
|
||||
|
||||
|
||||
# Service → the tracks-table column carrying that service's track ID (set by enrichment).
|
||||
# Trusted constants — never user input — so safe to interpolate into the SELECT below.
|
||||
_SERVICE_ID_COLUMNS = {"spotify": "spotify_track_id", "deezer": "deezer_id"}
|
||||
|
||||
|
||||
def db_service_track_id(artist: str, title: str, service: str) -> Optional[str]:
|
||||
"""The service track ID (``spotify_track_id`` / ``deezer_id``) stored on a matched
|
||||
library track — what lets a mirrored playlist be exported BACK to Spotify/Deezer
|
||||
without re-searching, since enrichment already pinned it (#945). Text-matches by
|
||||
(artist, title), same as the MBID resolver. Fail-safe: any miss/error returns None."""
|
||||
column = _SERVICE_ID_COLUMNS.get((service or "").lower())
|
||||
if not column or not title:
|
||||
return None
|
||||
try:
|
||||
from database.music_database import get_database
|
||||
db = get_database()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
f"SELECT t.{column} FROM tracks t JOIN artists a ON t.artist_id = a.id "
|
||||
"WHERE LOWER(t.title) = LOWER(?) AND LOWER(a.name) = LOWER(?) LIMIT 1",
|
||||
(title, artist),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
val = row[0] if not hasattr(row, "keys") else row[column]
|
||||
return val or None
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug(f"export service-id lookup failed for '{artist} - {title}' ({service}): {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def build_service_resolve_fn(service: str) -> Callable[[str, str], Tuple[Optional[str], Optional[str]]]:
|
||||
"""resolve_fn for service-playlist export: ``(artist, title) -> (service_track_id, 'library')``.
|
||||
Plugs into ``resolve_playlist_tracks(..., id_key='service_track_id')`` exactly like the
|
||||
MBID resolver plugs in for ListenBrainz."""
|
||||
def resolve_fn(artist: str, title: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
tid = db_service_track_id(artist, title, service)
|
||||
return (tid, "library" if tid else None)
|
||||
return resolve_fn
|
||||
|
||||
|
||||
def service_id_from_extra_data(track: Any, service: str) -> Optional[str]:
|
||||
"""The target-service track ID the DISCOVERY step already resolved for this mirrored
|
||||
track, read from its ``extra_data`` blob (#945 — Boulder: "all 50 are discovered to
|
||||
Deezer already, it's not using any of that"). This is free (no API call) and reliable
|
||||
(it's the same id used to mirror the track).
|
||||
|
||||
Only trusted when the track was discovered ON the export's target service — a
|
||||
Deezer-discovered track carries a Deezer id under ``matched_data['id']``, and its
|
||||
``provider`` is the service name. A ``wing_it_fallback`` provider (the low-confidence
|
||||
guess path) deliberately does NOT match here, so those fall through to the library/
|
||||
none path rather than risk a wrong track in the exported playlist."""
|
||||
raw = track.get("extra_data") if isinstance(track, dict) else None
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(raw) if isinstance(raw, str) else raw
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(data, dict) or not data.get("discovered"):
|
||||
return None
|
||||
if str(data.get("provider") or "").lower() != str(service or "").lower():
|
||||
return None
|
||||
matched = data.get("matched_data")
|
||||
tid = matched.get("id") if isinstance(matched, dict) else None
|
||||
return str(tid) if tid else None
|
||||
|
||||
|
||||
def _track_field(track: Dict[str, Any], *names: str) -> str:
|
||||
for n in names:
|
||||
v = track.get(n)
|
||||
if v:
|
||||
return str(v)
|
||||
return ""
|
||||
|
||||
|
||||
def resolve_service_track_ids(
|
||||
tracks: List[Dict[str, Any]],
|
||||
service: str,
|
||||
*,
|
||||
db_fn: Optional[Callable[[str, str, str], Optional[str]]] = None,
|
||||
search_id_fn: Optional[Callable[[str, str], Optional[str]]] = None,
|
||||
on_progress: Optional[Callable[[int, int, Dict[str, Any]], None]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Resolve a mirrored playlist's tracks to target-service track IDs for export.
|
||||
|
||||
Waterfall per track: the discovery cache (``extra_data`` — free + already confidently
|
||||
matched) → the library track's stored service id → (only when ``search_id_fn`` is
|
||||
given, i.e. the opt-in backfill toggle) a confident live-search match. A track that
|
||||
clears none of these is reported unmatched (caller skips it — never a guessed/wrong
|
||||
id). Returns ``{"resolved": [{artist, title, album, service_track_id}], "stats":
|
||||
{...}}`` with ``from_cache`` / ``from_library`` / ``from_search`` / ``unmatched``
|
||||
tallies for the status display.
|
||||
"""
|
||||
db_fn = db_fn or db_service_track_id
|
||||
total = len(tracks or [])
|
||||
resolved: List[Dict[str, Any]] = []
|
||||
stats: Dict[str, Any] = {
|
||||
"total": total, "resolved": 0, "unmatched": 0,
|
||||
"from_cache": 0, "from_library": 0, "from_search": 0,
|
||||
}
|
||||
for i, t in enumerate(tracks or []):
|
||||
if not isinstance(t, dict):
|
||||
t = {}
|
||||
artist = _track_field(t, "artist", "artist_name", "creator")
|
||||
title = _track_field(t, "title", "track_name", "name")
|
||||
album = _track_field(t, "album", "album_name", "release_name")
|
||||
|
||||
tid = service_id_from_extra_data(t, service)
|
||||
if tid:
|
||||
stats["from_cache"] += 1
|
||||
else:
|
||||
tid = db_fn(artist, title, service)
|
||||
if tid:
|
||||
stats["from_library"] += 1
|
||||
elif search_id_fn is not None:
|
||||
tid = search_id_fn(artist, title)
|
||||
if tid:
|
||||
stats["from_search"] += 1
|
||||
|
||||
resolved.append({"artist": artist, "title": title, "album": album,
|
||||
"service_track_id": tid or None})
|
||||
stats["resolved" if tid else "unmatched"] += 1
|
||||
if on_progress is not None:
|
||||
try:
|
||||
on_progress(i + 1, total, stats)
|
||||
except Exception: # noqa: S110 — a progress error must never fail the export
|
||||
pass
|
||||
|
||||
return {"resolved": resolved, "stats": stats}
|
||||
|
||||
|
||||
# Confidence floor for backfill, on the score_track scale (~1.5 = exact title + exact
|
||||
# artist, thanks to the 1.5x artist boost). A cover/karaoke (x0.05) or a wrong-artist hit
|
||||
# (no boost, caps ~1.0) can't clear this — so backfill never adds a guessed/wrong version.
|
||||
BACKFILL_MIN_SCORE = 1.2
|
||||
|
||||
|
||||
def search_service_track_id(
|
||||
artist: str,
|
||||
title: str,
|
||||
*,
|
||||
search_fn: Callable[[str], List[Any]],
|
||||
min_score: float = BACKFILL_MIN_SCORE,
|
||||
) -> Optional[str]:
|
||||
"""Confident live-search match for export backfill (#945): search the target service
|
||||
for (artist, title), rerank by relevance, and return the top match's id ONLY if it
|
||||
clears the confidence floor. Below the floor → None: the track is left out of the
|
||||
export rather than risk a wrong/cover/karaoke version (the whole point of backfill is
|
||||
coverage WITHOUT the wrong-track risk). ``search_fn(query) -> List[Track]`` is injected
|
||||
so this is unit-testable without a live service."""
|
||||
if not title:
|
||||
return None
|
||||
from core.metadata.relevance import build_combined_search_query, filter_and_rerank
|
||||
query = build_combined_search_query(title, artist)
|
||||
try:
|
||||
candidates = list(search_fn(query) or [])
|
||||
except Exception as exc:
|
||||
logger.debug(f"export backfill search failed for '{artist} - {title}': {exc}")
|
||||
return None
|
||||
if not candidates:
|
||||
return None
|
||||
ranked = filter_and_rerank(
|
||||
candidates, expected_title=title, expected_artist=artist, min_score=min_score,
|
||||
)
|
||||
if not ranked:
|
||||
return None
|
||||
tid = getattr(ranked[0], "id", None)
|
||||
return str(tid) if tid else None
|
||||
|
||||
|
||||
def file_recording_mbid(artist: str, title: str) -> Optional[str]:
|
||||
"""Recording MBID read from the matched track's file tag (set on import post-processing)."""
|
||||
_mbid, fpath = _db_match(artist, title)
|
||||
if not fpath:
|
||||
return None
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(fpath)
|
||||
if audio is None or not getattr(audio, "tags", None):
|
||||
return None
|
||||
tags = audio.tags
|
||||
# ID3 UFID (MusicBrainz), Vorbis/MP4 musicbrainz_trackid, etc.
|
||||
for key in ("UFID:http://musicbrainz.org", "musicbrainz_trackid",
|
||||
"MUSICBRAINZ_TRACKID", "----:com.apple.iTunes:MusicBrainz Track Id"):
|
||||
try:
|
||||
val = tags.get(key)
|
||||
except Exception:
|
||||
val = None
|
||||
if not val:
|
||||
continue
|
||||
if hasattr(val, "data"): # ID3 UFID frame
|
||||
val = val.data.decode("utf-8", "ignore")
|
||||
if isinstance(val, (list, tuple)):
|
||||
val = val[0] if val else ""
|
||||
if isinstance(val, bytes):
|
||||
val = val.decode("utf-8", "ignore")
|
||||
val = str(val).strip()
|
||||
if val:
|
||||
return val
|
||||
except Exception as exc:
|
||||
logger.debug(f"export file_recording_mbid failed for {fpath}: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
_mb_service = None
|
||||
_mb_service_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_mb_service():
|
||||
"""Shared MusicBrainzService (client + cache + DB), created lazily so importing this
|
||||
module never triggers a DB/network connection on paths that don't export."""
|
||||
global _mb_service
|
||||
if _mb_service is None:
|
||||
with _mb_service_lock:
|
||||
if _mb_service is None:
|
||||
from core.musicbrainz_service import MusicBrainzService
|
||||
from database.music_database import get_database
|
||||
_mb_service = MusicBrainzService(get_database())
|
||||
return _mb_service
|
||||
|
||||
|
||||
def musicbrainz_recording_mbid(artist: str, title: str) -> Optional[str]:
|
||||
"""Live MusicBrainz ``match_recording`` — the rate-limited tail."""
|
||||
if not title:
|
||||
return None
|
||||
try:
|
||||
svc = _get_mb_service()
|
||||
if not svc:
|
||||
return None
|
||||
result = svc.match_recording(title, artist)
|
||||
if result and result.get("mbid"):
|
||||
return result["mbid"]
|
||||
except Exception as exc:
|
||||
logger.debug(f"export musicbrainz_recording_mbid failed for '{artist} - {title}': {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def build_resolve_fn(
|
||||
*,
|
||||
db_fn: Callable[[str, str], Optional[str]] = db_recording_mbid,
|
||||
file_fn: Callable[[str, str], Optional[str]] = file_recording_mbid,
|
||||
mb_fn: Callable[[str, str], Optional[str]] = musicbrainz_recording_mbid,
|
||||
cache_lookup: Optional[Callable[[str], Optional[str]]] = None,
|
||||
cache_record: Optional[Callable[[str, str], bool]] = None,
|
||||
) -> Callable[[str, str], Tuple[Optional[str], Optional[str]]]:
|
||||
"""Assemble the export ``resolve_fn(artist, title) -> (mbid, source_label)``.
|
||||
|
||||
Runs cache -> DB -> file -> MusicBrainz, and writes a fresh (non-cache) hit back to the
|
||||
persistent cache. All sources are injectable so the wiring is unit-testable; defaults
|
||||
use the real cache module.
|
||||
"""
|
||||
if cache_lookup is None or cache_record is None:
|
||||
from core.exports import recording_mbid_cache as _cache
|
||||
cache_lookup = cache_lookup or _cache.lookup
|
||||
cache_record = cache_record or _cache.record
|
||||
|
||||
def resolve_fn(artist: str, title: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
sources = [
|
||||
(SRC_CACHE, lambda a, t: cache_lookup(normalize_key(a, t))),
|
||||
(SRC_DB, db_fn),
|
||||
(SRC_FILE, file_fn),
|
||||
(SRC_MUSICBRAINZ, mb_fn),
|
||||
]
|
||||
mbid, label = resolve_recording_mbid(artist, title, sources)
|
||||
if mbid and label and label != SRC_CACHE:
|
||||
try:
|
||||
cache_record(normalize_key(artist, title), mbid)
|
||||
except Exception: # noqa: S110 — cache write is best-effort
|
||||
pass
|
||||
return (mbid, label)
|
||||
|
||||
return resolve_fn
|
||||
|
||||
|
||||
__all__ = [
|
||||
"build_resolve_fn",
|
||||
"db_recording_mbid",
|
||||
"file_recording_mbid",
|
||||
"musicbrainz_recording_mbid",
|
||||
]
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
"""Build a JSPF playlist (ListenBrainz-compatible) from resolved SoulSync tracks.
|
||||
|
||||
ListenBrainz's ``POST /1/playlist/create`` requires JSPF where **every track carries a
|
||||
``identifier`` of ``https://musicbrainz.org/recording/<recording-mbid>``** — text-only
|
||||
entries (title/creator alone) are rejected. So a track can only be exported once we've
|
||||
resolved its MusicBrainz *recording* MBID (see ``mbid_resolver``); tracks without one are
|
||||
dropped here and surfaced to the user as "unmatched".
|
||||
|
||||
Pure + I/O-free: callers pass already-resolved track dicts, this returns the JSPF dict
|
||||
(and a small coverage summary). The same JSPF is used for both the downloadable ``.jspf``
|
||||
file and the direct create-playlist POST, so there's one source of truth for the shape.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
MB_RECORDING_PREFIX = "https://musicbrainz.org/recording/"
|
||||
|
||||
# A MusicBrainz MBID is a canonical UUID. Validate to avoid emitting garbage identifiers
|
||||
# that LB would reject (or, worse, that silently point nowhere).
|
||||
_UUID_RE = re.compile(
|
||||
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def is_valid_recording_mbid(mbid: Any) -> bool:
|
||||
"""True when ``mbid`` is a well-formed MusicBrainz UUID."""
|
||||
return bool(mbid) and isinstance(mbid, str) and bool(_UUID_RE.match(mbid.strip()))
|
||||
|
||||
|
||||
def _track_entry(track: Dict[str, Any]) -> Dict[str, Any] | None:
|
||||
"""Build one JSPF track entry, or None if the track has no valid recording MBID."""
|
||||
mbid = (track.get("recording_mbid") or "").strip() if isinstance(track.get("recording_mbid"), str) else ""
|
||||
if not is_valid_recording_mbid(mbid):
|
||||
return None
|
||||
entry: Dict[str, Any] = {"identifier": f"{MB_RECORDING_PREFIX}{mbid}"}
|
||||
# Optional, human-friendly fields — LB ignores them on create but they make the
|
||||
# downloaded .jspf readable and round-trippable.
|
||||
if track.get("title"):
|
||||
entry["title"] = str(track["title"])
|
||||
if track.get("artist"):
|
||||
entry["creator"] = str(track["artist"])
|
||||
if track.get("album"):
|
||||
entry["album"] = str(track["album"])
|
||||
return entry
|
||||
|
||||
|
||||
def build_jspf(
|
||||
title: str,
|
||||
tracks: List[Dict[str, Any]],
|
||||
*,
|
||||
creator: str = "",
|
||||
) -> Tuple[Dict[str, Any], Dict[str, int]]:
|
||||
"""Build a ListenBrainz-compatible JSPF dict from resolved tracks.
|
||||
|
||||
``tracks`` is an ordered list of dicts with ``recording_mbid`` (required to be
|
||||
included), plus optional ``title`` / ``artist`` / ``album``. Tracks without a valid
|
||||
recording MBID are skipped (LB rejects them).
|
||||
|
||||
Returns ``(jspf, summary)`` where ``jspf`` is ``{"playlist": {...}}`` and ``summary``
|
||||
is ``{"total", "included", "skipped"}`` for the coverage display.
|
||||
"""
|
||||
jspf_tracks: List[Dict[str, Any]] = []
|
||||
for t in tracks or []:
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
entry = _track_entry(t)
|
||||
if entry is not None:
|
||||
jspf_tracks.append(entry)
|
||||
|
||||
playlist: Dict[str, Any] = {
|
||||
"title": (title or "SoulSync Export").strip() or "SoulSync Export",
|
||||
"track": jspf_tracks,
|
||||
}
|
||||
if creator:
|
||||
playlist["creator"] = str(creator)
|
||||
|
||||
total = sum(1 for t in (tracks or []) if isinstance(t, dict))
|
||||
summary = {
|
||||
"total": total,
|
||||
"included": len(jspf_tracks),
|
||||
"skipped": total - len(jspf_tracks),
|
||||
}
|
||||
return {"playlist": playlist}, summary
|
||||
|
||||
|
||||
__all__ = ["build_jspf", "is_valid_recording_mbid", "MB_RECORDING_PREFIX"]
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
"""Resolve a playlist track's MusicBrainz *recording* MBID, cheapest source first.
|
||||
|
||||
A ListenBrainz playlist export needs each track's recording MBID (``jspf_export``). A
|
||||
SoulSync track can supply it from several places, in increasing cost:
|
||||
|
||||
1. **resolution cache** — a prior (artist,title)->mbid result (persistent; reused across
|
||||
playlists and runs, so the same song never costs twice).
|
||||
2. **library DB** — ``tracks.musicbrainz_recording_id`` (set by the MusicBrainz
|
||||
enrichment worker).
|
||||
3. **file tags** — ``MUSICBRAINZ_RECORDING_ID`` written into the audio file on import
|
||||
post-processing (catches tracks enriched at import but not via the worker).
|
||||
4. **MusicBrainz lookup** — a live ``match_recording(artist, title)`` (rate-limited
|
||||
~1 req/s; the slow tail — only hit when 1–3 miss).
|
||||
|
||||
This module is the **pure waterfall**: the caller passes ordered ``(label, fn)`` sources,
|
||||
each ``fn(artist, title) -> mbid | None``, and ``resolve_recording_mbid`` returns the
|
||||
first valid hit plus its label (for the live status / stats). The actual I/O (DB query,
|
||||
mutagen read, MB request, cache read/write) lives in the export job that wires the real
|
||||
sources — so this stays trivially unit-testable and short-circuits correctly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Callable, List, Optional, Tuple
|
||||
|
||||
# Source labels (also used in the live-status breakdown).
|
||||
SRC_CACHE = "cache"
|
||||
SRC_DB = "db"
|
||||
SRC_FILE = "file"
|
||||
SRC_MUSICBRAINZ = "musicbrainz"
|
||||
SRC_NONE = None
|
||||
|
||||
_UUID_RE = re.compile(
|
||||
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE
|
||||
)
|
||||
|
||||
Source = Tuple[str, Callable[[str, str], Optional[str]]]
|
||||
|
||||
|
||||
def _valid(mbid: Any) -> Optional[str]:
|
||||
"""Return the trimmed MBID if it's a well-formed UUID, else None."""
|
||||
if not isinstance(mbid, str):
|
||||
return None
|
||||
m = mbid.strip()
|
||||
return m if _UUID_RE.match(m) else None
|
||||
|
||||
|
||||
def normalize_key(artist: Any, title: Any) -> str:
|
||||
"""Stable cache key for an (artist, title) pair — lower, punctuation-stripped,
|
||||
whitespace-collapsed — so trivial variations share a cache entry."""
|
||||
def _n(v: Any) -> str:
|
||||
s = re.sub(r"[^\w\s]", "", str(v or "").lower())
|
||||
return re.sub(r"\s+", " ", s).strip()
|
||||
return f"{_n(artist)}␟{_n(title)}"
|
||||
|
||||
|
||||
def resolve_recording_mbid(
|
||||
artist: str,
|
||||
title: str,
|
||||
sources: List[Source],
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Walk ``sources`` in order; return ``(mbid, label)`` of the first that yields a
|
||||
valid recording MBID, or ``(None, None)`` when every source misses.
|
||||
|
||||
Each source is ``(label, fn)`` and ``fn(artist, title)`` returns an MBID or None. A
|
||||
source that raises is treated as a miss (never aborts the waterfall) — so one flaky
|
||||
lookup (e.g. a MusicBrainz timeout) can't fail the whole export. Short-circuits: a
|
||||
later/expensive source isn't called once an earlier one hits.
|
||||
"""
|
||||
for label, fn in sources or []:
|
||||
try:
|
||||
mbid = _valid(fn(artist, title))
|
||||
except Exception:
|
||||
mbid = None
|
||||
if mbid:
|
||||
return (mbid, label)
|
||||
return (None, None)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"resolve_recording_mbid",
|
||||
"normalize_key",
|
||||
"SRC_CACHE",
|
||||
"SRC_DB",
|
||||
"SRC_FILE",
|
||||
"SRC_MUSICBRAINZ",
|
||||
]
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
"""Orchestrate resolving a playlist's tracks to recording MBIDs for export (#903).
|
||||
|
||||
This is the testable heart of the export job: walk the playlist's tracks, resolve each to a
|
||||
MusicBrainz recording MBID via an injected ``resolve_fn`` (which the job wires to the
|
||||
cache -> DB -> file -> MusicBrainz waterfall), dedup repeated songs within the run so they
|
||||
only cost one resolution, build the ordered "pseudo-playlist" of resolved tracks, and tally
|
||||
live stats (resolved / unmatched / per-source / deduped) for the on-card status display.
|
||||
|
||||
Pure: all I/O (DB, file reads, MusicBrainz, cache) is behind ``resolve_fn`` and the optional
|
||||
``on_progress`` callback, so the dedup + accounting logic is unit-testable without any
|
||||
network or database. The returned ``resolved`` list feeds straight into ``jspf_export``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from core.exports.mbid_resolver import normalize_key
|
||||
|
||||
# resolve_fn(artist, title) -> (recording_mbid|None, source_label|None)
|
||||
ResolveFn = Callable[[str, str], Tuple[Optional[str], Optional[str]]]
|
||||
ProgressFn = Callable[[int, int, Dict[str, Any]], None]
|
||||
|
||||
|
||||
def _field(track: Dict[str, Any], *names: str) -> str:
|
||||
"""First non-empty value among ``names`` (handles both playlist + LB-cache shapes)."""
|
||||
for n in names:
|
||||
v = track.get(n)
|
||||
if v:
|
||||
return str(v)
|
||||
return ""
|
||||
|
||||
|
||||
def resolve_playlist_tracks(
|
||||
tracks: List[Dict[str, Any]],
|
||||
resolve_fn: ResolveFn,
|
||||
*,
|
||||
on_progress: Optional[ProgressFn] = None,
|
||||
id_key: str = "recording_mbid",
|
||||
) -> Dict[str, Any]:
|
||||
"""Resolve every track to an ID and build the export pseudo-playlist.
|
||||
|
||||
``resolve_fn(artist, title) -> (id, source)`` returns whatever ID the target needs —
|
||||
a MusicBrainz recording MBID for ListenBrainz/JSPF (the default), or a Spotify/Deezer
|
||||
track ID for service export. ``id_key`` names the field that ID lands under in each
|
||||
resolved entry (defaults to ``recording_mbid`` so existing LB/JSPF callers are
|
||||
untouched). The dedup + stats + ordering logic is identical regardless of ID type.
|
||||
|
||||
``tracks`` items may use ``artist``/``artist_name`` and ``title``/``track_name`` and
|
||||
``album``/``album_name`` (both the mirrored-playlist and LB-cache shapes are accepted).
|
||||
|
||||
Returns ``{"resolved": [...], "stats": {...}}`` where each resolved entry is
|
||||
``{artist, title, album, <id_key>}`` (the ID is None when unmatched), in original
|
||||
order, and stats carries ``total, resolved, unmatched, deduped, by_source``.
|
||||
"""
|
||||
total = len(tracks or [])
|
||||
memo: Dict[str, Tuple[Optional[str], Optional[str]]] = {}
|
||||
resolved: List[Dict[str, Any]] = []
|
||||
stats: Dict[str, Any] = {
|
||||
"total": total, "resolved": 0, "unmatched": 0, "deduped": 0, "by_source": {},
|
||||
}
|
||||
|
||||
for i, t in enumerate(tracks or []):
|
||||
if not isinstance(t, dict):
|
||||
t = {}
|
||||
artist = _field(t, "artist", "artist_name", "creator")
|
||||
title = _field(t, "title", "track_name", "name")
|
||||
album = _field(t, "album", "album_name", "release_name")
|
||||
key = normalize_key(artist, title)
|
||||
|
||||
if key in memo:
|
||||
mbid, source = memo[key]
|
||||
stats["deduped"] += 1
|
||||
fresh = False
|
||||
else:
|
||||
mbid, source = resolve_fn(artist, title)
|
||||
memo[key] = (mbid, source)
|
||||
fresh = True
|
||||
|
||||
resolved.append({"artist": artist, "title": title, "album": album, id_key: mbid})
|
||||
|
||||
if mbid:
|
||||
stats["resolved"] += 1
|
||||
if fresh and source:
|
||||
stats["by_source"][source] = stats["by_source"].get(source, 0) + 1
|
||||
else:
|
||||
stats["unmatched"] += 1
|
||||
|
||||
if on_progress is not None:
|
||||
try:
|
||||
on_progress(i + 1, total, stats)
|
||||
except Exception: # noqa: S110 — a progress-display error must never fail the export
|
||||
pass
|
||||
|
||||
return {"resolved": resolved, "stats": stats}
|
||||
|
||||
|
||||
__all__ = ["resolve_playlist_tracks"]
|
||||
|
|
@ -1,126 +0,0 @@
|
|||
"""Persistent (artist,title) -> MusicBrainz recording-MBID cache for playlist export.
|
||||
|
||||
The export waterfall (``core.exports.mbid_resolver``) ends in a live MusicBrainz lookup
|
||||
that's rate-limited to ~1 req/s — the slow tail of exporting a big playlist. Remembering a
|
||||
resolved recording MBID ONCE means the same song never costs a second lookup, across every
|
||||
future export and every playlist it appears in.
|
||||
|
||||
Mirrors ``core.metadata.album_mbid_cache`` exactly: a tiny SQLite table, lazy DB accessor,
|
||||
every function wrapped so any DB error degrades to a cache miss / no-op. If this module
|
||||
breaks, exports still work — they just re-resolve via the live waterfall like a cold cache.
|
||||
Key is the normalized ``track_key`` from ``mbid_resolver.normalize_key(artist, title)``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("exports.recording_mbid_cache")
|
||||
|
||||
_db_factory_lock = threading.Lock()
|
||||
_db_factory = None
|
||||
|
||||
|
||||
def _get_database():
|
||||
"""Resolve the MusicDatabase singleton lazily; None on any failure (treated as miss)."""
|
||||
global _db_factory
|
||||
with _db_factory_lock:
|
||||
if _db_factory is None:
|
||||
try:
|
||||
from database.music_database import get_database
|
||||
_db_factory = get_database
|
||||
except Exception as exc:
|
||||
logger.warning(f"Recording-MBID cache: could not load database module: {exc}")
|
||||
return None
|
||||
try:
|
||||
return _db_factory()
|
||||
except Exception as exc:
|
||||
logger.warning(f"Recording-MBID cache: database accessor failed: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def lookup(track_key: str) -> Optional[str]:
|
||||
"""Read a cached recording MBID for ``track_key``; None on miss or any DB error."""
|
||||
if not track_key:
|
||||
return None
|
||||
db = _get_database()
|
||||
if db is None:
|
||||
return None
|
||||
conn = None
|
||||
try:
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT recording_mbid FROM mb_recording_cache WHERE track_key = ? LIMIT 1",
|
||||
(track_key,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
return (row[0] if not hasattr(row, "keys") else row["recording_mbid"]) or None
|
||||
except Exception as exc:
|
||||
logger.debug(f"Recording-MBID cache lookup failed: {exc}")
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110 — finally cleanup
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def record(track_key: str, recording_mbid: str) -> bool:
|
||||
"""Persist ``track_key`` -> ``recording_mbid`` (idempotent). False on any failure."""
|
||||
if not track_key or not recording_mbid:
|
||||
return False
|
||||
db = _get_database()
|
||||
if db is None:
|
||||
return False
|
||||
conn = None
|
||||
try:
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO mb_recording_cache "
|
||||
"(track_key, recording_mbid, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)",
|
||||
(track_key, recording_mbid),
|
||||
)
|
||||
conn.commit()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug(f"Recording-MBID cache record failed: {exc}")
|
||||
return False
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110 — finally cleanup
|
||||
pass
|
||||
|
||||
|
||||
def clear_all() -> bool:
|
||||
"""Wipe the cache (tests / forced re-resolve)."""
|
||||
db = _get_database()
|
||||
if db is None:
|
||||
return False
|
||||
conn = None
|
||||
try:
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM mb_recording_cache")
|
||||
conn.commit()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning(f"Recording-MBID cache clear failed: {exc}")
|
||||
return False
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110 — finally cleanup
|
||||
pass
|
||||
|
||||
|
||||
__all__ = ["lookup", "record", "clear_all"]
|
||||
|
|
@ -36,34 +36,9 @@ import requests as http_requests
|
|||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.quality.source_map import quality_from_tidal_tier, quality_tier_for_source
|
||||
|
||||
logger = get_logger("hifi_client")
|
||||
|
||||
# A media playlist whose total runtime is below this fraction of the track's
|
||||
# real duration is a preview (some Monochrome instances only have 30s Tidal
|
||||
# DOWNLOAD access — a 220s track comes back as ~30s of segments + ENDLIST).
|
||||
_PREVIEW_DURATION_RATIO = 0.85
|
||||
_EXTINF_RE = re.compile(r'#EXTINF:\s*([0-9.]+)')
|
||||
|
||||
|
||||
def hls_total_seconds(playlist_text: str) -> float:
|
||||
"""Sum the ``#EXTINF`` segment durations in an HLS media playlist."""
|
||||
return sum(float(x) for x in _EXTINF_RE.findall(playlist_text or ''))
|
||||
|
||||
|
||||
def is_preview_playlist(playlist_s: float, track_s: float,
|
||||
ratio: float = _PREVIEW_DURATION_RATIO) -> bool:
|
||||
"""True when the playlist runtime is far shorter than the track's real
|
||||
duration (a preview). Returns False when either duration is unknown, so a
|
||||
missing reference never false-positives — the post-download audio guard is
|
||||
the safety net.
|
||||
"""
|
||||
if not playlist_s or not track_s or track_s <= 0:
|
||||
return False
|
||||
return playlist_s < track_s * ratio
|
||||
|
||||
|
||||
# HLS quality presets mapping to /trackManifests/ format parameters
|
||||
HLS_QUALITY_MAP = {
|
||||
'hires': {
|
||||
|
|
@ -165,81 +140,6 @@ def compute_new_default_pushes(all_defaults, offered, legacy_baseline, existing)
|
|||
return to_add, new_offered
|
||||
|
||||
|
||||
_EXTINF_RE = re.compile(r'#EXTINF:\s*([0-9]+(?:\.[0-9]+)?)')
|
||||
|
||||
|
||||
def sum_hls_segment_seconds(playlist_text: str) -> float:
|
||||
"""Total audio seconds an HLS media playlist actually provides — the sum of its
|
||||
``#EXTINF`` segment durations. This is the authoritative "how much audio is really
|
||||
here" signal: a PREVIEW manifest serves only ~30s of segments even though the track
|
||||
is full-length, so summing EXTINF catches it before we waste the download. Returns
|
||||
0.0 when the playlist has no EXTINF lines (master playlists, legacy manifests) — the
|
||||
caller treats 0 as 'unknown', never as 'preview'."""
|
||||
total = 0.0
|
||||
for m in _EXTINF_RE.finditer(playlist_text or ''):
|
||||
try:
|
||||
total += float(m.group(1))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return total
|
||||
|
||||
|
||||
def is_short_audio(actual_seconds: float, expected_seconds: float, threshold: float = 0.8) -> bool:
|
||||
"""True when ``actual`` is meaningfully shorter than ``expected`` — i.e. a preview
|
||||
clip or a truncated/corrupt download. Conservative: returns False whenever either
|
||||
value is missing/zero (unknown ⇒ never reject), and only trips below ``threshold``
|
||||
of the expected length (previews are ~15% of full, so the margin is huge)."""
|
||||
try:
|
||||
a, e = float(actual_seconds or 0), float(expected_seconds or 0)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if a <= 0 or e <= 0:
|
||||
return False
|
||||
return a < e * threshold
|
||||
|
||||
|
||||
def is_fake_lossless_bitrate(size_bytes, claimed_seconds, sample_rate, bits_per_sample,
|
||||
channels, min_ratio: float = 0.30) -> bool:
|
||||
"""True when a 'lossless' file's data is FAR too small for its claimed length — the
|
||||
fingerprint of a ~30s preview whose STREAMINFO/container was faked to the full
|
||||
duration (so every length header reads 'full' and only the bitrate gives it away).
|
||||
Real FLAC is ~40-75% of raw PCM; a preview padded to full length implies single-digit
|
||||
%. Conservative: 0 / bad inputs return False (never reject on unknowns)."""
|
||||
try:
|
||||
sz, secs = float(size_bytes or 0), float(claimed_seconds or 0)
|
||||
sr, bits, ch = int(sample_rate or 0), int(bits_per_sample or 0), int(channels or 0)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if sz <= 0 or secs <= 0 or sr <= 0 or bits <= 0 or ch <= 0:
|
||||
return False
|
||||
return (sz * 8 / secs) < (sr * bits * ch) * min_ratio
|
||||
|
||||
|
||||
def parse_ffmpeg_time(stderr_text) -> float:
|
||||
"""The last ``time=HH:MM:SS.xx`` ffmpeg prints while decoding — the REAL decoded
|
||||
length (immune to a faked container/STREAMINFO duration). 0.0 if not found."""
|
||||
last = 0.0
|
||||
for m in re.finditer(r'time=(\d+):(\d+):(\d+(?:\.\d+)?)', stderr_text or ''):
|
||||
last = int(m.group(1)) * 3600 + int(m.group(2)) * 60 + float(m.group(3))
|
||||
return last
|
||||
|
||||
|
||||
def is_preview_download(real_seconds, reference_seconds, *, is_lossless, size_bytes,
|
||||
sample_rate, bits_per_sample, channels):
|
||||
"""Is a finished file a preview/truncated fake? Two independent signals, so it fires
|
||||
even when the fakery declares full length at every layer:
|
||||
1. DECODED length far below the reference (the ground truth, when a decoder ran);
|
||||
2. for lossless, an impossibly-low implied bitrate (no decoder needed).
|
||||
Returns ``(is_fake, reason)``."""
|
||||
if real_seconds and is_short_audio(real_seconds, reference_seconds):
|
||||
return True, "decoded %.0fs of %.0fs" % (real_seconds, reference_seconds)
|
||||
if is_lossless and is_fake_lossless_bitrate(size_bytes, reference_seconds, sample_rate,
|
||||
bits_per_sample, channels):
|
||||
kbps = (float(size_bytes) * 8 / reference_seconds / 1000) if reference_seconds else 0
|
||||
return True, "%.0fkbps lossless over %.0fs (far too low — a ~30s preview)" % (kbps, reference_seconds)
|
||||
return False, ""
|
||||
|
||||
|
||||
# Run the new-default push at most once per process.
|
||||
_pushed_new_defaults = False
|
||||
|
||||
|
|
@ -257,10 +157,7 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
self.download_path = Path(download_path)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._instances = []
|
||||
self._instance_lock = threading.Lock()
|
||||
|
|
@ -725,8 +622,7 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
|
||||
return init_uri, segment_uris
|
||||
|
||||
def _get_hls_manifest(self, track_id: int, quality: str = 'lossless',
|
||||
expected_duration_s: float = 0) -> Optional[Dict]:
|
||||
def _get_hls_manifest(self, track_id: int, quality: str = 'lossless') -> Optional[Dict]:
|
||||
q_info = HLS_QUALITY_MAP.get(quality, HLS_QUALITY_MAP['lossless'])
|
||||
formats = q_info['formats']
|
||||
|
||||
|
|
@ -766,7 +662,6 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
logger.warning(f"Failed to parse HLS playlist for track {track_id}: {e}")
|
||||
return None
|
||||
|
||||
media_text = playlist_text # the playlist that actually carries the EXTINF segments
|
||||
if '#EXT-X-STREAM-INF' in playlist_text and segment_uris:
|
||||
playlist_uri = segment_uris[0]
|
||||
try:
|
||||
|
|
@ -774,26 +669,11 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
variant_resp = self.session.get(playlist_uri, allow_redirects=True, timeout=30)
|
||||
variant_resp.raise_for_status()
|
||||
variant_text = variant_resp.text
|
||||
media_text = variant_text
|
||||
init_uri, segment_uris = self._parse_hls_playlist(variant_text, playlist_uri)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch variant playlist for track {track_id}: {e}")
|
||||
return None
|
||||
|
||||
# Preview detection — some instances only have 30s Tidal DOWNLOAD
|
||||
# access, returning a playlist far shorter than the real track. Decline
|
||||
# it (and rotate the instance) so the orchestrator falls through to a
|
||||
# real source instead of fetching a 30s file that gets quarantined.
|
||||
playlist_s = hls_total_seconds(media_text)
|
||||
if is_preview_playlist(playlist_s, expected_duration_s):
|
||||
logger.warning(
|
||||
f"HiFi manifest for track {track_id} ({quality}) is a "
|
||||
f"{playlist_s:.0f}s preview of a {expected_duration_s:.0f}s track — "
|
||||
f"declining this instance"
|
||||
)
|
||||
self._rotate_instance(self._current_instance)
|
||||
return None
|
||||
|
||||
if init_uri:
|
||||
logger.info(f"HiFi HLS manifest for track {track_id}: "
|
||||
f"init segment + {len(segment_uris)} segments ({quality})")
|
||||
|
|
@ -807,9 +687,6 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
'extension': q_info['extension'],
|
||||
'codec': q_info['codec'],
|
||||
'quality': quality,
|
||||
# Real audio length the manifest provides (sum of EXTINF) — used to reject
|
||||
# preview manifests before downloading. 0.0 = unknown (don't reject).
|
||||
'manifest_duration': sum_hls_segment_seconds(media_text),
|
||||
}
|
||||
|
||||
def _get_legacy_track_manifest(self, track_id: int, quality: str = 'lossless') -> Optional[Dict]:
|
||||
|
|
@ -834,57 +711,15 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
'quality': quality,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _probe_audio_seconds(path) -> float:
|
||||
"""Real decoded audio length of a finished file, via mutagen (already a dep).
|
||||
0.0 on any failure — the caller treats 0 as 'unknown' and never rejects on it."""
|
||||
try:
|
||||
from mutagen import File as _MutagenFile
|
||||
mf = _MutagenFile(str(path))
|
||||
info = getattr(mf, 'info', None) if mf is not None else None
|
||||
if info is not None:
|
||||
return float(getattr(info, 'length', 0) or 0)
|
||||
except Exception as _probe_err: # noqa: BLE001
|
||||
logger.debug("mutagen audio-length probe failed for %s: %s", path, _probe_err)
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _find_ffmpeg():
|
||||
ff = shutil.which('ffmpeg')
|
||||
if ff:
|
||||
return ff
|
||||
cand = Path(__file__).parent.parent / 'tools' / ('ffmpeg.exe' if os.name == 'nt' else 'ffmpeg')
|
||||
return str(cand) if cand.exists() else None
|
||||
|
||||
def _probe_real_seconds(self, path) -> float:
|
||||
"""REAL decoded audio length via ffmpeg — decodes the actual frames, so it sees
|
||||
through a faked STREAMINFO/container duration (a 30s preview claiming full
|
||||
length decodes to 30s). 0.0 if ffmpeg is unavailable or on error."""
|
||||
ff = self._find_ffmpeg()
|
||||
if not ff:
|
||||
return 0.0
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[ff, '-hide_banner', '-nostdin', '-i', str(path), '-map', '0:a:0', '-f', 'null', '-'],
|
||||
capture_output=True, text=True, timeout=180)
|
||||
return parse_ffmpeg_time(proc.stderr)
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _flac_props(path):
|
||||
"""(sample_rate, bits_per_sample, channels) for the bitrate sanity check, or None."""
|
||||
try:
|
||||
from mutagen.flac import FLAC
|
||||
si = FLAC(str(path)).info
|
||||
return (si.sample_rate, si.bits_per_sample, si.channels)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _demux_flac(self, input_path: Path, output_path: Path) -> None:
|
||||
ffmpeg = self._find_ffmpeg()
|
||||
ffmpeg = shutil.which('ffmpeg')
|
||||
if not ffmpeg:
|
||||
raise RuntimeError('ffmpeg is required to demux FLAC from MP4. Install ffmpeg and retry.')
|
||||
tools_dir = Path(__file__).parent.parent / 'tools'
|
||||
ffmpeg_candidate = tools_dir / ('ffmpeg.exe' if os.name == 'nt' else 'ffmpeg')
|
||||
if ffmpeg_candidate.exists():
|
||||
ffmpeg = str(ffmpeg_candidate)
|
||||
else:
|
||||
raise RuntimeError('ffmpeg is required to demux FLAC from MP4. Install ffmpeg and retry.')
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
|
|
@ -916,18 +751,13 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
loop = asyncio.get_event_loop()
|
||||
tracks = await loop.run_in_executor(None, lambda: self.search_raw(query))
|
||||
|
||||
quality_key = quality_tier_for_source('hifi', default='lossless')
|
||||
quality_key = config_manager.get('hifi_download.quality', 'lossless')
|
||||
q_info = HLS_QUALITY_MAP.get(quality_key, HLS_QUALITY_MAP['lossless'])
|
||||
|
||||
# HiFi is Tidal-backed; stamp the configured tier so the global
|
||||
# ranker sees real sample_rate/bit_depth, not just 'flac'.
|
||||
tier_quality = quality_from_tidal_tier(quality_key)
|
||||
|
||||
results = []
|
||||
for t in tracks:
|
||||
try:
|
||||
tr = self._to_track_result(t, q_info)
|
||||
tr.set_quality(tier_quality)
|
||||
results.append(tr)
|
||||
except Exception as e:
|
||||
logger.debug(f"Skipping track result conversion: {e}")
|
||||
|
|
@ -998,7 +828,7 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
)
|
||||
|
||||
def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]:
|
||||
quality_key = quality_tier_for_source('hifi', default='lossless')
|
||||
quality_key = config_manager.get('hifi_download.quality', 'lossless')
|
||||
chain = ['hires', 'lossless', 'high', 'low']
|
||||
start = chain.index(quality_key) if quality_key in chain else 1
|
||||
allow_fallback = config_manager.get('hifi_download.allow_fallback', True)
|
||||
|
|
@ -1006,26 +836,12 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
|
||||
MIN_AUDIO_SIZE = 100 * 1024
|
||||
|
||||
# Expected track length, drives every preview/truncation guard here:
|
||||
# * _get_hls_manifest's pre-download is_preview_playlist check
|
||||
# * the pre-download is_short_audio manifest check
|
||||
# * the post-download is_preview_download faked-header decode check
|
||||
# Best-effort: a 0 here just disables the duration checks, never rejects.
|
||||
expected_s = 0.0
|
||||
try:
|
||||
info = self.get_track_info(track_id) or {}
|
||||
expected_s = float(info.get('duration_s') or 0)
|
||||
except Exception:
|
||||
expected_s = 0.0
|
||||
expected_duration_s = expected_s # alias for _get_hls_manifest's param name
|
||||
|
||||
for q_key in chain:
|
||||
if self.shutdown_check and self.shutdown_check():
|
||||
logger.info("Shutdown detected, aborting HiFi download")
|
||||
return None
|
||||
|
||||
manifest_info = self._get_hls_manifest(track_id, quality=q_key,
|
||||
expected_duration_s=expected_duration_s)
|
||||
manifest_info = self._get_hls_manifest(track_id, quality=q_key)
|
||||
if (
|
||||
not manifest_info
|
||||
or (
|
||||
|
|
@ -1036,18 +852,6 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
logger.warning(f"No HLS manifest at quality {q_key}, trying next")
|
||||
continue
|
||||
|
||||
# Preview guard #1 (pre-download): a preview manifest serves only ~30s of
|
||||
# segments for a full-length track. A preview means THIS SOURCE only has a
|
||||
# preview of the track — lower quality tiers are the SAME preview — so abort
|
||||
# HiFi entirely and let the orchestrator fall through to the next SOURCE
|
||||
# (soulseek/youtube/…), rather than landing a lower-tier preview.
|
||||
manifest_s = float(manifest_info.get('manifest_duration') or 0)
|
||||
if is_short_audio(manifest_s, expected_s):
|
||||
logger.warning(
|
||||
"HiFi has only a PREVIEW of '%s' (manifest %.0fs of %.0fs at %s) — "
|
||||
"failing HiFi so the next source is tried", display_name, manifest_s, expected_s, q_key)
|
||||
return None
|
||||
|
||||
extension = manifest_info['extension']
|
||||
safe_name = re.sub(r'[<>:"/\\|?*]', '_', display_name)
|
||||
out_filename = f"{safe_name}.{extension}"
|
||||
|
|
@ -1127,31 +931,6 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
out_path.unlink(missing_ok=True)
|
||||
continue
|
||||
|
||||
# Preview guard #2 (post-download): the real catch. HiFi previews fake the
|
||||
# FULL length in every header — manifest EXTINF, m4a moov, FLAC
|
||||
# total_samples — so only the DECODED audio (or, for lossless, the
|
||||
# bitrate) reveals the ~30s truth. Reference = the largest length any
|
||||
# header claims (so the file's own faked claim becomes the bar its real
|
||||
# audio must clear); is_preview_download decodes + bitrate-checks.
|
||||
ref_s = max(expected_s, self._probe_audio_seconds(out_path))
|
||||
real_s = self._probe_real_seconds(out_path)
|
||||
props = self._flac_props(out_path) if is_flac else None
|
||||
fake, why = is_preview_download(
|
||||
real_s, ref_s, is_lossless=is_flac, size_bytes=final_size,
|
||||
sample_rate=(props[0] if props else 0),
|
||||
bits_per_sample=(props[1] if props else 0),
|
||||
channels=(props[2] if props else 0))
|
||||
if fake:
|
||||
# A preview at this tier means the SOURCE only has a preview — every
|
||||
# lower tier is the same 30s clip (and the lossy ones dodge the
|
||||
# bitrate check). Abort HiFi so the orchestrator tries the next
|
||||
# SOURCE, instead of cascading down into an accepted lower-tier preview.
|
||||
logger.warning(
|
||||
"HiFi has only a PREVIEW of '%s' (%s at %s) — failing HiFi so the "
|
||||
"next source is tried", display_name, why, q_key)
|
||||
out_path.unlink(missing_ok=True)
|
||||
return None
|
||||
|
||||
logger.info(f"HiFi download complete ({q_key}): {out_path} "
|
||||
f"({final_size / (1024*1024):.1f} MB)")
|
||||
return str(out_path)
|
||||
|
|
|
|||
|
|
@ -1,98 +0,0 @@
|
|||
"""Canonical album grouping for the SoulSync standalone import.
|
||||
|
||||
SoulSync grouped imported tracks into albums by the album NAME string
|
||||
(``_stable_soulsync_id("artist::album_name")``). That splits one release into
|
||||
several album rows whenever the name string drifts between imports (case,
|
||||
punctuation, ``(Deluxe Edition)`` suffixes, source-A-vs-B spelling), and every
|
||||
downstream tool (Library Re-tag, Cover-Art Filler) then dresses each split row
|
||||
in its own cover — so songs that belong to one album end up with different art
|
||||
(Sokhi).
|
||||
|
||||
This module is the pure, seam-testable heart of "group by canonical id, not
|
||||
name": when an imported track carries a metadata-source RELEASE id, prefer
|
||||
matching an existing album row by that id over the fragile name string, so the
|
||||
SAME release always lands in ONE album row regardless of how its name was typed.
|
||||
|
||||
Scope (deliberate): this unifies differently-named imports of the SAME release.
|
||||
It does NOT merge a track that genuinely matched a SINGLE release (a different
|
||||
release id) into its parent album — that needs single->album resolution upstream
|
||||
and is a separate change. New imports only; existing rows are left untouched.
|
||||
|
||||
Pure SQL-over-a-cursor; no app singletons, so it tests against an in-memory DB.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("imports.album_grouping")
|
||||
|
||||
# Album source-id columns this grouping may key on. An allowlist (not arbitrary
|
||||
# interpolation) — the column name IS spliced into SQL, so it must be a known,
|
||||
# trusted identifier. Mirrors get_library_source_id_columns()' 'album' values.
|
||||
ALLOWED_ALBUM_SOURCE_COLS = frozenset({
|
||||
"spotify_album_id",
|
||||
"itunes_album_id",
|
||||
"deezer_id",
|
||||
"soul_id",
|
||||
"discogs_id",
|
||||
"musicbrainz_release_id",
|
||||
})
|
||||
|
||||
|
||||
def find_existing_soulsync_album_id(
|
||||
cursor: Any,
|
||||
*,
|
||||
name_key_id: str,
|
||||
artist_id: str,
|
||||
album_name: str,
|
||||
album_source_col: Optional[str] = None,
|
||||
album_source_id: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve the existing ``soulsync`` album row a track should join, or None
|
||||
(caller inserts a new row keyed by ``name_key_id``).
|
||||
|
||||
Match precedence:
|
||||
1. ``name_key_id`` — the exact prior stable-name-hash id (unchanged
|
||||
behaviour: a re-import with the identical name hits its own row).
|
||||
2. ``album_source_col == album_source_id`` — CANONICAL grouping: an
|
||||
existing row already carrying THIS release's source id, so a
|
||||
differently-named import of the same release unifies instead of
|
||||
splitting. Only when the column is allow-listed and the id is non-empty.
|
||||
3. ``(title, artist_id)`` — the legacy name match (kept so nothing that
|
||||
grouped before stops grouping now).
|
||||
"""
|
||||
cursor.execute(
|
||||
"SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'",
|
||||
(name_key_id,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
|
||||
if album_source_col in ALLOWED_ALBUM_SOURCE_COLS and album_source_id:
|
||||
try:
|
||||
cursor.execute(
|
||||
f"SELECT id FROM albums WHERE {album_source_col} = ? "
|
||||
"AND server_source = 'soulsync' LIMIT 1",
|
||||
(album_source_id,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
except Exception as exc:
|
||||
# That source has no dedicated album column on this DB (e.g. Deezer
|
||||
# doesn't split per-entity id columns) — fall through to the name
|
||||
# match rather than break the import. Mirrors the guarded source-id
|
||||
# UPDATE the caller already does on insert.
|
||||
logger.debug("album source-id lookup skipped (%s): %s", album_source_col, exc)
|
||||
|
||||
cursor.execute(
|
||||
"SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? "
|
||||
"AND server_source = 'soulsync' LIMIT 1",
|
||||
(album_name, artist_id),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
return row[0] if row else None
|
||||
|
|
@ -156,11 +156,8 @@ def score_file_against_track(
|
|||
score = 0.0
|
||||
|
||||
# Title similarity (TITLE_WEIGHT). Falls back to filename stem when
|
||||
# the file has no title tag — strip a leading track-number prefix off that
|
||||
# stem (#890) so "01 - Sun It Rises" scores against "Sun It Rises".
|
||||
# the file has no title tag.
|
||||
title = file_tags.get('title') or os.path.splitext(os.path.basename(file_path))[0]
|
||||
from core.imports.paths import strip_leading_track_number
|
||||
title = strip_leading_track_number(title)
|
||||
track_name = track.get('name', '')
|
||||
score += similarity(title, track_name) * TITLE_WEIGHT
|
||||
|
||||
|
|
|
|||
|
|
@ -1,94 +0,0 @@
|
|||
"""Resolve a track's position WITHIN its album's track list.
|
||||
|
||||
The bug this fixes: a track auto-downloaded from the playlist pipeline / wishlist /
|
||||
watchlist is identified as belonging to an album, but the per-track position is
|
||||
unknown — Deezer's search/track and MusicBrainz's recording lookups don't carry a
|
||||
track position (only their album endpoint does). ``detect_album_info_web`` then
|
||||
leaves ``track_number = None``, the import pipeline falls through to the default-1
|
||||
floor, and the file lands as ``01/1`` even though the album is known
|
||||
(``core/imports/context.py``). Verified live: e.g. Deezer says "Obelisk" is track
|
||||
9 of *The Grand Mirage*, but it was tagged 1/1.
|
||||
|
||||
This is the pure matcher: given the album's track list (fetched by the caller via
|
||||
``core.metadata.album_tracks.get_album_tracks_for_source`` — so this stays
|
||||
source-agnostic and I/O-free) plus the track's own identifiers, return its real
|
||||
``(track_number, disc_number)``. Match priority is by reliability:
|
||||
|
||||
1. **ISRC** — an exact recording identity; trusted immediately.
|
||||
2. **source track id** — exact within this album.
|
||||
3. **normalized title** — last resort.
|
||||
|
||||
Returns ``(None, None)`` on no confident match, so the caller keeps its existing
|
||||
behaviour (never worse than today).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, List, Optional, Tuple
|
||||
|
||||
|
||||
def _norm_title(value: Any) -> str:
|
||||
"""Lower, strip punctuation, collapse whitespace — for tolerant title match."""
|
||||
s = re.sub(r"[^\w\s]", "", str(value or "").lower())
|
||||
return re.sub(r"\s+", " ", s).strip()
|
||||
|
||||
|
||||
def _pos_int(value: Any) -> Optional[int]:
|
||||
try:
|
||||
n = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return n if n >= 1 else None
|
||||
|
||||
|
||||
def resolve_track_position_in_album(
|
||||
album_tracks: List[dict],
|
||||
*,
|
||||
title: str = "",
|
||||
track_id: str = "",
|
||||
isrc: str = "",
|
||||
) -> Tuple[Optional[int], Optional[int]]:
|
||||
"""Return ``(track_number, disc_number)`` for this track within ``album_tracks``,
|
||||
or ``(None, None)`` when no confident match is found.
|
||||
|
||||
``album_tracks`` is the list under ``get_album_tracks_for_source(...)['tracks']``
|
||||
— each entry has ``track_number`` / ``disc_number`` / ``id`` / ``name`` / ``isrc``.
|
||||
Entries without a valid positive ``track_number`` are skipped. Pure: no I/O.
|
||||
"""
|
||||
if not album_tracks:
|
||||
return (None, None)
|
||||
|
||||
want_isrc = str(isrc or "").strip().upper()
|
||||
want_id = str(track_id or "").strip()
|
||||
want_title = _norm_title(title)
|
||||
|
||||
by_id: Optional[Tuple[int, int]] = None
|
||||
by_title: Optional[Tuple[int, int]] = None
|
||||
|
||||
for t in album_tracks:
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
tn = _pos_int(t.get("track_number"))
|
||||
if tn is None:
|
||||
continue
|
||||
dn = _pos_int(t.get("disc_number")) or 1
|
||||
|
||||
# 1) ISRC — exact recording. Win immediately.
|
||||
if want_isrc and str(t.get("isrc") or "").strip().upper() == want_isrc:
|
||||
return (tn, dn)
|
||||
# 2) source track id — exact within the album.
|
||||
if by_id is None and want_id and str(t.get("id") or "").strip() == want_id:
|
||||
by_id = (tn, dn)
|
||||
# 3) normalized title — last resort.
|
||||
if by_title is None and want_title and _norm_title(t.get("name")) == want_title:
|
||||
by_title = (tn, dn)
|
||||
|
||||
if by_id is not None:
|
||||
return by_id
|
||||
if by_title is not None:
|
||||
return by_title
|
||||
return (None, None)
|
||||
|
||||
|
||||
__all__ = ["resolve_track_position_in_album"]
|
||||
|
|
@ -8,10 +8,6 @@ from __future__ import annotations
|
|||
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("imports.context")
|
||||
|
||||
|
||||
def _as_dict(value: Any) -> Dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
|
@ -183,12 +179,7 @@ def get_import_clean_title(
|
|||
if not title:
|
||||
track_info = get_import_track_info(context)
|
||||
title = _first_value(track_info, "name", "title", default="")
|
||||
title = str(title or default)
|
||||
# #890: strip a leading track-number prefix that leaked from a filename stem
|
||||
# (e.g. "01 - Sun It Rises" → "Sun It Rises") so it matches the canonical title.
|
||||
# Conservative — clean source titles ("7 Rings" etc.) pass through untouched.
|
||||
from core.imports.paths import strip_leading_track_number
|
||||
return strip_leading_track_number(title)
|
||||
return str(title or default)
|
||||
|
||||
|
||||
def get_import_clean_album(
|
||||
|
|
@ -328,11 +319,7 @@ def build_import_album_info(
|
|||
(album_info or {}).get("track_number")
|
||||
or track_info.get("track_number")
|
||||
or original_search.get("track_number")
|
||||
# "Track 01" bug: default to 0 (the codebase's "unknown" sentinel,
|
||||
# same as total_tracks below), NOT 1. A fabricated 1 looks
|
||||
# authoritative and blocks the pipeline's downstream recovery
|
||||
# (embedded file tag / resolve chain); 0 lets it fall through.
|
||||
or 0
|
||||
or 1
|
||||
)
|
||||
disc_number = (
|
||||
(album_info or {}).get("disc_number")
|
||||
|
|
@ -433,152 +420,20 @@ def detect_album_info_web(context, artist_context=None):
|
|||
track_name.strip().lower(),
|
||||
artist_name.strip().lower(),
|
||||
}:
|
||||
_tn = track_info.get("track_number")
|
||||
_dn = track_info.get("disc_number")
|
||||
# The album is identified but discovery often doesn't carry the per-track
|
||||
# POSITION — Deezer's search/track and MusicBrainz's recording lookups omit
|
||||
# it (only their album endpoint has it). Without a position the pipeline
|
||||
# falls through to the default-1 floor and files an album track as 01/1
|
||||
# (e.g. Deezer says "Obelisk" is track 9 of The Grand Mirage). Resolve the
|
||||
# REAL position from the album's own track list when we have its id.
|
||||
# Fail-safe: leaves the numbers untouched on any miss, so behaviour is
|
||||
# never worse than the old preserve-None-and-fall-through.
|
||||
if _tn is None:
|
||||
_tn, _dn = _resolve_album_position_from_source(context, artist_context, _dn)
|
||||
return build_import_album_info(
|
||||
context,
|
||||
album_info={
|
||||
"album_name": album_name,
|
||||
"track_number": _tn,
|
||||
"disc_number": _dn,
|
||||
# Preserve missing numbers as None so the import pipeline
|
||||
# can fall through to ``extract_track_number_from_filename``
|
||||
# at ``core/imports/pipeline.py:652`` instead of locking
|
||||
# to track/disc 01 for every wishlist re-attempt.
|
||||
"track_number": track_info.get("track_number"),
|
||||
"disc_number": track_info.get("disc_number"),
|
||||
"album_image_url": album_ctx.get("image_url", ""),
|
||||
"confidence": 0.5,
|
||||
},
|
||||
force_album=True,
|
||||
)
|
||||
|
||||
# Last resort: the track matched a SINGLE with no usable album context —
|
||||
# look up the parent ALBUM that actually contains it (gated, fail-safe).
|
||||
return _resolve_single_to_parent_album(context, artist_context)
|
||||
|
||||
|
||||
def _resolve_album_position_from_source(context, artist_context, current_disc):
|
||||
"""Look up a track's real ``(track_number, disc_number)`` from its album's track
|
||||
list, for the case where the album is known but discovery didn't carry a
|
||||
position (Deezer/MusicBrainz search omit it).
|
||||
|
||||
Uses the SAME album id discovery already resolved (``get_import_source_ids`` →
|
||||
``album_id``), so it re-homes the track onto its own album with no re-search and
|
||||
no edition guessing. Matches by ISRC → source track id → title via the pure
|
||||
``core.imports.album_position`` seam. Returns ``(None, current_disc)`` on any
|
||||
miss/error so the caller falls back exactly as before — never worse than today.
|
||||
"""
|
||||
try:
|
||||
source = get_import_source(context)
|
||||
ids = get_import_source_ids(context)
|
||||
album_id = str(ids.get("album_id") or "")
|
||||
if not source or not album_id:
|
||||
return None, current_disc
|
||||
|
||||
from core.metadata.album_tracks import get_album_tracks_for_source
|
||||
payload = get_album_tracks_for_source(source, album_id) or {}
|
||||
tracks = payload.get("tracks") or []
|
||||
if not tracks:
|
||||
return None, current_disc
|
||||
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
title = (track_info.get("name") or original_search.get("title") or "").strip()
|
||||
isrc = str(track_info.get("isrc") or original_search.get("isrc") or "")
|
||||
|
||||
from core.imports.album_position import resolve_track_position_in_album
|
||||
tn, dn = resolve_track_position_in_album(
|
||||
tracks, title=title, track_id=str(ids.get("track_id") or ""), isrc=isrc)
|
||||
if tn is not None:
|
||||
logger.info("album-position: resolved '%s' to track %s/disc %s from album %s (%s)",
|
||||
title, tn, dn, album_id, source)
|
||||
return tn, (dn if dn is not None else current_disc)
|
||||
return None, current_disc
|
||||
except Exception as e:
|
||||
logger.debug("album-position resolution failed: %s", e)
|
||||
return None, current_disc
|
||||
|
||||
|
||||
def _resolve_single_to_parent_album(context, artist_context):
|
||||
"""A single-matched track -> a promoted album_info for its parent album, or
|
||||
None. GATED by ``metadata_enhancement.single_to_album`` (default OFF — it's a
|
||||
per-import metadata lookup, so it's opt-in). Fail-safe: any miss/error returns
|
||||
None so the track stays exactly as it was matched (never worse than today)."""
|
||||
try:
|
||||
from core.metadata.common import get_config_manager
|
||||
if not get_config_manager().get("metadata_enhancement.single_to_album", False):
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
try:
|
||||
source = get_import_source(context)
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
track_title = (track_info.get("name") or original_search.get("title") or "").strip()
|
||||
artist_name = (extract_artist_name(artist_context)
|
||||
or get_import_clean_artist(context, default="")).strip()
|
||||
if not source or not track_title or not artist_name:
|
||||
return None
|
||||
artist_id = str(get_import_source_ids(context).get("artist_id") or "")
|
||||
|
||||
from core.metadata.album_tracks import (
|
||||
get_artist_albums_for_source,
|
||||
get_artist_album_tracks,
|
||||
)
|
||||
from core.imports.single_to_album import resolve_single_to_album
|
||||
|
||||
def _acc(o, *ks):
|
||||
for k in ks:
|
||||
v = o.get(k) if isinstance(o, dict) else getattr(o, k, None)
|
||||
if v:
|
||||
return v
|
||||
return None
|
||||
|
||||
def fetch_candidates():
|
||||
albums = get_artist_albums_for_source(
|
||||
source, artist_id, artist_name=artist_name,
|
||||
album_type="album", limit=20) or []
|
||||
return [{"name": _acc(a, "name", "title"),
|
||||
"album_type": _acc(a, "album_type") or "album",
|
||||
"id": _acc(a, "id", "album_id")} for a in albums]
|
||||
|
||||
def fetch_tracks(alb):
|
||||
payload = get_artist_album_tracks(
|
||||
str(alb.get("id") or ""), artist_name=artist_name,
|
||||
album_name=alb.get("name") or "") or {}
|
||||
return [(_acc(t, "title", "name", "track_name") or "")
|
||||
for t in (payload.get("tracks") or [])]
|
||||
|
||||
album = resolve_single_to_album(
|
||||
track_title,
|
||||
fetch_album_candidates=fetch_candidates,
|
||||
fetch_album_tracks=fetch_tracks)
|
||||
if not album or not album.get("name"):
|
||||
return None
|
||||
logger.info("single->album: re-homed '%s' onto parent album '%s'",
|
||||
track_title, album["name"])
|
||||
promoted = build_import_album_info(
|
||||
context,
|
||||
album_info={
|
||||
"album_name": album["name"],
|
||||
"track_number": track_info.get("track_number"),
|
||||
"disc_number": track_info.get("disc_number"),
|
||||
"album_image_url": "",
|
||||
"confidence": 0.5,
|
||||
},
|
||||
force_album=True,
|
||||
)
|
||||
# build_import_album_info resolves album_name via get_import_clean_album,
|
||||
# which prefers original_search.album (the SINGLE's name); override it
|
||||
# with the resolved parent album so grouping + tags use the album.
|
||||
promoted["album_name"] = album["name"]
|
||||
return promoted
|
||||
except Exception as e:
|
||||
logger.debug("single->album resolution failed: %s", e)
|
||||
return None
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -52,14 +52,6 @@ _DEFAULT_LENGTH_TOLERANCE_S = 3.0
|
|||
_LENGTH_TOLERANCE_LONG_TRACK_S = 5.0
|
||||
_LONG_TRACK_THRESHOLD_S = 600.0 # 10 minutes
|
||||
|
||||
# A file that runs LONGER than the expected metadata is the opposite of a truncated
|
||||
# download — it's almost always a different master/version (a remaster with a longer
|
||||
# outro, an extended fade, an album cut vs the radio edit). The duration check exists to
|
||||
# catch TRUNCATION (short files) and wildly-wrong matches, so on the auto default we allow
|
||||
# more drift in the longer direction and keep the tight bound for short files. A wrong-song
|
||||
# match still trips this — it's usually off by far more than 15s. (#937)
|
||||
_LONGER_VERSION_TOLERANCE_S = 15.0
|
||||
|
||||
# Upper bound for the user-configurable override. Anything past 60s
|
||||
# means the check is effectively off — cap defends against accidental
|
||||
# nonsense like 9999 making logs misleading. Users who genuinely want
|
||||
|
|
@ -250,32 +242,18 @@ def check_audio_integrity(
|
|||
if expected_length_s > _LONG_TRACK_THRESHOLD_S
|
||||
else _DEFAULT_LENGTH_TOLERANCE_S
|
||||
)
|
||||
user_pinned_tolerance = False
|
||||
else:
|
||||
user_pinned_tolerance = True
|
||||
checks["length_tolerance_s"] = length_tolerance_s
|
||||
|
||||
# Positive drift = the file runs LONGER than expected (not truncation). On the auto
|
||||
# default, give the longer direction more room so legit longer masters/versions aren't
|
||||
# quarantined (#937); a user-pinned tolerance is honoured symmetrically.
|
||||
signed_drift_s = actual_length_s - expected_length_s
|
||||
drift_s = abs(signed_drift_s)
|
||||
drift_s = abs(actual_length_s - expected_length_s)
|
||||
checks["length_drift_s"] = drift_s
|
||||
effective_tolerance_s = length_tolerance_s
|
||||
if signed_drift_s > 0 and not user_pinned_tolerance:
|
||||
effective_tolerance_s = max(length_tolerance_s, _LONGER_VERSION_TOLERANCE_S)
|
||||
checks["effective_tolerance_s"] = effective_tolerance_s
|
||||
|
||||
if drift_s > effective_tolerance_s:
|
||||
runs_long = signed_drift_s > 0
|
||||
if drift_s > length_tolerance_s:
|
||||
return IntegrityResult(
|
||||
ok=False,
|
||||
reason=f"Duration mismatch: file is {actual_length_s:.1f}s, "
|
||||
f"expected {expected_length_s:.1f}s "
|
||||
f"(drift {drift_s:.1f}s > tolerance {effective_tolerance_s:.1f}s) — "
|
||||
+ ("runs longer than expected — likely a different version/master or wrong file"
|
||||
if runs_long
|
||||
else "likely truncated download or wrong file matched"),
|
||||
f"(drift {drift_s:.1f}s > tolerance {length_tolerance_s:.1f}s) — "
|
||||
"likely truncated download or wrong file matched",
|
||||
checks=checks,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
|
@ -103,41 +102,6 @@ def cleanup_slskd_dedup_siblings(source_path) -> List[str]:
|
|||
return deleted
|
||||
|
||||
|
||||
def _atomic_cross_device_move(src: Path, dst: Path) -> None:
|
||||
"""Move ``src`` to ``dst`` across filesystems WITHOUT ever exposing a partial file at
|
||||
the final path.
|
||||
|
||||
Copies into a hidden temp sibling of ``dst`` (same filesystem), fsyncs, then does an
|
||||
atomic ``os.replace`` into place, then deletes ``src``. A media-server file watcher
|
||||
(Jellyfin/Plex real-time monitoring) therefore only ever indexes the COMPLETE file —
|
||||
an incremental in-place copy was what Jellyfin could catch mid-write and cache with
|
||||
null/incomplete metadata (tracks landing with no disc). Cleans up the temp on failure.
|
||||
"""
|
||||
src, dst = Path(src), Path(dst)
|
||||
tmp = dst.parent / f".{dst.name}.ssync-tmp"
|
||||
try:
|
||||
with open(src, "rb") as f_src, open(tmp, "wb") as f_dst:
|
||||
shutil.copyfileobj(f_src, f_dst)
|
||||
f_dst.flush()
|
||||
os.fsync(f_dst.fileno())
|
||||
try:
|
||||
shutil.copystat(str(src), str(tmp)) # preserve mtime/permissions (copy2-like)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(str(tmp), str(dst)) # atomic within dst's filesystem
|
||||
except Exception:
|
||||
try:
|
||||
if tmp.exists():
|
||||
tmp.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
try:
|
||||
src.unlink()
|
||||
except OSError:
|
||||
logger.info(f"Could not delete source after cross-device move (may be owned by another process): {src}")
|
||||
|
||||
|
||||
def safe_move_file(src, dst):
|
||||
"""Move a file safely across filesystems."""
|
||||
src = Path(src)
|
||||
|
|
@ -165,13 +129,7 @@ def safe_move_file(src, dst):
|
|||
break
|
||||
|
||||
try:
|
||||
# Same-filesystem move: an atomic rename that also overwrites dst. A media-server
|
||||
# watcher (Jellyfin/Plex real-time monitoring) therefore never sees a partial file
|
||||
# at the final name. Cross-filesystem raises EXDEV (some network mounts raise
|
||||
# EPERM/EACCES) and we copy atomically below instead of letting the move write the
|
||||
# destination incrementally — the partial-file-at-final-name is what caused tracks
|
||||
# to land in Jellyfin with null/incomplete metadata (no disc).
|
||||
os.replace(str(src), str(dst))
|
||||
shutil.move(str(src), str(dst))
|
||||
return
|
||||
except FileNotFoundError:
|
||||
if dst.exists():
|
||||
|
|
@ -179,6 +137,8 @@ def safe_move_file(src, dst):
|
|||
return
|
||||
raise
|
||||
except (OSError, PermissionError) as e:
|
||||
error_msg = str(e).lower()
|
||||
|
||||
if dst.exists() and dst.stat().st_size > 0:
|
||||
logger.warning(f"Move raised {type(e).__name__} but destination exists, treating as success: {e}")
|
||||
try:
|
||||
|
|
@ -187,21 +147,23 @@ def safe_move_file(src, dst):
|
|||
logger.info(f"Could not delete source file (may be owned by another process): {src}")
|
||||
return
|
||||
|
||||
error_msg = str(e).lower()
|
||||
cross_device = (
|
||||
getattr(e, "errno", None) in (errno.EXDEV, errno.EPERM, errno.EACCES)
|
||||
or "cross-device" in error_msg
|
||||
or "operation not permitted" in error_msg
|
||||
or "permission denied" in error_msg
|
||||
)
|
||||
if cross_device:
|
||||
logger.warning(f"Cross-device move, using atomic copy+rename: {e}")
|
||||
if "cross-device" in error_msg or "operation not permitted" in error_msg or "permission denied" in error_msg:
|
||||
logger.warning(f"Cross-device move detected, using fallback copy method: {e}")
|
||||
try:
|
||||
_atomic_cross_device_move(src, dst)
|
||||
logger.info(f"Successfully moved file atomically across filesystems: {src} -> {dst}")
|
||||
with open(src, "rb") as f_src:
|
||||
with open(dst, "wb") as f_dst:
|
||||
shutil.copyfileobj(f_src, f_dst)
|
||||
f_dst.flush()
|
||||
os.fsync(f_dst.fileno())
|
||||
|
||||
try:
|
||||
src.unlink()
|
||||
except PermissionError:
|
||||
logger.info(f"Could not delete source file (may be owned by another process): {src}")
|
||||
logger.info(f"Successfully moved file using fallback method: {src} -> {dst}")
|
||||
return
|
||||
except Exception as fallback_error:
|
||||
logger.error(f"Atomic cross-device move failed: {fallback_error}")
|
||||
logger.error(f"Fallback copy also failed: {fallback_error}")
|
||||
raise
|
||||
raise
|
||||
|
||||
|
|
@ -230,9 +192,6 @@ def get_audio_quality_string(file_path):
|
|||
if ext == ".flac":
|
||||
from mutagen.flac import FLAC
|
||||
audio = FLAC(file_path)
|
||||
sr = getattr(audio.info, "sample_rate", 0) or 0
|
||||
if sr:
|
||||
return f"FLAC {audio.info.bits_per_sample}bit/{sr / 1000:g}kHz"
|
||||
return f"FLAC {audio.info.bits_per_sample}bit"
|
||||
|
||||
if ext == ".mp3":
|
||||
|
|
@ -266,122 +225,6 @@ def get_audio_quality_string(file_path):
|
|||
return ""
|
||||
|
||||
|
||||
def probe_audio_quality(file_path: str):
|
||||
"""Read the actual file and return an AudioQuality with real measured values.
|
||||
|
||||
Uses mutagen to extract sample_rate, bit_depth, and bitrate from the
|
||||
downloaded file — these are ground-truth values, not estimates.
|
||||
Returns None when the file cannot be read.
|
||||
"""
|
||||
from core.quality.model import AudioQuality
|
||||
try:
|
||||
ext = os.path.splitext(file_path)[1].lower().lstrip('.')
|
||||
|
||||
if ext == 'flac':
|
||||
from mutagen.flac import FLAC
|
||||
audio = FLAC(file_path)
|
||||
return AudioQuality(
|
||||
format='flac',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
bit_depth=audio.info.bits_per_sample,
|
||||
)
|
||||
|
||||
if ext == 'mp3':
|
||||
from mutagen.mp3 import MP3
|
||||
audio = MP3(file_path)
|
||||
return AudioQuality(
|
||||
format='mp3',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext in ('m4a', 'aac', 'mp4'):
|
||||
from mutagen.mp4 import MP4
|
||||
audio = MP4(file_path)
|
||||
# .m4a can carry AAC (lossy) OR ALAC (lossless) — only the real
|
||||
# codec tells them apart, which is why extension-based classification
|
||||
# defaults to 'aac' and we correct it here from the probed file.
|
||||
codec = (getattr(audio.info, 'codec', '') or '').lower()
|
||||
if 'alac' in codec:
|
||||
return AudioQuality(
|
||||
format='alac',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
bit_depth=getattr(audio.info, 'bits_per_sample', None) or None,
|
||||
)
|
||||
return AudioQuality(
|
||||
format='aac',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext == 'ogg':
|
||||
from mutagen.oggvorbis import OggVorbis
|
||||
audio = OggVorbis(file_path)
|
||||
return AudioQuality(
|
||||
format='ogg',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext == 'opus':
|
||||
from mutagen.oggopus import OggOpus
|
||||
audio = OggOpus(file_path)
|
||||
return AudioQuality(
|
||||
format='opus',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext in ('wav', 'aiff', 'aif'):
|
||||
# AIFF must use mutagen.aiff.AIFF — WAVE() can't parse it and would
|
||||
# raise, making the file fail open and silently bypass the quality
|
||||
# filter. Both are uncompressed PCM, so they share the 'wav' tier.
|
||||
if ext == 'wav':
|
||||
from mutagen.wave import WAVE
|
||||
audio = WAVE(file_path)
|
||||
else:
|
||||
from mutagen.aiff import AIFF
|
||||
audio = AIFF(file_path)
|
||||
return AudioQuality(
|
||||
format='wav',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
bit_depth=getattr(audio.info, 'bits_per_sample', None),
|
||||
)
|
||||
|
||||
if ext == 'wma':
|
||||
from mutagen.asf import ASF
|
||||
audio = ASF(file_path)
|
||||
return AudioQuality(
|
||||
format='wma',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=getattr(audio.info, 'sample_rate', None),
|
||||
)
|
||||
|
||||
if ext in ('dsf', 'dff'):
|
||||
# DSD (DSD Stream File / DSDIFF) — 1-bit hi-res lossless (#939). mutagen
|
||||
# reads .dsf (rate/bitrate/bit_depth); .dff has no mutagen reader, so it
|
||||
# still classifies as the lossless 'dsf' tier just without measured detail.
|
||||
sr = bd = br = None
|
||||
if ext == 'dsf':
|
||||
try:
|
||||
from mutagen.dsf import DSF
|
||||
info = DSF(file_path).info
|
||||
sr = getattr(info, 'sample_rate', None)
|
||||
bd = getattr(info, 'bits_per_sample', None)
|
||||
br = info.bitrate // 1000 if getattr(info, 'bitrate', None) else None
|
||||
except Exception: # noqa: S110 — unreadable DSF still classifies lossless, just without measured detail
|
||||
pass
|
||||
return AudioQuality(format='dsf', bitrate=br, sample_rate=sr, bit_depth=bd)
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug("probe_audio_quality failed for %s: %s", file_path, e)
|
||||
return None
|
||||
|
||||
|
||||
def get_quality_tier_from_extension(file_path):
|
||||
"""Classify a file extension into a quality tier."""
|
||||
if not file_path:
|
||||
|
|
@ -524,29 +367,14 @@ def downsample_hires_flac(final_path, context):
|
|||
return None
|
||||
|
||||
|
||||
def m4a_codec(path):
|
||||
"""Codec of an .m4a/.mp4 container ('alac', 'aac', …) or None — lets the
|
||||
lossless check tell ALAC (lossless) from AAC (lossy), since both are .m4a."""
|
||||
try:
|
||||
from mutagen.mp4 import MP4
|
||||
return (getattr(MP4(path).info, 'codec', '') or '').lower() or None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def create_lossy_copy(final_path):
|
||||
"""Convert a lossless file (FLAC / ALAC / WAV / AIFF / DSD) to a lossy copy
|
||||
using the configured codec. Non-lossless inputs are skipped (#941)."""
|
||||
from core.quality.lossless import (
|
||||
is_lossless_audio_path,
|
||||
lossy_output_would_overwrite_source,
|
||||
)
|
||||
"""Convert a FLAC file to a lossy copy using the configured codec."""
|
||||
from mutagen.flac import FLAC
|
||||
|
||||
if not config_manager.get("lossy_copy.enabled", False):
|
||||
return None
|
||||
|
||||
# Was FLAC-only; now any lossless source. .m4a is probed (ALAC vs AAC).
|
||||
if not is_lossless_audio_path(final_path, probe_codec=m4a_codec):
|
||||
if os.path.splitext(final_path)[1].lower() != ".flac":
|
||||
return None
|
||||
|
||||
codec = config_manager.get("lossy_copy.codec", "mp3").lower()
|
||||
|
|
@ -575,16 +403,6 @@ def create_lossy_copy(final_path):
|
|||
out_basename = out_basename.replace(original_quality, quality_label)
|
||||
out_path = os.path.join(os.path.dirname(out_path), out_basename)
|
||||
|
||||
# Safety invariant: never write the lossy copy over its own source (an .m4a
|
||||
# ALAC source + AAC target lands on the same .m4a path). ffmpeg runs with -y,
|
||||
# so this guard MUST precede it — the later delete-original guard is too late.
|
||||
if lossy_output_would_overwrite_source(final_path, out_path):
|
||||
logger.info(
|
||||
f"[Lossy Copy] Skipping — {codec.upper()} output would overwrite the "
|
||||
f"source: {os.path.basename(final_path)}"
|
||||
)
|
||||
return None
|
||||
|
||||
ffmpeg_bin = shutil.which("ffmpeg")
|
||||
if not ffmpeg_bin:
|
||||
local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg")
|
||||
|
|
|
|||
|
|
@ -105,72 +105,30 @@ def move_to_quarantine(file_path: str, context: dict, reason: str, automation_en
|
|||
|
||||
|
||||
def check_flac_bit_depth(file_path: str, context: dict) -> Optional[str]:
|
||||
"""Legacy wrapper — delegates to check_quality_target.
|
||||
|
||||
Kept for callers that still pass trigger='bit_depth'; the new guard
|
||||
covers bit_depth as part of the full quality target check.
|
||||
"""
|
||||
return check_quality_target(file_path, context)
|
||||
|
||||
|
||||
def check_quality_target(file_path: str, context: dict) -> Optional[str]:
|
||||
"""Return a rejection message when the downloaded file does not satisfy
|
||||
the user's quality priority list.
|
||||
|
||||
Probes the actual file with mutagen (ground-truth sample_rate,
|
||||
bit_depth, bitrate) and checks it against the profile's
|
||||
``ranked_targets``. Falls back gracefully when fallback_enabled=True.
|
||||
|
||||
Works for all formats and all download sources — no Soulseek-specific
|
||||
logic here.
|
||||
"""
|
||||
from core.imports.file_ops import probe_audio_quality
|
||||
from core.quality.selection import targets_from_profile, quality_meets_profile
|
||||
|
||||
# Master toggle (Settings → Import). When OFF, the quality check is skipped
|
||||
# entirely and files import regardless of quality — the user opted out of
|
||||
# quality-filtering on import. Default ON preserves existing behaviour. The
|
||||
# library Quality Upgrade scanner still flags below-profile files either way.
|
||||
if _get_config_manager().get("import.quality_filter_enabled", True) is False:
|
||||
logger.debug(
|
||||
"[QualityGuard] import.quality_filter_enabled=False — skipping quality "
|
||||
"filter for %s", os.path.basename(file_path),
|
||||
)
|
||||
"""Return a rejection message if a FLAC file violates the configured bit depth."""
|
||||
if not context.get("_audio_quality", "").startswith("FLAC"):
|
||||
return None
|
||||
|
||||
aq = probe_audio_quality(file_path)
|
||||
if aq is None:
|
||||
logger.debug("[QualityGuard] Could not probe %s — skipping check", os.path.basename(file_path))
|
||||
quality_profile = MusicDatabase().get_quality_profile()
|
||||
flac_config = quality_profile.get("qualities", {}).get("flac", {})
|
||||
flac_pref = flac_config.get("bit_depth", "any")
|
||||
if flac_pref == "any":
|
||||
return None
|
||||
|
||||
profile = MusicDatabase().get_quality_profile()
|
||||
targets, fallback_enabled = targets_from_profile(profile)
|
||||
|
||||
if not targets:
|
||||
actual_bits = context["_audio_quality"].replace("FLAC ", "").replace("bit", "")
|
||||
if actual_bits == flac_pref:
|
||||
return None
|
||||
|
||||
flac_fallback = flac_config.get("bit_depth_fallback", True)
|
||||
downsample_enabled = _get_config_manager().get("lossy_copy.downsample_hires", False)
|
||||
|
||||
matched = quality_meets_profile(aq, targets)
|
||||
|
||||
track_info = context.get("track_info", {})
|
||||
track_name = track_info.get("name", os.path.basename(file_path))
|
||||
actual_label = aq.label()
|
||||
|
||||
if matched:
|
||||
logger.info("[QualityGuard] %s meets profile: %s", track_name, actual_label)
|
||||
if flac_fallback or downsample_enabled:
|
||||
if downsample_enabled:
|
||||
logger.info("[FLAC Downsample] Accepted %s-bit FLAC (will be downsampled to %s-bit): %s", actual_bits, flac_pref, track_name)
|
||||
else:
|
||||
logger.warning("[FLAC Fallback] Accepted %s-bit FLAC (preferred %s-bit): %s", actual_bits, flac_pref, track_name)
|
||||
return None
|
||||
|
||||
# No target matched
|
||||
best_label = targets[0].label if targets else "?"
|
||||
if fallback_enabled or downsample_enabled:
|
||||
logger.warning(
|
||||
"[QualityGuard] %s did not match any target (got %s, wanted %s) — accepting via fallback",
|
||||
track_name, actual_label, best_label,
|
||||
)
|
||||
return None
|
||||
|
||||
return (
|
||||
f"Quality mismatch: file is {actual_label}, "
|
||||
f"does not satisfy any configured target (best wanted: {best_label})"
|
||||
)
|
||||
return f"FLAC bit depth mismatch: file is {actual_bits}-bit, preference is {flac_pref}-bit"
|
||||
|
|
|
|||
|
|
@ -158,33 +158,6 @@ def clean_track_title(track_title: str, artist_name: str) -> str:
|
|||
return cleaned if cleaned else original
|
||||
|
||||
|
||||
# A leading track-number prefix is EITHER a zero-padded number (01, 04, 099 — no
|
||||
# real song title starts with one), OR a plain number followed by a real separator
|
||||
# AND a space ("3 - ", "12. "). Deliberately NOT a bare "number space word", so it
|
||||
# leaves "7 Rings", "99 Luftballons", "50 Ways to Leave Your Lover" and
|
||||
# "1-800-273-8255" untouched.
|
||||
_TRACK_NUM_PREFIX_RE = re.compile(r"^\s*(?:0\d{1,2}[\s._)\-]*|\d{1,3}\s*[._)\-]\s+)(?=\S)")
|
||||
|
||||
|
||||
def strip_leading_track_number(title: str) -> str:
|
||||
"""Conservatively remove a leading track-number prefix from a track title.
|
||||
|
||||
Fixes #890 — files named ``01 - Sun It Rises.flac`` whose stem leaks into the
|
||||
title as ``01 - Sun It Rises``, which then never matches the canonical
|
||||
``Sun It Rises`` (false "missing"). Only strips an unambiguous track-number
|
||||
prefix; a coincidental leading number that's part of the title is preserved, and
|
||||
it never reduces a title to empty or a bare number."""
|
||||
s = (title or "").strip()
|
||||
if not s:
|
||||
return title or ""
|
||||
stripped = _TRACK_NUM_PREFIX_RE.sub("", s, count=1).strip()
|
||||
# Keep the original if stripping left nothing real — empty, a bare number, or
|
||||
# only punctuation (e.g. "01 - " → "-"). A real title has a letter/digit.
|
||||
if stripped.isdigit() or not re.search(r"[^\W_]", stripped):
|
||||
return s
|
||||
return stripped
|
||||
|
||||
|
||||
|
||||
|
||||
def get_album_type_display(raw_type, track_count) -> str:
|
||||
|
|
|
|||
|
|
@ -34,11 +34,9 @@ from core.imports.context import (
|
|||
)
|
||||
from core.imports.file_integrity import check_audio_integrity, expected_duration_for_check, resolve_duration_tolerance
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
from core.imports.guards import check_flac_bit_depth, check_quality_target, move_to_quarantine
|
||||
from core.imports.silence import detect_broken_audio
|
||||
from core.imports.guards import check_flac_bit_depth, move_to_quarantine
|
||||
from core.imports.quarantine import (
|
||||
approve_quarantine_entry,
|
||||
delete_quarantine_entry,
|
||||
entry_id_from_quarantined_filename,
|
||||
list_quarantine_entries,
|
||||
)
|
||||
|
|
@ -162,9 +160,7 @@ def import_rejection_reason(context: dict) -> str | None:
|
|||
f"{context.get('_acoustid_failure_msg', 'fingerprint mismatch')}"
|
||||
)
|
||||
if context.get('_bitdepth_rejected'):
|
||||
return "rejected by quality filter"
|
||||
if context.get('_silence_rejected'):
|
||||
return "rejected by audio guard (incomplete or silent audio)"
|
||||
return "rejected by bit-depth filter"
|
||||
if context.get('_race_guard_failed'):
|
||||
return "source file disappeared before import completed"
|
||||
return None
|
||||
|
|
@ -325,14 +321,11 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.error(f"File quarantined due to integrity failure: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
# Quarantine MOVE failed (e.g. cross-device / permission on a NAS).
|
||||
# Do NOT delete — destroying a download we couldn't even quarantine is
|
||||
# data loss and forces a re-download. Leave it in place so it can be
|
||||
# retried; the task is still marked failed below either way (#kettui).
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file in place "
|
||||
f"for retry (not deleting): {file_path}"
|
||||
)
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting broken file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as del_error:
|
||||
logger.error(f"Could not delete broken file either: {del_error}")
|
||||
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
|
|
@ -360,118 +353,6 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
f"drift={integrity.checks.get('length_drift_s', 'n/a')})"
|
||||
)
|
||||
|
||||
# Audio-completeness guard — runs right where the length is verified,
|
||||
# BEFORE the AcoustID/quality gates, so a truncated file (container
|
||||
# claims the full length but only ~30s actually decodes, e.g. HiFi/
|
||||
# Monochrome HLS assembly) or a mostly-silent file is caught regardless
|
||||
# of its quality verdict and gets the right reason. Same quarantine +
|
||||
# next-candidate retry pattern as the integrity check.
|
||||
#
|
||||
# Opt-in (default OFF): this is the one check that fully DECODES the file
|
||||
# with ffmpeg, so it is the most CPU-expensive step in post-processing.
|
||||
# Most preview/truncation cases are already caught at the source
|
||||
# (HiFi/Qobuz have their own guards), so it stays off unless the user
|
||||
# turns it on under Settings → Post-processing.
|
||||
_audio_guard_enabled = config_manager.get('post_processing.audio_completeness_check', False)
|
||||
_skip_audio = (not _audio_guard_enabled) or _should_skip_quarantine_check(context, 'silence')
|
||||
audio_reason = None if _skip_audio else detect_broken_audio(file_path)
|
||||
if audio_reason:
|
||||
logger.error(f"[AudioGuard] Rejected {_basename}: {audio_reason}")
|
||||
context['_silence_rejected'] = True
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path, context, audio_reason, automation_engine,
|
||||
trigger='silence',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.warning("File quarantined — incomplete/silent audio: %s", quarantine_path)
|
||||
except Exception as quarantine_error:
|
||||
# Don't delete a file we couldn't quarantine — leave it for retry
|
||||
# instead of forcing a re-download (data loss). See integrity block.
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file in place "
|
||||
f"for retry (not deleting): {file_path}"
|
||||
)
|
||||
|
||||
with matched_context_lock:
|
||||
matched_downloads_context.pop(context_key, None)
|
||||
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
# Try the next-best candidate before giving up — same pattern as
|
||||
# the integrity / AcoustID / quality failures.
|
||||
if _requeue_quarantined_task_for_retry(task_id, batch_id, 'silence'):
|
||||
logger.info(
|
||||
"Incomplete/silent audio for task %s — retrying next-best candidate: %s",
|
||||
task_id, audio_reason,
|
||||
)
|
||||
return
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Audio guard: {audio_reason}"
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
# QUALITY GATE — runs BEFORE AcoustID so (1) a wrong-quality file is
|
||||
# rejected without paying for an AcoustID fingerprint, and (2) the real
|
||||
# audio quality is recorded on the context (→ quarantine sidecar) for
|
||||
# EVERY trigger, so it's known when reviewing/approving any quarantined
|
||||
# file. force_import still never fires on a quality mismatch.
|
||||
context['_audio_quality'] = get_audio_quality_string(file_path)
|
||||
if context['_audio_quality']:
|
||||
logger.info(f"Audio quality detected: {context['_audio_quality']}")
|
||||
|
||||
_skip_quality = _should_skip_quarantine_check(context, 'bit_depth') or \
|
||||
_should_skip_quarantine_check(context, 'quality')
|
||||
rejection_reason = None if _skip_quality else check_quality_target(file_path, context)
|
||||
if _skip_quality:
|
||||
logger.info(f"[QualityGuard] Skipped (user approval) for {_basename}")
|
||||
if rejection_reason:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
rejection_reason,
|
||||
automation_engine,
|
||||
trigger='quality',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.info(f"File quarantined due to quality mismatch: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
# Don't delete a file we couldn't quarantine — leave it for retry
|
||||
# instead of forcing a re-download (data loss). See integrity block.
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file in place "
|
||||
f"for retry (not deleting): {file_path}"
|
||||
)
|
||||
|
||||
context['_bitdepth_rejected'] = True
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
with matched_context_lock:
|
||||
matched_downloads_context.pop(context_key, None)
|
||||
|
||||
# Try the next-best candidate before giving up — same pattern
|
||||
# as AcoustID and integrity failures.
|
||||
if _requeue_quarantined_task_for_retry(task_id, batch_id, 'quality'):
|
||||
logger.info(
|
||||
"Quality mismatch for task %s — retrying next-best candidate: %s",
|
||||
task_id, rejection_reason,
|
||||
)
|
||||
return
|
||||
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Quality filter: {rejection_reason}"
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
_skip_acoustid = _should_skip_quarantine_check(context, 'acoustid')
|
||||
if _skip_acoustid:
|
||||
logger.info(f"[AcoustID] Skipped (user approval) for {_basename}")
|
||||
|
|
@ -509,44 +390,24 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}")
|
||||
context['_acoustid_result'] = verification_result.value
|
||||
|
||||
# Fail-closed mode: when the user requires a hard AcoustID
|
||||
# PASS, a SKIP (ran but couldn't confirm — no fingerprint
|
||||
# match / cross-script metadata) is treated like a FAIL:
|
||||
# quarantine + try the next candidate, instead of importing
|
||||
# an unverified file. ERROR (rate-limit / infra) is NOT
|
||||
# blocked — that would stall the whole pipeline during an
|
||||
# outage; those still import with their existing flag.
|
||||
require_verified = config_manager.get('acoustid.require_verified', False)
|
||||
_skip_as_fail = (
|
||||
require_verified
|
||||
and verification_result == VerificationResult.SKIP
|
||||
)
|
||||
if _skip_as_fail:
|
||||
verification_msg = (
|
||||
f"AcoustID could not confirm the track and 'require verified' "
|
||||
f"is on — rejecting unverified file ({verification_msg})"
|
||||
)
|
||||
logger.warning("[AcoustID] Require-verified: SKIP treated as FAIL — %s", verification_msg)
|
||||
|
||||
if verification_result == VerificationResult.FAIL or _skip_as_fail:
|
||||
if verification_result == VerificationResult.FAIL:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
verification_msg,
|
||||
automation_engine,
|
||||
trigger='acoustid_unverified' if _skip_as_fail else 'acoustid',
|
||||
trigger='acoustid',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.error(f"File quarantined due to verification failure: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
# Don't delete a file we couldn't quarantine — leave it for
|
||||
# retry instead of forcing a re-download (data loss). The
|
||||
# task is still marked failed / requeued below. See integrity.
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file "
|
||||
f"in place for retry (not deleting): {file_path}"
|
||||
)
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}")
|
||||
logger.error(f"Quarantine failed, deleting wrong file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as del_error:
|
||||
logger.error(f"Could not delete wrong file either: {del_error}")
|
||||
|
||||
context['_acoustid_quarantined'] = True
|
||||
context['_acoustid_failure_msg'] = verification_msg
|
||||
|
|
@ -716,6 +577,48 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
album_info.get('album_name', 'None'),
|
||||
)
|
||||
|
||||
context['_audio_quality'] = get_audio_quality_string(file_path)
|
||||
if context['_audio_quality']:
|
||||
logger.info(f"Audio quality detected: {context['_audio_quality']}")
|
||||
|
||||
_skip_bit_depth = _should_skip_quarantine_check(context, 'bit_depth')
|
||||
rejection_reason = None if _skip_bit_depth else check_flac_bit_depth(file_path, context)
|
||||
if _skip_bit_depth:
|
||||
logger.info(f"[BitDepth] Skipped (user approval) for {_basename}")
|
||||
if rejection_reason:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
rejection_reason,
|
||||
automation_engine,
|
||||
trigger='bit_depth',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.info(f"File quarantined due to bit depth filter: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as e:
|
||||
logger.debug("delete quarantine fallback: %s", e)
|
||||
|
||||
context['_bitdepth_rejected'] = True
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}"
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
file_ext = os.path.splitext(file_path)[1]
|
||||
clean_track_name = get_import_clean_title(
|
||||
context,
|
||||
|
|
@ -726,17 +629,9 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
# See ``core/imports/track_number.py`` for the resolution
|
||||
# chain — pure function, unit-tested in isolation, single
|
||||
# place to fix the rule.
|
||||
from core.imports.track_number import resolve_track_number, read_embedded_track_number
|
||||
from core.imports.track_number import resolve_track_number
|
||||
track_info_for_resolve = context.get('track_info') if isinstance(context, dict) else None
|
||||
# "Track 01" bug: a single Deezer track is matched via an endpoint
|
||||
# that omits track_position, so the context never carried the real
|
||||
# number. The downloaded file itself does (deemix/source wrote it),
|
||||
# so read it as a source between metadata and the filename guess.
|
||||
embedded_track_number = read_embedded_track_number(file_path)
|
||||
track_number = resolve_track_number(
|
||||
album_info, track_info_for_resolve, file_path,
|
||||
embedded_track_number=embedded_track_number,
|
||||
)
|
||||
track_number = resolve_track_number(album_info, track_info_for_resolve, file_path)
|
||||
logger.debug(
|
||||
"Final track_number processing: source=%s album_info=%s resolved=%s",
|
||||
album_info.get('source', 'unknown'),
|
||||
|
|
@ -752,16 +647,6 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
album_info['clean_track_name'] = clean_track_name
|
||||
logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata")
|
||||
|
||||
# Sync the disc number the SAME way (and via the SAME resolver) the embedded
|
||||
# tag will use — otherwise the "Disc N" folder is built from album_info's
|
||||
# original disc while the tag takes the per-track disc, so a disc-2/3 track
|
||||
# lands in the Disc 1 folder and every disc's tracks collapse together (Sokhi).
|
||||
from core.imports.track_number import resolve_disc_for_track
|
||||
_resolved_disc = resolve_disc_for_track(get_import_original_search(context), album_info)
|
||||
if album_info.get('disc_number') != _resolved_disc:
|
||||
logger.info(f"[FIX] Updated album_info disc_number to {_resolved_disc} for consistent metadata")
|
||||
album_info['disc_number'] = _resolved_disc
|
||||
|
||||
final_path, _ = build_final_path_for_track(context, artist_context, album_info, file_ext)
|
||||
logger.info(f"Resolved path: '{final_path}'")
|
||||
context['_final_processed_path'] = final_path
|
||||
|
|
@ -1084,22 +969,6 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
if original_batch_id:
|
||||
context['batch_id'] = original_batch_id
|
||||
|
||||
# Quality / audio-guard quarantine. Unlike acoustid/integrity (whose
|
||||
# retry+fail is driven by THIS wrapper below), the inner pipeline fully
|
||||
# owns the quality and audio-guard outcome: it quarantines the file and
|
||||
# then either re-queues the next-best candidate or marks the task failed
|
||||
# and notifies. The wrapper must NOT continue to the "assume success"
|
||||
# fall-through — that marked the quarantined file Completed, so the same
|
||||
# track showed in BOTH Completed and Quarantine (e.g. an MP3-VBR rejected
|
||||
# by a FLAC-only profile). It must also NOT mark it failed here, which
|
||||
# would clobber a successful next-candidate retry. Just return.
|
||||
if context.get('_bitdepth_rejected') or context.get('_silence_rejected'):
|
||||
logger.info(
|
||||
f"Task {task_id} quarantined by the quality/audio guard — inner "
|
||||
f"pipeline already handled retry/fail; wrapper not marking completed"
|
||||
)
|
||||
return
|
||||
|
||||
if context.get('_race_guard_failed'):
|
||||
logger.info(f"Race guard: source file gone for task {task_id} — marking as failed")
|
||||
with tasks_lock:
|
||||
|
|
@ -1113,33 +982,6 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
return
|
||||
|
||||
if context.get('_acoustid_quarantined'):
|
||||
# Race-condition guard: if the user approved an alternative quarantine
|
||||
# entry for this track while this download was already in flight, the
|
||||
# pipeline will have created a new quarantine entry for the just-finished
|
||||
# file. Delete that entry and exit — the user's choice already won.
|
||||
_approved_alt = False
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
_ct = download_tasks.get(task_id)
|
||||
if isinstance(_ct, dict) and _ct.get('_quarantine_approved_alternative'):
|
||||
_approved_alt = True
|
||||
if _approved_alt:
|
||||
_eid = context.get('_quarantine_entry_id')
|
||||
if _eid:
|
||||
try:
|
||||
from core.imports.guards import _get_config_manager
|
||||
_dl_dir = _get_config_manager().get('soulseek.download_path', './downloads')
|
||||
_qdir = os.path.join(docker_resolve_path(_dl_dir), 'ss_quarantine')
|
||||
delete_quarantine_entry(_qdir, _eid)
|
||||
logger.info(
|
||||
f"[Quarantine] Discarded late entry {_eid} — task {task_id} "
|
||||
f"was cancelled by prior quarantine approval"
|
||||
)
|
||||
except Exception as _dqe:
|
||||
logger.debug(f"[Quarantine] Could not clean up late entry {_eid}: {_dqe}")
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed')
|
||||
# Before failing outright, try the next-best candidate. The wrong
|
||||
# file was just quarantined; re-running the worker (with the bad
|
||||
|
|
@ -1256,10 +1098,6 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
_mark_task_completed(task_id, context.get('track_info'))
|
||||
if context.get('_verification_status'):
|
||||
download_tasks[task_id]['verification_status'] = context['_verification_status']
|
||||
if context.get('_history_id'):
|
||||
download_tasks[task_id]['history_id'] = context['_history_id']
|
||||
if context.get('_audio_quality'):
|
||||
download_tasks[task_id]['quality'] = context['_audio_quality']
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
|
@ -1274,10 +1112,6 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
download_tasks[task_id]['metadata_enhanced'] = True
|
||||
if context.get('_verification_status'):
|
||||
download_tasks[task_id]['verification_status'] = context['_verification_status']
|
||||
if context.get('_history_id'):
|
||||
download_tasks[task_id]['history_id'] = context['_history_id']
|
||||
if context.get('_audio_quality'):
|
||||
download_tasks[task_id]['quality'] = context['_audio_quality']
|
||||
redownload_ctx = download_tasks[task_id].get('_redownload_context')
|
||||
|
||||
with matched_context_lock:
|
||||
|
|
|
|||
|
|
@ -153,76 +153,6 @@ def get_quarantined_source_keys(quarantine_dir: str) -> set:
|
|||
return keys
|
||||
|
||||
|
||||
def quarantine_group_key(
|
||||
expected_artist: Any, expected_track: Any, context: Any = None
|
||||
) -> Optional[str]:
|
||||
"""Grouping key for "the same intended download target".
|
||||
|
||||
#876: when several sources are downloaded for one wishlist/queue
|
||||
track they each fail verification and land in quarantine as separate
|
||||
entries. They are *alternatives for the same song*, so they should
|
||||
group together — and once the user accepts one, the rest are
|
||||
redundant failed attempts at a song they now own.
|
||||
|
||||
The key identifies the *intended* target — what SoulSync was trying to
|
||||
fetch — NOT the downloaded file's own tags. That matters: the file's
|
||||
metadata is frequently *wrong* (that's why it failed acoustid /
|
||||
integrity), whereas the target is fixed and identical across every
|
||||
alternative for one song.
|
||||
|
||||
Uses ISRC when available (truly universal across sources and batches).
|
||||
Falls back to normalized artist|track name, which is stable across
|
||||
different batches and sources.
|
||||
|
||||
Source-specific IDs (Spotify track id, Qobuz id, uri) are intentionally
|
||||
NOT used: the same song imported from different playlists or sources gets
|
||||
different source IDs, so id-based keys break cross-batch sibling matching.
|
||||
|
||||
Returns ``None`` when nothing identifies the target (no usable id and
|
||||
both name fields empty). Callers treat a ``None`` key as "its own
|
||||
singleton group" — ungroupable entries must never collapse together.
|
||||
"""
|
||||
ti = {}
|
||||
if isinstance(context, dict):
|
||||
maybe_ti = context.get("track_info")
|
||||
if isinstance(maybe_ti, dict):
|
||||
ti = maybe_ti
|
||||
isrc = str(ti.get("isrc") or "").strip().lower()
|
||||
if isrc:
|
||||
return f"isrc:{isrc}"
|
||||
artist = " ".join(str(expected_artist or "").split()).lower()
|
||||
track = " ".join(str(expected_track or "").split()).lower()
|
||||
if not artist and not track:
|
||||
return None
|
||||
return f"nm:{artist}|{track}"
|
||||
|
||||
|
||||
def find_quarantine_siblings(quarantine_dir: str, entry_id: str) -> List[str]:
|
||||
"""Other entry ids that share ``entry_id``'s intended-target group key.
|
||||
|
||||
Returns the ids of every *other* quarantine entry whose
|
||||
`expected_artist`/`expected_track` normalize to the same key as
|
||||
``entry_id`` (see :func:`quarantine_group_key`). Excludes ``entry_id``
|
||||
itself. Returns ``[]`` when the entry is missing, has an ungroupable
|
||||
(``None``) key, or has no siblings. Never raises.
|
||||
"""
|
||||
if not entry_id:
|
||||
return []
|
||||
entries = list_quarantine_entries(quarantine_dir)
|
||||
target_key = None
|
||||
for e in entries:
|
||||
if e.get("id") == entry_id:
|
||||
target_key = e.get("group_key")
|
||||
break
|
||||
if target_key is None:
|
||||
return []
|
||||
return [
|
||||
e["id"]
|
||||
for e in entries
|
||||
if e.get("id") != entry_id and e.get("group_key") == target_key
|
||||
]
|
||||
|
||||
|
||||
def list_quarantine_entries(quarantine_dir: str) -> List[Dict[str, Any]]:
|
||||
"""Enumerate quarantined files paired with their sidecars.
|
||||
|
||||
|
|
@ -283,11 +213,6 @@ def list_quarantine_entries(quarantine_dir: str) -> List[Dict[str, Any]]:
|
|||
"reason": sidecar.get("quarantine_reason", "Unknown reason"),
|
||||
"expected_track": sidecar.get("expected_track", ""),
|
||||
"expected_artist": sidecar.get("expected_artist", ""),
|
||||
"group_key": quarantine_group_key(
|
||||
sidecar.get("expected_artist", ""),
|
||||
sidecar.get("expected_track", ""),
|
||||
ctx,
|
||||
),
|
||||
"timestamp": sidecar.get("timestamp", ""),
|
||||
"size_bytes": size_bytes,
|
||||
"has_full_context": isinstance(sidecar.get("context"), dict),
|
||||
|
|
@ -295,10 +220,6 @@ def list_quarantine_entries(quarantine_dir: str) -> List[Dict[str, Any]]:
|
|||
"source_username": source_username,
|
||||
"source_filename": source_filename,
|
||||
"thumb_url": _extract_context_thumb(ctx),
|
||||
# Real probed audio quality (recorded on the context before the
|
||||
# quality/AcoustID gates) so the review UI shows what the file
|
||||
# actually is when deciding to approve/delete.
|
||||
"quality": ctx.get("_audio_quality", "") if isinstance(ctx, dict) else "",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,92 +0,0 @@
|
|||
"""#889 Phase 4/5: apply a re-identify — stage the library file + write the hint.
|
||||
|
||||
When the user confirms a release in the Re-identify modal, we:
|
||||
1. COPY (never move) the track's library file into the auto-import staging folder,
|
||||
so the original is untouched until the re-import succeeds,
|
||||
2. fingerprint the staged copy (rename-proof binding), and
|
||||
3. write a single-use hint carrying the chosen release's IDs (+ ``replace_track_id``
|
||||
when 'replace original' is ticked).
|
||||
|
||||
The auto-import worker then picks the staged file up, finds the hint, and re-imports
|
||||
it against the user-chosen release (Phase 2). The pieces here are split so the
|
||||
naming + hint construction are pure/unit-tested and the actual copy is injectable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from core.imports.paths import sanitize_filename
|
||||
from core.imports.rematch_hints import RematchHint, quick_file_signature
|
||||
|
||||
|
||||
def staged_destination(staging_dir: str, real_path: str, library_track_id: Any) -> str:
|
||||
"""Where the staged copy lands: a single loose file in the staging ROOT (so the
|
||||
worker treats it as a single-track candidate), named to keep the extension and
|
||||
be unique + traceable to the track it re-identifies. The filename is cosmetic —
|
||||
matching is driven by the hint, not the name."""
|
||||
base = os.path.basename(real_path)
|
||||
stem, ext = os.path.splitext(base)
|
||||
safe_stem = sanitize_filename(stem).strip() or "track"
|
||||
name = f"{safe_stem} [reid-{library_track_id}]{ext}"
|
||||
return os.path.join(staging_dir, name)
|
||||
|
||||
|
||||
def stage_file_for_reidentify(
|
||||
real_path: str,
|
||||
staging_dir: str,
|
||||
library_track_id: Any,
|
||||
*,
|
||||
copy_fn: Callable[[str, str], object] = shutil.copy2,
|
||||
signature_fn: Callable[[str], Optional[str]] = quick_file_signature,
|
||||
) -> Dict[str, Any]:
|
||||
"""Copy the library file into staging and fingerprint the copy. Returns
|
||||
``{staged_path, content_hash}``. Raises ``FileNotFoundError`` if the source is
|
||||
gone (caller surfaces a clear error rather than writing a dangling hint)."""
|
||||
if not real_path or not os.path.isfile(real_path):
|
||||
raise FileNotFoundError(real_path or "(empty path)")
|
||||
os.makedirs(staging_dir, exist_ok=True)
|
||||
dest = staged_destination(staging_dir, real_path, library_track_id)
|
||||
copy_fn(real_path, dest)
|
||||
return {"staged_path": dest, "content_hash": signature_fn(dest)}
|
||||
|
||||
|
||||
def build_reidentify_hint(
|
||||
library_track_id: Any,
|
||||
hint_fields: Dict[str, Any],
|
||||
staged_path: str,
|
||||
content_hash: Optional[str],
|
||||
*,
|
||||
replace: bool,
|
||||
) -> RematchHint:
|
||||
"""Pure: assemble the RematchHint from the resolved release fields + staging
|
||||
info. ``replace_track_id`` is the library row to delete on success, but only
|
||||
when 'replace original' was ticked. ``exempt_dedup`` is always True — a
|
||||
re-identify is explicit and must bypass dedup-skip."""
|
||||
return RematchHint(
|
||||
staged_path=staged_path,
|
||||
content_hash=content_hash,
|
||||
source=hint_fields.get("source") or "",
|
||||
isrc=hint_fields.get("isrc"),
|
||||
track_id=hint_fields.get("track_id"),
|
||||
album_id=hint_fields.get("album_id"),
|
||||
artist_id=hint_fields.get("artist_id"),
|
||||
track_title=hint_fields.get("track_title"),
|
||||
album_name=hint_fields.get("album_name"),
|
||||
artist_name=hint_fields.get("artist_name"),
|
||||
album_type=hint_fields.get("album_type"),
|
||||
track_number=hint_fields.get("track_number"),
|
||||
disc_number=hint_fields.get("disc_number"),
|
||||
replace_track_id=(int(library_track_id) if replace and str(library_track_id).isdigit() else
|
||||
(library_track_id if replace else None)),
|
||||
exempt_dedup=True,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"staged_destination",
|
||||
"stage_file_for_reidentify",
|
||||
"build_reidentify_hint",
|
||||
]
|
||||
|
|
@ -1,334 +0,0 @@
|
|||
"""Re-identify hints (#889) — a single-use, user-designated answer to "which
|
||||
release does this already-imported track belong to".
|
||||
|
||||
Flow: the user clicks *Re-identify* on a library track, searches a source, and
|
||||
picks the exact release (single / EP / album) it should live under. We write a
|
||||
**hint** here and stage the file for auto-import. The import flow then reads the
|
||||
hint at the very TOP of matching — before any fuzzy tier — builds the match from
|
||||
these exact IDs, and consumes the row. So the original ambiguity that mis-filed
|
||||
the track (which release?) is gone: the user already answered it.
|
||||
|
||||
Two safety properties live in the hint, not the import code:
|
||||
|
||||
- ``replace_track_id`` — the library row to delete AFTER the re-import lands (so a
|
||||
re-identify *replaces* rather than *duplicates*). Cleanup is deferred to success
|
||||
so a failed import can never lose the file.
|
||||
- ``exempt_dedup`` — always set: a re-identify is an explicit user action and must
|
||||
not be silently dropped by the quality dedup-skip (which would otherwise see the
|
||||
incoming file as a duplicate of the very row we're replacing).
|
||||
|
||||
This module is pure DB mechanics over an injected ``cursor`` (sqlite3-style,
|
||||
``?`` params) — no connection management, no app state — so the create / find /
|
||||
consume seam is unit-tested against an in-memory DB with no live metadata client.
|
||||
The binding is keyed on the staged path, with ``content_hash`` as a rename-proof
|
||||
fallback in case the staging watcher normalizes the filename on ingest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
# Columns in INSERT/SELECT order — single source of truth so the dataclass, the
|
||||
# write, and the read can't drift apart.
|
||||
_FIELDS = (
|
||||
"staged_path",
|
||||
"content_hash",
|
||||
"source",
|
||||
"isrc",
|
||||
"track_id",
|
||||
"album_id",
|
||||
"artist_id",
|
||||
"track_title",
|
||||
"album_name",
|
||||
"artist_name",
|
||||
"album_type",
|
||||
"track_number",
|
||||
"disc_number",
|
||||
"replace_track_id",
|
||||
"exempt_dedup",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RematchHint:
|
||||
"""One user-designated re-identify answer. ``id``/``status`` are set by the DB."""
|
||||
staged_path: str
|
||||
source: str
|
||||
content_hash: Optional[str] = None
|
||||
isrc: Optional[str] = None
|
||||
track_id: Optional[str] = None
|
||||
album_id: Optional[str] = None
|
||||
artist_id: Optional[str] = None
|
||||
track_title: Optional[str] = None
|
||||
album_name: Optional[str] = None
|
||||
artist_name: Optional[str] = None
|
||||
album_type: Optional[str] = None
|
||||
track_number: Optional[int] = None
|
||||
disc_number: Optional[int] = None
|
||||
replace_track_id: Optional[int] = None
|
||||
exempt_dedup: bool = True
|
||||
id: Optional[int] = None
|
||||
status: str = "pending"
|
||||
|
||||
def _values(self) -> tuple:
|
||||
return (
|
||||
self.staged_path,
|
||||
self.content_hash,
|
||||
self.source,
|
||||
self.isrc,
|
||||
self.track_id,
|
||||
self.album_id,
|
||||
self.artist_id,
|
||||
self.track_title,
|
||||
self.album_name,
|
||||
self.artist_name,
|
||||
self.album_type,
|
||||
self.track_number,
|
||||
self.disc_number,
|
||||
self.replace_track_id,
|
||||
1 if self.exempt_dedup else 0,
|
||||
)
|
||||
|
||||
|
||||
def _row_to_hint(row: Any) -> RematchHint:
|
||||
"""Map a sqlite3.Row (or any mapping/sequence-by-name) to a RematchHint."""
|
||||
def g(key, default=None):
|
||||
try:
|
||||
return row[key]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return default
|
||||
return RematchHint(
|
||||
id=g("id"),
|
||||
staged_path=g("staged_path") or "",
|
||||
content_hash=g("content_hash"),
|
||||
source=g("source") or "",
|
||||
isrc=g("isrc"),
|
||||
track_id=g("track_id"),
|
||||
album_id=g("album_id"),
|
||||
artist_id=g("artist_id"),
|
||||
track_title=g("track_title"),
|
||||
album_name=g("album_name"),
|
||||
artist_name=g("artist_name"),
|
||||
album_type=g("album_type"),
|
||||
track_number=g("track_number"),
|
||||
disc_number=g("disc_number"),
|
||||
replace_track_id=g("replace_track_id"),
|
||||
exempt_dedup=bool(g("exempt_dedup", 1)),
|
||||
status=g("status") or "pending",
|
||||
)
|
||||
|
||||
|
||||
def create_hint(cursor: Any, hint: RematchHint) -> int:
|
||||
"""Insert a pending hint; return its new id. Caller owns commit."""
|
||||
placeholders = ", ".join("?" for _ in _FIELDS)
|
||||
cursor.execute(
|
||||
f"INSERT INTO rematch_hints ({', '.join(_FIELDS)}) VALUES ({placeholders})",
|
||||
hint._values(),
|
||||
)
|
||||
new_id = cursor.lastrowid
|
||||
hint.id = new_id
|
||||
return new_id
|
||||
|
||||
|
||||
def find_hint_for_file(
|
||||
cursor: Any,
|
||||
staged_path: str,
|
||||
content_hash: Optional[str] = None,
|
||||
) -> Optional[RematchHint]:
|
||||
"""Return the newest PENDING hint for a staged file, or ``None``.
|
||||
|
||||
Matched by exact ``staged_path`` first; if that misses and a ``content_hash``
|
||||
is given, fall back to it (covers a staging watcher that renamed the file on
|
||||
ingest). Only ``status='pending'`` rows are returned, so a consumed hint is
|
||||
never reused."""
|
||||
if staged_path:
|
||||
cursor.execute(
|
||||
"SELECT * FROM rematch_hints WHERE staged_path = ? AND status = 'pending' "
|
||||
"ORDER BY id DESC LIMIT 1",
|
||||
(staged_path,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row is not None:
|
||||
return _row_to_hint(row)
|
||||
# Try by basename too — the watcher may move the file into a different dir.
|
||||
base = os.path.basename(staged_path)
|
||||
if base and base != staged_path:
|
||||
cursor.execute(
|
||||
"SELECT * FROM rematch_hints WHERE staged_path LIKE ? AND status = 'pending' "
|
||||
"ORDER BY id DESC LIMIT 1",
|
||||
("%/" + base,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row is not None:
|
||||
return _row_to_hint(row)
|
||||
|
||||
if content_hash:
|
||||
cursor.execute(
|
||||
"SELECT * FROM rematch_hints WHERE content_hash = ? AND status = 'pending' "
|
||||
"ORDER BY id DESC LIMIT 1",
|
||||
(content_hash,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row is not None:
|
||||
return _row_to_hint(row)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def consume_hint(cursor: Any, hint_id: int) -> None:
|
||||
"""Mark a hint consumed (single-use). Caller owns commit."""
|
||||
cursor.execute(
|
||||
"UPDATE rematch_hints SET status = 'consumed', consumed_at = CURRENT_TIMESTAMP "
|
||||
"WHERE id = ?",
|
||||
(hint_id,),
|
||||
)
|
||||
|
||||
|
||||
def list_pending_hints(cursor: Any) -> list:
|
||||
"""All pending hints (newest first) — for a 'pending re-identify' view and
|
||||
orphan recovery when a staged file never imports."""
|
||||
cursor.execute("SELECT * FROM rematch_hints WHERE status = 'pending' ORDER BY id DESC")
|
||||
return [_row_to_hint(r) for r in cursor.fetchall()]
|
||||
|
||||
|
||||
def build_identification_from_hint(hint: RematchHint) -> dict:
|
||||
"""Turn a hint into the ``identification`` dict the auto-import matcher expects,
|
||||
so a re-identify SKIPS the guessing tiers entirely and matches straight against
|
||||
the user-chosen release. Mirrors the shape `_identify_folder` returns (album_id
|
||||
/ source / track_number drive the album fetch + file→track match)."""
|
||||
return {
|
||||
"album_id": hint.album_id or None,
|
||||
"album_name": hint.album_name or hint.track_title or "",
|
||||
"artist_name": hint.artist_name or "",
|
||||
"artist_id": hint.artist_id or "",
|
||||
"track_name": hint.track_title or "",
|
||||
"track_id": hint.track_id or "",
|
||||
"image_url": "",
|
||||
"release_date": "",
|
||||
"track_number": hint.track_number or 1,
|
||||
"total_tracks": 1,
|
||||
"source": hint.source,
|
||||
"method": "rematch_hint",
|
||||
"identification_confidence": 1.0,
|
||||
# is_single reflects the CHOSEN release, but force_album_match makes the
|
||||
# matcher FETCH that release (even for a lone staged file) instead of taking
|
||||
# the singles fast-path — so the re-imported track gets the real album
|
||||
# metadata: year, the correct in-album track number, and the album art.
|
||||
"is_single": (str(hint.album_type or "").lower() == "single"),
|
||||
"force_album_match": True,
|
||||
"album_type": hint.album_type,
|
||||
}
|
||||
|
||||
|
||||
def _canonical(path: Optional[str]) -> str:
|
||||
"""Canonical form of a path for same-file comparison (symlinks + case + sep)."""
|
||||
if not path:
|
||||
return ""
|
||||
try:
|
||||
return os.path.normcase(os.path.realpath(path))
|
||||
except OSError:
|
||||
return os.path.normcase(os.path.normpath(path))
|
||||
|
||||
|
||||
def delete_replaced_track(
|
||||
cursor: Any,
|
||||
replace_track_id: Any,
|
||||
*,
|
||||
unlink=os.remove,
|
||||
resolve_fn: Optional[Callable[[str], Optional[str]]] = None,
|
||||
new_paths: Optional[list] = None,
|
||||
) -> Optional[str]:
|
||||
"""Remove the OLD library row a re-identify replaces, and its file.
|
||||
|
||||
Called only AFTER the re-import has landed the track at its new home, so the
|
||||
original is never lost on failure. Safe by construction:
|
||||
|
||||
* **Same-home guard (CRITICAL):** if the re-import landed at the SAME file as the
|
||||
old one (``new_paths`` — the paths the import actually wrote), this is a no-op:
|
||||
we DON'T delete the row or the file, because that file IS the re-imported track.
|
||||
This is what stops "re-identify to the release it's already in" from deleting
|
||||
the file (the import reuses the same row, so deleting it would orphan the file).
|
||||
* the file is unlinked only if it still exists and **no other track row references
|
||||
it** (guards against yanking a file a different row legitimately points to).
|
||||
|
||||
Returns the path it removed, or ``None`` if there was nothing to do. ``unlink`` is
|
||||
injectable for tests. ``resolve_fn`` maps the STORED DB path to the file's actual
|
||||
on-disk location (the stored path may be a Docker/media-server view this process
|
||||
can't read literally — without it we'd delete the row but orphan the file)."""
|
||||
if not replace_track_id:
|
||||
return None
|
||||
cursor.execute("SELECT file_path FROM tracks WHERE id = ?", (replace_track_id,))
|
||||
row = cursor.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
old_path = (row["file_path"] if not isinstance(row, (tuple, list)) else row[0]) or ""
|
||||
if not old_path:
|
||||
cursor.execute("DELETE FROM tracks WHERE id = ?", (replace_track_id,))
|
||||
return None
|
||||
|
||||
# Resolve the old stored path to its real on-disk location up front.
|
||||
real_path = old_path
|
||||
if resolve_fn is not None:
|
||||
try:
|
||||
real_path = resolve_fn(old_path) or old_path
|
||||
except Exception:
|
||||
real_path = old_path
|
||||
|
||||
# Same-home guard: if the re-import wrote to this very file, do NOTHING — the row
|
||||
# is the re-imported track's row and the file is its file. Deleting either would
|
||||
# be data loss (the "picked the same release" bug).
|
||||
if new_paths:
|
||||
landed = {_canonical(p) for p in new_paths if p}
|
||||
if _canonical(real_path) in landed or _canonical(old_path) in landed:
|
||||
return None
|
||||
|
||||
cursor.execute("DELETE FROM tracks WHERE id = ?", (replace_track_id,))
|
||||
# Only unlink if no surviving row still points at this file (rows store the
|
||||
# stored path, so compare against the stored path, not the resolved one).
|
||||
cursor.execute("SELECT 1 FROM tracks WHERE file_path = ? LIMIT 1", (old_path,))
|
||||
if cursor.fetchone() is not None:
|
||||
return None
|
||||
try:
|
||||
if os.path.exists(real_path): # real_path resolved above
|
||||
unlink(real_path)
|
||||
return real_path
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def quick_file_signature(path: str, *, chunk: int = 65536) -> Optional[str]:
|
||||
"""A cheap, rename-proof content fingerprint: size + first/last chunk, hashed.
|
||||
|
||||
Audio files are large, so a full hash is wasteful when we only need to re-bind
|
||||
a hint to *this* file after a possible rename. Size + head + tail is plenty to
|
||||
distinguish staged files in practice. Returns ``None`` if the file can't be
|
||||
read (caller falls back to path-only binding)."""
|
||||
import hashlib
|
||||
|
||||
try:
|
||||
size = os.path.getsize(path)
|
||||
h = hashlib.sha256()
|
||||
h.update(str(size).encode())
|
||||
with open(path, "rb") as f:
|
||||
h.update(f.read(chunk))
|
||||
if size > chunk:
|
||||
f.seek(max(0, size - chunk))
|
||||
h.update(f.read(chunk))
|
||||
return h.hexdigest()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RematchHint",
|
||||
"create_hint",
|
||||
"find_hint_for_file",
|
||||
"consume_hint",
|
||||
"list_pending_hints",
|
||||
"build_identification_from_hint",
|
||||
"delete_replaced_track",
|
||||
"quick_file_signature",
|
||||
]
|
||||
|
|
@ -1,247 +0,0 @@
|
|||
"""#889 Phase 3: search a metadata source for the releases a track appears on.
|
||||
|
||||
The Re-identify modal lets the user search ANY configured source (tabs, defaulting
|
||||
to the active one) and shows the SAME song across its different collections —
|
||||
single / EP / album — so they can pick which release the track should be filed
|
||||
under. Two steps, deliberately split:
|
||||
|
||||
* ``search_release_candidates(source, query)`` — lightweight DISPLAY rows from the
|
||||
normal typed ``search_tracks`` (title, artist, release name, type badge, year,
|
||||
track count, art, ISRC, track_id). No album_id needed to draw the list.
|
||||
* ``resolve_hint_fields(source, track_id)`` — runs ONCE, on the row the user
|
||||
picks: ``get_track_details`` yields the album_id / isrc / track#/disc the hint
|
||||
needs. We don't pay that lookup for every search result, only the chosen one.
|
||||
|
||||
Pure normalization + injected client factory, so the search/normalize/resolve seam
|
||||
is unit-tested with a fake client and no network.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
|
||||
def _get(obj: Any, key: str, default=None):
|
||||
"""Read ``key`` from either an object (attr) or a mapping (item)."""
|
||||
if obj is None:
|
||||
return default
|
||||
if isinstance(obj, dict):
|
||||
return obj.get(key, default)
|
||||
return getattr(obj, key, default)
|
||||
|
||||
|
||||
def _year(release_date: Any) -> Optional[str]:
|
||||
s = str(release_date or "").strip()
|
||||
return s[:4] if len(s) >= 4 and s[:4].isdigit() else None
|
||||
|
||||
|
||||
def infer_release_type(album_type: Any, total_tracks: Any) -> str:
|
||||
"""Normalize a source's release type to one of album / ep / single / compilation.
|
||||
|
||||
Sources disagree: Spotify has no 'EP' — EPs come back as ``album_type='single'``
|
||||
with several tracks; MusicBrainz/Deezer label EPs properly. So when a 'single'
|
||||
carries more than a handful of tracks, call it an EP for the badge. The actual
|
||||
filing is unaffected — that's driven by the real album_id, not this label."""
|
||||
t = str(album_type or "").strip().lower()
|
||||
try:
|
||||
n = int(total_tracks) if total_tracks is not None else 0
|
||||
except (TypeError, ValueError):
|
||||
n = 0
|
||||
if t in ("compilation", "comp"):
|
||||
return "compilation"
|
||||
if t == "ep":
|
||||
return "ep"
|
||||
if t == "album":
|
||||
return "album" # an explicit album stays an album; only 'single' gets promoted to EP
|
||||
if t == "single":
|
||||
# 1–3 tracks → single; 4+ → almost always an EP in practice.
|
||||
return "ep" if n >= 4 else "single"
|
||||
# Unknown type: infer purely from track count.
|
||||
if n >= 7:
|
||||
return "album"
|
||||
if n >= 4:
|
||||
return "ep"
|
||||
if n >= 1:
|
||||
return "single"
|
||||
return t or "album"
|
||||
|
||||
|
||||
def normalize_search_result(result: Any, source: str) -> Optional[Dict[str, Any]]:
|
||||
"""One typed search Track (or raw dict) → a display row, or ``None`` if it has
|
||||
no usable id/title. ``album`` is just a name at search time; album_id is
|
||||
resolved later for the picked row only."""
|
||||
track_id = _get(result, "id") or _get(result, "track_id")
|
||||
title = _get(result, "name") or _get(result, "title")
|
||||
if not track_id or not title:
|
||||
return None
|
||||
|
||||
artists = _get(result, "artists")
|
||||
if isinstance(artists, list):
|
||||
artist_name = ", ".join(str(_get(a, "name", a) if not isinstance(a, str) else a) for a in artists)
|
||||
else:
|
||||
artist_name = str(artists or _get(result, "artist") or "")
|
||||
|
||||
album = _get(result, "album")
|
||||
album_name = album if isinstance(album, str) else (_get(album, "name") or "")
|
||||
raw_type = _get(result, "album_type")
|
||||
total = _get(result, "total_tracks")
|
||||
ext = _get(result, "external_ids") or {}
|
||||
isrc = _get(result, "isrc") or (ext.get("isrc") if isinstance(ext, dict) else None)
|
||||
|
||||
return {
|
||||
"source": source,
|
||||
"track_id": str(track_id),
|
||||
"track_title": str(title),
|
||||
"artist_name": artist_name,
|
||||
"album_name": str(album_name or ""),
|
||||
"album_type": infer_release_type(raw_type, total),
|
||||
"raw_album_type": str(raw_type or ""),
|
||||
"total_tracks": int(total) if isinstance(total, int) else None,
|
||||
"year": _year(_get(result, "release_date")),
|
||||
"image_url": _get(result, "image_url") or "",
|
||||
"isrc": isrc or None,
|
||||
}
|
||||
|
||||
|
||||
def search_release_candidates(
|
||||
source: str,
|
||||
query: str,
|
||||
*,
|
||||
limit: int = 25,
|
||||
client_factory: Optional[Callable[[str], Any]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search ``source`` for tracks matching ``query`` → normalized display rows.
|
||||
|
||||
Returns ``[]`` (never raises) when the source has no client or errors — the UI
|
||||
just shows an empty tab. Rows keep duplicate releases; the UI groups them."""
|
||||
query = (query or "").strip()
|
||||
if not query:
|
||||
return []
|
||||
factory = client_factory or _default_client_factory
|
||||
try:
|
||||
client = factory(source)
|
||||
except Exception:
|
||||
client = None
|
||||
if client is None or not hasattr(client, "search_tracks"):
|
||||
return []
|
||||
try:
|
||||
results = client.search_tracks(query, limit=limit)
|
||||
except TypeError:
|
||||
results = client.search_tracks(query) # clients with no limit kwarg
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
rows: List[Dict[str, Any]] = []
|
||||
for r in results or []:
|
||||
row = normalize_search_result(r, source)
|
||||
if row is not None:
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def resolve_hint_fields(
|
||||
source: str,
|
||||
track_id: str,
|
||||
*,
|
||||
client_factory: Optional[Callable[[str], Any]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Resolve the picked track to the fields a hint needs (album_id critically,
|
||||
plus isrc / track# / disc# / album name+type). One lookup for one chosen row.
|
||||
Returns ``None`` if it can't be resolved (caller surfaces an error)."""
|
||||
factory = client_factory or _default_client_factory
|
||||
try:
|
||||
client = factory(source)
|
||||
except Exception:
|
||||
client = None
|
||||
if client is None or not hasattr(client, "get_track_details"):
|
||||
return None
|
||||
try:
|
||||
details = client.get_track_details(track_id)
|
||||
except Exception:
|
||||
return None
|
||||
if not details:
|
||||
return None
|
||||
|
||||
album = _get(details, "album") or {}
|
||||
album_id = _get(album, "id") if not isinstance(album, str) else None
|
||||
album_name = _get(album, "name") if not isinstance(album, str) else album
|
||||
album_type = _get(album, "album_type") or _get(details, "album_type")
|
||||
total = _get(album, "total_tracks") or _get(details, "total_tracks")
|
||||
|
||||
artists = _get(details, "artists") or []
|
||||
artist_id = None
|
||||
artist_name = ""
|
||||
if isinstance(artists, list) and artists:
|
||||
artist_id = _get(artists[0], "id")
|
||||
artist_name = ", ".join(str(_get(a, "name", a) if not isinstance(a, str) else a) for a in artists)
|
||||
|
||||
ext = _get(details, "external_ids") or {}
|
||||
isrc = _get(details, "isrc") or (ext.get("isrc") if isinstance(ext, dict) else None)
|
||||
|
||||
if not album_id:
|
||||
return None # without an album_id the import can't fetch the tracklist
|
||||
|
||||
return {
|
||||
"source": source,
|
||||
"track_id": str(track_id),
|
||||
"album_id": str(album_id),
|
||||
"artist_id": str(artist_id) if artist_id else None,
|
||||
"track_title": _get(details, "name") or _get(details, "title") or "",
|
||||
"album_name": str(album_name or ""),
|
||||
"artist_name": artist_name,
|
||||
"album_type": infer_release_type(album_type, total),
|
||||
"track_number": _get(details, "track_number"),
|
||||
"disc_number": _get(details, "disc_number") or 1,
|
||||
"isrc": isrc or None,
|
||||
}
|
||||
|
||||
|
||||
def _default_client_factory(source: str):
|
||||
from core.metadata.registry import get_client_for_source
|
||||
return get_client_for_source(source)
|
||||
|
||||
|
||||
def available_sources() -> List[Dict[str, Any]]:
|
||||
"""The source tabs for the modal: every metadata source with a live client,
|
||||
the primary one flagged ``active`` so the UI selects it by default."""
|
||||
from core.metadata.registry import (
|
||||
METADATA_SOURCE_PRIORITY,
|
||||
get_client_for_source,
|
||||
get_primary_source,
|
||||
)
|
||||
|
||||
try:
|
||||
primary = get_primary_source()
|
||||
except Exception:
|
||||
primary = None
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
seen = set()
|
||||
for src in METADATA_SOURCE_PRIORITY:
|
||||
if src in seen:
|
||||
continue
|
||||
seen.add(src)
|
||||
try:
|
||||
client = get_client_for_source(src)
|
||||
except Exception:
|
||||
client = None
|
||||
if client is None or not hasattr(client, "search_tracks"):
|
||||
continue
|
||||
out.append({
|
||||
"source": src,
|
||||
"label": src.replace("_", " ").title(),
|
||||
"active": src == primary,
|
||||
})
|
||||
# Guarantee the primary is selectable + first even if priority ordering missed it.
|
||||
if primary and not any(s["active"] for s in out):
|
||||
out.insert(0, {"source": primary, "label": primary.replace("_", " ").title(), "active": True})
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"infer_release_type",
|
||||
"normalize_search_result",
|
||||
"search_release_candidates",
|
||||
"resolve_hint_fields",
|
||||
"available_sources",
|
||||
]
|
||||
|
|
@ -3,12 +3,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import as_completed
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
from typing import Any, Callable, Dict
|
||||
|
||||
from core.imports.album import build_album_import_context, build_album_import_match_payload, resolve_album_artist_context
|
||||
from core.imports.context import get_import_context_artist, get_import_track_info, normalize_import_context
|
||||
|
|
@ -18,7 +16,6 @@ from core.imports.staging import (
|
|||
AUDIO_EXTENSIONS,
|
||||
get_import_suggestions_cache,
|
||||
get_primary_source as _get_primary_source,
|
||||
get_primary_source_label as _get_primary_source_label,
|
||||
get_staging_path as _get_staging_path,
|
||||
read_staging_file_metadata as _read_staging_file_metadata,
|
||||
refresh_import_suggestions_cache as _refresh_import_suggestions_cache,
|
||||
|
|
@ -51,7 +48,6 @@ class ImportRouteRuntime:
|
|||
read_staging_file_metadata: Callable[[str, str], Dict[str, Any]] = _read_staging_file_metadata
|
||||
read_tags: Callable[[str], Any] = _default_read_tags
|
||||
get_primary_source: Callable[[], str] = _get_primary_source
|
||||
get_primary_source_label: Callable[[], str] = _get_primary_source_label
|
||||
search_import_albums: Callable[..., list] = _search_import_albums
|
||||
search_import_tracks: Callable[..., list] = _search_import_tracks
|
||||
build_album_import_match_payload: Callable[..., Dict[str, Any]] = build_album_import_match_payload
|
||||
|
|
@ -73,219 +69,36 @@ class ImportRouteRuntime:
|
|||
logger: Any = module_logger
|
||||
|
||||
|
||||
# ── Shared staging scan ──────────────────────────────────────────────────────
|
||||
# Opening the Import page fires staging files/groups/hints together; each used to
|
||||
# os.walk the whole staging folder AND mutagen-read every file independently — 3×
|
||||
# the directory walk + 3× the tag I/O on every page open (the import-page scan
|
||||
# storm + memory spike, issue #935). They all need the same per-file tag data, so
|
||||
# scan ONCE and let all three derive their views in-memory. A short TTL + a lock
|
||||
# means the three near-simultaneous page-open requests (and any concurrent caller)
|
||||
# share a single scan instead of each kicking off a full re-read.
|
||||
_STAGING_SCAN_LOCK = threading.Lock()
|
||||
_STAGING_SCAN_TTL = 6.0 # seconds — covers the page-open burst; re-scans after
|
||||
_staging_scan_cache: Dict[str, Any] = {"path": None, "ts": 0.0, "records": None}
|
||||
# Bumped by invalidate_staging_scan_cache() so a background scan that finishes after an
|
||||
# import doesn't re-commit stale (pre-import) records (see the generation guard above).
|
||||
_staging_scan_generation: Dict[str, int] = {"value": 0}
|
||||
|
||||
# Background-scan plumbing: a large staging folder (whole-library migration, #947) makes
|
||||
# the synchronous scan exceed gunicorn's 120s request timeout. The runner moves the SAME
|
||||
# scan off the request thread; the endpoints report progress instead of blocking.
|
||||
_staging_scan_status: Dict[str, Any] = {
|
||||
"status": "idle", "scanned": 0, "total": 0, "path": None, "error": None,
|
||||
}
|
||||
_staging_scan_status_lock = threading.Lock()
|
||||
|
||||
|
||||
def _staging_cache_hit(staging_path: str) -> Optional[list]:
|
||||
"""The cached records for ``staging_path`` if still fresh, else None (no scan triggered)."""
|
||||
c = _staging_scan_cache
|
||||
if (c["records"] is not None and c["path"] == staging_path
|
||||
and (time.time() - c["ts"]) < _STAGING_SCAN_TTL):
|
||||
return c["records"]
|
||||
return None
|
||||
|
||||
|
||||
def ensure_background_staging_scan(runtime: ImportRouteRuntime, staging_path: str) -> None:
|
||||
"""Start a background scan for ``staging_path`` unless the cache is warm or a scan for
|
||||
this path is already running. Idempotent — safe to call on every request."""
|
||||
if _staging_cache_hit(staging_path) is not None:
|
||||
return
|
||||
with _staging_scan_status_lock:
|
||||
if (_staging_scan_status["status"] == "scanning"
|
||||
and _staging_scan_status["path"] == staging_path):
|
||||
return
|
||||
_staging_scan_status.update({"status": "scanning", "scanned": 0, "total": 0,
|
||||
"path": staging_path, "error": None})
|
||||
|
||||
def _run() -> None:
|
||||
try:
|
||||
_scan_staging_records(runtime, staging_path, progress=_staging_scan_status)
|
||||
with _staging_scan_status_lock:
|
||||
if _staging_scan_status["path"] == staging_path:
|
||||
_staging_scan_status["status"] = "done"
|
||||
except Exception as exc: # noqa: BLE001 — surface any scan error to the poller
|
||||
with _staging_scan_status_lock:
|
||||
_staging_scan_status.update({"status": "error", "error": str(exc)})
|
||||
|
||||
threading.Thread(target=_run, name="staging-scan", daemon=True).start()
|
||||
|
||||
|
||||
def get_staging_records_or_status(runtime: ImportRouteRuntime, staging_path: str,
|
||||
*, grace_seconds: float = 3.0) -> tuple[str, Any]:
|
||||
"""Non-blocking staging access for the page endpoints. Returns ``("ready", records)``
|
||||
when the cache is warm or the scan completes within ``grace_seconds`` (so small/normal
|
||||
folders still answer in a single request), otherwise ``("scanning", status_dict)`` after
|
||||
making sure a background scan is running."""
|
||||
records = _staging_cache_hit(staging_path)
|
||||
if records is not None:
|
||||
return ("ready", records)
|
||||
ensure_background_staging_scan(runtime, staging_path)
|
||||
deadline = time.time() + max(0.0, grace_seconds)
|
||||
while True:
|
||||
records = _staging_cache_hit(staging_path)
|
||||
if records is not None:
|
||||
return ("ready", records)
|
||||
with _staging_scan_status_lock:
|
||||
status = dict(_staging_scan_status)
|
||||
if status.get("status") == "error":
|
||||
return ("error", status)
|
||||
if time.time() >= deadline:
|
||||
return ("scanning", status)
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def _records_or_scanning_payload(runtime: ImportRouteRuntime, staging_path: str):
|
||||
"""Shared helper for the page endpoints: returns ``(records, None)`` when the scan is
|
||||
ready, or ``(None, payload)`` when a background scan is still running — the caller
|
||||
returns that payload so the page polls + shows progress instead of blocking/timing out.
|
||||
|
||||
A scan error is re-raised so the endpoint's own try/except logs + returns it exactly as
|
||||
when the scan ran inline (preserves the existing error contract)."""
|
||||
state, val = get_staging_records_or_status(runtime, staging_path)
|
||||
if state == "error":
|
||||
raise RuntimeError(val.get("error") or "staging scan failed")
|
||||
if state == "scanning":
|
||||
return None, {"success": True, "scanning": True,
|
||||
"progress": {"scanned": val.get("scanned", 0),
|
||||
"total": val.get("total", 0)}}
|
||||
return val, None
|
||||
|
||||
|
||||
def staging_scan_status(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
||||
"""Lightweight, instant scan-progress poll for the page (no grace-wait, no file I/O) —
|
||||
``ready`` true once the cache is warm and the files/groups/hints calls will answer fast."""
|
||||
try:
|
||||
staging_path = runtime.get_staging_path()
|
||||
except Exception as exc:
|
||||
return {"success": False, "error": str(exc)}, 500
|
||||
with _staging_scan_status_lock:
|
||||
st = dict(_staging_scan_status)
|
||||
return {
|
||||
"success": True,
|
||||
"ready": _staging_cache_hit(staging_path) is not None,
|
||||
"status": st.get("status", "idle"),
|
||||
"scanned": st.get("scanned", 0),
|
||||
"total": st.get("total", 0),
|
||||
"error": st.get("error"),
|
||||
}, 200
|
||||
|
||||
|
||||
def _scan_staging_records(runtime: ImportRouteRuntime, staging_path: str,
|
||||
*, progress: Optional[Dict[str, Any]] = None) -> list[Dict[str, Any]]:
|
||||
"""Walk staging + read each audio file's tags ONCE, returning per-file records
|
||||
that staging files/groups/hints all derive from. Briefly cached + locked so the
|
||||
page-open trio shares a single scan rather than each re-walking and re-reading.
|
||||
|
||||
``progress`` (optional, default None = unchanged behaviour) is a dict the scan
|
||||
updates live with ``total`` (audio-file count, from a fast first pass) and ``scanned``
|
||||
(tag-reads done so far) so a background runner can report progress. A generation guard
|
||||
keeps a scan that finishes AFTER an import (which bumped ``_staging_scan_generation``)
|
||||
from committing stale records to the cache."""
|
||||
now = time.time()
|
||||
cached = _staging_scan_cache
|
||||
if (cached["records"] is not None and cached["path"] == staging_path
|
||||
and (now - cached["ts"]) < _STAGING_SCAN_TTL):
|
||||
return cached["records"]
|
||||
|
||||
with _STAGING_SCAN_LOCK:
|
||||
# Double-check: another request may have filled the cache while we waited.
|
||||
now = time.time()
|
||||
if (cached["records"] is not None and cached["path"] == staging_path
|
||||
and (now - cached["ts"]) < _STAGING_SCAN_TTL):
|
||||
return cached["records"]
|
||||
|
||||
start_generation = _staging_scan_generation["value"]
|
||||
|
||||
# Pass 1 (fast): collect the audio-file list — no tag I/O — so we know the total.
|
||||
audio_files: list[tuple[str, str, Optional[str]]] = []
|
||||
if os.path.isdir(staging_path):
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
rel_dir = os.path.relpath(root, staging_path)
|
||||
top_folder = rel_dir.split(os.sep)[0] if rel_dir != "." else None
|
||||
for fname in filenames:
|
||||
if os.path.splitext(fname)[1].lower() in AUDIO_EXTENSIONS:
|
||||
audio_files.append((root, fname, top_folder))
|
||||
if progress is not None:
|
||||
progress["total"] = len(audio_files)
|
||||
progress["scanned"] = 0
|
||||
|
||||
# Pass 2 (slow): read each file's tags, updating progress as we go.
|
||||
records: list[Dict[str, Any]] = []
|
||||
for root, fname, top_folder in audio_files:
|
||||
full_path = os.path.join(root, fname)
|
||||
rel_path = os.path.relpath(full_path, staging_path)
|
||||
meta = runtime.read_staging_file_metadata(full_path, rel_path)
|
||||
records.append({
|
||||
"filename": fname, "rel_path": rel_path, "full_path": full_path,
|
||||
"extension": os.path.splitext(fname)[1].lower(),
|
||||
"title": meta["title"], "album": meta["album"],
|
||||
"artist": meta["artist"], "albumartist": meta["albumartist"],
|
||||
"track_number": meta["track_number"], "disc_number": meta["disc_number"],
|
||||
"top_folder": top_folder,
|
||||
})
|
||||
if progress is not None:
|
||||
progress["scanned"] += 1
|
||||
|
||||
# Generation guard: if an import invalidated the cache mid-scan, these records are
|
||||
# stale — return them to this caller but do NOT commit them as the shared cache.
|
||||
if _staging_scan_generation["value"] == start_generation:
|
||||
_staging_scan_cache.update({"path": staging_path, "ts": time.time(), "records": records})
|
||||
return records
|
||||
|
||||
|
||||
def invalidate_staging_scan_cache() -> None:
|
||||
"""Drop the cached staging scan (call after an import moves/removes files so the
|
||||
next files/groups/hints request reflects the new state immediately). Also bumps the
|
||||
scan generation so an in-flight background scan won't re-commit pre-import records."""
|
||||
_staging_scan_generation["value"] += 1
|
||||
_staging_scan_cache.update({"path": None, "ts": 0.0, "records": None})
|
||||
|
||||
|
||||
def staging_files(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
||||
"""Scan the staging folder and return audio files with tag metadata."""
|
||||
try:
|
||||
staging_path = runtime.get_staging_path()
|
||||
os.makedirs(staging_path, exist_ok=True)
|
||||
|
||||
records, scanning = _records_or_scanning_payload(runtime, staging_path)
|
||||
if scanning is not None:
|
||||
return scanning, 200
|
||||
files = []
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
for fname in filenames:
|
||||
ext = os.path.splitext(fname)[1].lower()
|
||||
if ext not in AUDIO_EXTENSIONS:
|
||||
continue
|
||||
full_path = os.path.join(root, fname)
|
||||
rel_path = os.path.relpath(full_path, staging_path)
|
||||
|
||||
files = [
|
||||
{
|
||||
"filename": r["filename"],
|
||||
"rel_path": r["rel_path"],
|
||||
"full_path": r["full_path"],
|
||||
"title": r["title"],
|
||||
"artist": r["albumartist"] or r["artist"] or "Unknown Artist",
|
||||
"album": r["album"],
|
||||
"track_number": r["track_number"],
|
||||
"disc_number": r["disc_number"],
|
||||
"extension": r["extension"],
|
||||
}
|
||||
for r in records
|
||||
]
|
||||
meta = runtime.read_staging_file_metadata(full_path, rel_path)
|
||||
|
||||
files.append(
|
||||
{
|
||||
"filename": fname,
|
||||
"rel_path": rel_path,
|
||||
"full_path": full_path,
|
||||
"title": meta["title"],
|
||||
"artist": meta["albumartist"] or meta["artist"] or "Unknown Artist",
|
||||
"album": meta["album"],
|
||||
"track_number": meta["track_number"],
|
||||
"disc_number": meta["disc_number"],
|
||||
"extension": ext,
|
||||
}
|
||||
)
|
||||
|
||||
files.sort(key=lambda f: f["filename"].lower())
|
||||
return {"success": True, "files": files, "staging_path": staging_path}, 200
|
||||
|
|
@ -301,28 +114,32 @@ def staging_groups(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
|||
if not os.path.isdir(staging_path):
|
||||
return {"success": True, "groups": []}, 200
|
||||
|
||||
records, scanning = _records_or_scanning_payload(runtime, staging_path)
|
||||
if scanning is not None:
|
||||
return scanning, 200
|
||||
|
||||
album_groups = {}
|
||||
for r in records:
|
||||
album = r["album"]
|
||||
artist = r["albumartist"] or r["artist"]
|
||||
if not album or not artist:
|
||||
continue
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
for fname in filenames:
|
||||
ext = os.path.splitext(fname)[1].lower()
|
||||
if ext not in AUDIO_EXTENSIONS:
|
||||
continue
|
||||
full_path = os.path.join(root, fname)
|
||||
rel_path = os.path.relpath(full_path, staging_path)
|
||||
|
||||
key = (album.lower().strip(), artist.lower().strip())
|
||||
if key not in album_groups:
|
||||
album_groups[key] = {"album": album.strip(), "artist": artist.strip(), "files": []}
|
||||
album_groups[key]["files"].append(
|
||||
{
|
||||
"filename": r["filename"],
|
||||
"full_path": r["full_path"],
|
||||
"title": r["title"],
|
||||
"track_number": r["track_number"],
|
||||
}
|
||||
)
|
||||
meta = runtime.read_staging_file_metadata(full_path, rel_path)
|
||||
album = meta["album"]
|
||||
artist = meta["albumartist"] or meta["artist"]
|
||||
if not album or not artist:
|
||||
continue
|
||||
|
||||
key = (album.lower().strip(), artist.lower().strip())
|
||||
if key not in album_groups:
|
||||
album_groups[key] = {"album": album.strip(), "artist": artist.strip(), "files": []}
|
||||
album_groups[key]["files"].append(
|
||||
{
|
||||
"filename": fname,
|
||||
"full_path": full_path,
|
||||
"title": meta["title"],
|
||||
"track_number": meta["track_number"],
|
||||
}
|
||||
)
|
||||
|
||||
groups = []
|
||||
for group in album_groups.values():
|
||||
|
|
@ -352,21 +169,30 @@ def staging_hints(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
|||
if not os.path.isdir(staging_path):
|
||||
return {"success": True, "hints": []}, 200
|
||||
|
||||
records, scanning = _records_or_scanning_payload(runtime, staging_path)
|
||||
if scanning is not None:
|
||||
return scanning, 200
|
||||
|
||||
tag_albums = {}
|
||||
folder_hints = {}
|
||||
for r in records:
|
||||
if r["top_folder"]:
|
||||
folder_hints[r["top_folder"]] = folder_hints.get(r["top_folder"], 0) + 1
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS]
|
||||
if not audio_files:
|
||||
continue
|
||||
|
||||
album = r["album"]
|
||||
artist = r["artist"] or r["albumartist"]
|
||||
if album:
|
||||
key = (album.strip(), (artist or "").strip())
|
||||
tag_albums[key] = tag_albums.get(key, 0) + 1
|
||||
rel_dir = os.path.relpath(root, staging_path)
|
||||
if rel_dir != ".":
|
||||
top_folder = rel_dir.split(os.sep)[0]
|
||||
folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files)
|
||||
|
||||
for fname in audio_files:
|
||||
full_path = os.path.join(root, fname)
|
||||
try:
|
||||
tags = runtime.read_tags(full_path)
|
||||
if tags:
|
||||
album = (tags.get("album") or [None])[0]
|
||||
artist = (tags.get("artist") or (tags.get("albumartist") or [None]))[0]
|
||||
if album:
|
||||
key = (album.strip(), (artist or "").strip())
|
||||
tag_albums[key] = tag_albums.get(key, 0) + 1
|
||||
except Exception as exc:
|
||||
runtime.logger.debug("tag read failed: %s", exc)
|
||||
|
||||
queries = []
|
||||
seen_queries_lower = set()
|
||||
|
|
@ -396,7 +222,7 @@ def staging_suggestions() -> tuple[Dict[str, Any], int]:
|
|||
"success": True,
|
||||
"suggestions": cache["suggestions"],
|
||||
"ready": cache["built"],
|
||||
"primary_source": _get_primary_source_label(),
|
||||
"primary_source": _get_primary_source(),
|
||||
}, 200
|
||||
|
||||
|
||||
|
|
@ -413,10 +239,7 @@ def search_albums(runtime: ImportRouteRuntime, query: str, limit: int = 12) -> t
|
|||
runtime.hydrabase_worker.enqueue(query, "albums")
|
||||
|
||||
albums = runtime.search_import_albums(query, limit=limit)
|
||||
# The label names the user's CONFIGURED source (Spotify Free reads as
|
||||
# 'spotify', not the deezer fallback the functional source downgrades to).
|
||||
return {"success": True, "albums": albums,
|
||||
"primary_source": runtime.get_primary_source_label()}, 200
|
||||
return {"success": True, "albums": albums, "primary_source": primary_source}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error searching albums for import: %s", exc)
|
||||
return {"success": False, "error": str(exc)}, 500
|
||||
|
|
@ -543,11 +366,6 @@ def album_process(runtime: ImportRouteRuntime, data: Dict[str, Any]) -> tuple[Di
|
|||
)
|
||||
runtime.refresh_import_suggestions_cache()
|
||||
|
||||
# Files just left staging — drop the shared scan so the next files/groups/hints
|
||||
# reflects reality immediately instead of waiting out the cache TTL.
|
||||
if processed > 0:
|
||||
invalidate_staging_scan_cache()
|
||||
|
||||
return {"success": True, "processed": processed, "total": len(matches), "errors": errors}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error processing album import: %s", exc)
|
||||
|
|
@ -567,8 +385,7 @@ def search_tracks(runtime: ImportRouteRuntime, query: str, limit: int = 10) -> t
|
|||
runtime.hydrabase_worker.enqueue(query, "tracks")
|
||||
|
||||
tracks = runtime.search_import_tracks(query, limit=limit)
|
||||
return {"success": True, "tracks": tracks,
|
||||
"primary_source": runtime.get_primary_source_label()}, 200
|
||||
return {"success": True, "tracks": tracks, "primary_source": primary_source}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error searching tracks for import: %s", exc)
|
||||
return {"success": False, "error": str(exc)}, 500
|
||||
|
|
@ -683,10 +500,6 @@ def singles_process(runtime: ImportRouteRuntime, files: list[Dict[str, Any]]) ->
|
|||
)
|
||||
runtime.refresh_import_suggestions_cache()
|
||||
|
||||
# Files just left staging — drop the shared scan so the list updates immediately.
|
||||
if processed > 0:
|
||||
invalidate_staging_scan_cache()
|
||||
|
||||
return {"success": True, "processed": processed, "total": len(files), "errors": errors}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error processing singles import: %s", exc)
|
||||
|
|
|
|||
|
|
@ -252,7 +252,7 @@ def record_library_history_download(context: Dict[str, Any]) -> None:
|
|||
origin, origin_context = derive_download_origin(context)
|
||||
|
||||
db = get_database()
|
||||
_history_id = db.add_library_history_entry(
|
||||
db.add_library_history_entry(
|
||||
event_type="download",
|
||||
title=title,
|
||||
artist_name=artist_name,
|
||||
|
|
@ -270,10 +270,6 @@ def record_library_history_download(context: Dict[str, Any]) -> None:
|
|||
origin_context=origin_context,
|
||||
verification_status=context.get("_verification_status"),
|
||||
)
|
||||
# Stash the row id so the live download task can link to its
|
||||
# library_history row (the Unverified review queue needs it).
|
||||
if isinstance(_history_id, int) and _history_id > 0:
|
||||
context["_history_id"] = _history_id
|
||||
except Exception as e:
|
||||
logger.debug("library history record failed: %s", e)
|
||||
|
||||
|
|
@ -569,22 +565,19 @@ def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[
|
|||
# ── Album row: same insert-or-fill-empty-fields shape ──
|
||||
album_source_col = source_columns.get("album")
|
||||
|
||||
# Group by CANONICAL release id when we have one (not just the name
|
||||
# string), so differently-named imports of the SAME release land in
|
||||
# one album row instead of splitting — which left the repair jobs
|
||||
# dressing each split row in its own cover art (Sokhi). Precedence:
|
||||
# name-hash id -> source release id -> (title, artist). Falls back to
|
||||
# the legacy name match, so nothing that grouped before stops now.
|
||||
from core.imports.album_grouping import find_existing_soulsync_album_id
|
||||
existing_album_id = find_existing_soulsync_album_id(
|
||||
cursor, name_key_id=album_id, artist_id=artist_id, album_name=album_name,
|
||||
album_source_col=album_source_col, album_source_id=album_source_id,
|
||||
cursor.execute(
|
||||
"SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'",
|
||||
(album_id,),
|
||||
)
|
||||
if existing_album_id is not None:
|
||||
album_id = existing_album_id
|
||||
row = (album_id,)
|
||||
else:
|
||||
row = None
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
cursor.execute(
|
||||
"SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? AND server_source = 'soulsync' LIMIT 1",
|
||||
(album_name, artist_id),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
album_id = row[0]
|
||||
|
||||
if row:
|
||||
_fill_empty_columns(
|
||||
|
|
|
|||
|
|
@ -1,289 +0,0 @@
|
|||
"""Audio-completeness guard — detect files whose container duration looks
|
||||
right but whose REAL audio is far shorter, or mostly silence.
|
||||
|
||||
Motivating bug: HiFi/Monochrome HLS assembly can yield a file whose container
|
||||
claims the full track length (e.g. 3:08) while only ~30s of audio actually
|
||||
decodes — the rest is missing. The duration-agreement and quality guards both
|
||||
pass (mutagen reads the container's 3:08 and the format/bitrate are fine), so
|
||||
nothing catches it until you listen. ffmpeg's ``time=`` even reports 0 with no
|
||||
error on such a file, so the robust signal is to DECODE the audio and compare
|
||||
the real duration (sample count / sample rate, via ``astats``) against the
|
||||
container duration. A separate ``silencedetect`` pass also flags genuine
|
||||
silence-padding.
|
||||
|
||||
The parsers here are pure and unit-tested; the ffmpeg invocations are
|
||||
integration glue that fails open (returns None) when ffmpeg or mutagen can't
|
||||
run, so a tooling problem never blocks a legitimate import.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("imports.silence")
|
||||
|
||||
# Real decoded audio must cover at least this fraction of the container
|
||||
# duration. A legit file decodes to ~100% (encoder padding aside); a truncated
|
||||
# file decodes to a small fraction (the Blossom file: 30s of a 188s container
|
||||
# = 16%). 0.85 leaves generous headroom against false positives.
|
||||
DEFAULT_MIN_DURATION_RATIO = 0.85
|
||||
|
||||
_SAMPLES_RE = re.compile(r"Number of samples:\s*([0-9]+)")
|
||||
|
||||
# Defaults: treat audio below -50 dB lasting >= 2s as silence, and reject when
|
||||
# more than half the track is silent. A normal song — even with quiet intros/
|
||||
# outros — sits far below 0.5; a 30s-real + padded-silence file sits near 0.85.
|
||||
DEFAULT_NOISE_DB = -50
|
||||
DEFAULT_MIN_SILENCE_S = 2.0
|
||||
DEFAULT_THRESHOLD = 0.5
|
||||
|
||||
_SILENCE_DURATION_RE = re.compile(r"silence_duration:\s*([0-9]+(?:\.[0-9]+)?)")
|
||||
|
||||
|
||||
def silence_ratio_from_output(ffmpeg_stderr: str, total_duration_s: float) -> float:
|
||||
"""Fraction of *total_duration_s* covered by detected silence.
|
||||
|
||||
Sums every ``silence_duration: X`` reported by ffmpeg ``silencedetect``
|
||||
and divides by the track length. Capped at 1.0; returns 0.0 when the
|
||||
duration is unknown/zero or no silence was reported.
|
||||
"""
|
||||
if not total_duration_s or total_duration_s <= 0:
|
||||
return 0.0
|
||||
total_silence = sum(float(m) for m in _SILENCE_DURATION_RE.findall(ffmpeg_stderr or ""))
|
||||
if total_silence <= 0:
|
||||
return 0.0
|
||||
return min(total_silence / total_duration_s, 1.0)
|
||||
|
||||
|
||||
def is_mostly_silent_reason(
|
||||
ffmpeg_stderr: str,
|
||||
total_duration_s: float,
|
||||
*,
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
) -> Optional[str]:
|
||||
"""Return a rejection reason when the silent fraction meets *threshold*."""
|
||||
ratio = silence_ratio_from_output(ffmpeg_stderr, total_duration_s)
|
||||
if ratio >= threshold:
|
||||
pct = round(ratio * 100)
|
||||
audible_s = round(total_duration_s * (1 - ratio))
|
||||
return (
|
||||
f"Audio is mostly silent: {pct}% silence (only ~{audible_s}s audible of "
|
||||
f"{round(total_duration_s)}s) — likely a truncated/preview file padded "
|
||||
f"to full length"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _ffmpeg_available() -> bool:
|
||||
try:
|
||||
subprocess.run(
|
||||
["ffmpeg", "-version"],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=10, check=True,
|
||||
)
|
||||
return True
|
||||
except (FileNotFoundError, subprocess.SubprocessError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def _probe_duration_s(file_path: str) -> Optional[float]:
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(file_path)
|
||||
if audio and audio.info and getattr(audio.info, "length", None):
|
||||
return float(audio.info.length)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("silence guard duration probe failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
|
||||
def detect_mostly_silent(
|
||||
file_path: str,
|
||||
*,
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
noise_db: int = DEFAULT_NOISE_DB,
|
||||
min_silence_s: float = DEFAULT_MIN_SILENCE_S,
|
||||
) -> Optional[str]:
|
||||
"""Run ffmpeg ``silencedetect`` over *file_path* and return a rejection
|
||||
reason when the file is mostly silence, else None.
|
||||
|
||||
Fails open: returns None when ffmpeg/mutagen are unavailable or error, so
|
||||
a tooling problem never quarantines a legitimate file.
|
||||
"""
|
||||
if not _ffmpeg_available():
|
||||
logger.debug("silence guard skipped — ffmpeg not available")
|
||||
return None
|
||||
|
||||
total_duration_s = _probe_duration_s(file_path)
|
||||
if not total_duration_s:
|
||||
return None
|
||||
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
"ffmpeg", "-hide_banner", "-nostats", "-i", file_path,
|
||||
"-af", f"silencedetect=noise={noise_db}dB:d={min_silence_s}",
|
||||
"-f", "null", "-",
|
||||
],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
|
||||
timeout=120,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
logger.debug("silence guard ffmpeg run failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace") if proc.stderr else ""
|
||||
return is_mostly_silent_reason(stderr, total_duration_s, threshold=threshold)
|
||||
|
||||
|
||||
# ── Truncation: real decoded duration vs container duration ────────────────
|
||||
|
||||
def measured_duration_from_astats(astats_stderr: str, sample_rate: int) -> Optional[float]:
|
||||
"""Real decoded audio duration in seconds from ffmpeg ``astats`` output.
|
||||
|
||||
``astats`` reports the per-channel ``Number of samples``; dividing by the
|
||||
sample rate gives the true decoded length. Returns None when the sample
|
||||
count or sample rate is unavailable.
|
||||
"""
|
||||
if not sample_rate or sample_rate <= 0:
|
||||
return None
|
||||
m = _SAMPLES_RE.search(astats_stderr or "")
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)) / float(sample_rate)
|
||||
|
||||
|
||||
def is_dsd_path(file_path: str) -> bool:
|
||||
"""True for DSD audio (.dsf / .dff). The decoded-samples truncation check is
|
||||
invalid for DSD: ffmpeg decodes DSD to PCM at a different rate than the DSD
|
||||
container's 2.8 MHz, so samples ÷ container-sample-rate massively under-counts
|
||||
and would falsely report the file as truncated (#939)."""
|
||||
return os.path.splitext(str(file_path or ''))[1].lower() in ('.dsf', '.dff')
|
||||
|
||||
|
||||
def incomplete_audio_reason(
|
||||
measured_s: Optional[float],
|
||||
container_s: Optional[float],
|
||||
*,
|
||||
min_ratio: float = DEFAULT_MIN_DURATION_RATIO,
|
||||
) -> Optional[str]:
|
||||
"""Return a rejection reason when the real decoded duration falls short of
|
||||
the container duration (a truncated file whose metadata over-states length).
|
||||
"""
|
||||
if not measured_s or not container_s or container_s <= 0:
|
||||
return None
|
||||
if measured_s >= container_s * min_ratio:
|
||||
return None
|
||||
pct = round(measured_s / container_s * 100)
|
||||
return (
|
||||
f"Incomplete audio: only ~{round(measured_s)}s actually decodes of a "
|
||||
f"{round(container_s)}s file ({pct}%) — truncated/broken download "
|
||||
f"(container duration over-states the real audio)"
|
||||
)
|
||||
|
||||
|
||||
def _measured_audio_duration_s(file_path: str, sample_rate: int) -> Optional[float]:
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["ffmpeg", "-hide_banner", "-nostats", "-i", file_path,
|
||||
"-af", "astats=metadata=1", "-f", "null", "-"],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=120,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
logger.debug("astats run failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace") if proc.stderr else ""
|
||||
return measured_duration_from_astats(stderr, sample_rate)
|
||||
|
||||
|
||||
def detect_incomplete_audio(
|
||||
file_path: str,
|
||||
*,
|
||||
min_ratio: float = DEFAULT_MIN_DURATION_RATIO,
|
||||
) -> Optional[str]:
|
||||
"""Decode the file and reject when the real audio is far shorter than the
|
||||
container claims. Fails open when ffmpeg/mutagen are unavailable.
|
||||
"""
|
||||
if not _ffmpeg_available():
|
||||
return None
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(file_path)
|
||||
if not (audio and audio.info):
|
||||
return None
|
||||
container_s = float(getattr(audio.info, "length", 0) or 0)
|
||||
sample_rate = int(getattr(audio.info, "sample_rate", 0) or 0)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("container probe failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
measured_s = _measured_audio_duration_s(file_path, sample_rate)
|
||||
return incomplete_audio_reason(measured_s, container_s, min_ratio=min_ratio)
|
||||
|
||||
|
||||
def detect_broken_audio(
|
||||
file_path: str,
|
||||
*,
|
||||
min_ratio: float = DEFAULT_MIN_DURATION_RATIO,
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
noise_db: int = DEFAULT_NOISE_DB,
|
||||
min_silence_s: float = DEFAULT_MIN_SILENCE_S,
|
||||
) -> Optional[str]:
|
||||
"""Combined post-download audio guard: reject a file that is truncated
|
||||
(real audio far shorter than the container) or mostly silence. Returns the
|
||||
first failure reason, or None when the audio looks complete.
|
||||
|
||||
Runs a SINGLE ffmpeg decode pass with both the ``astats`` (truncation) and
|
||||
``silencedetect`` (silence) filters chained — one decode of the file feeds
|
||||
both checks instead of two full decodes. Halves the CPU cost versus running
|
||||
``detect_incomplete_audio`` and ``detect_mostly_silent`` back to back.
|
||||
|
||||
Fails open: returns None when ffmpeg/mutagen are unavailable or error, so a
|
||||
tooling problem never quarantines a legitimate file.
|
||||
"""
|
||||
if not _ffmpeg_available():
|
||||
logger.debug("audio guard skipped — ffmpeg not available")
|
||||
return None
|
||||
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(file_path)
|
||||
if not (audio and audio.info):
|
||||
return None
|
||||
container_s = float(getattr(audio.info, "length", 0) or 0)
|
||||
sample_rate = int(getattr(audio.info, "sample_rate", 0) or 0)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("container probe failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
"ffmpeg", "-hide_banner", "-nostats", "-i", file_path,
|
||||
"-af", f"astats=metadata=1,silencedetect=noise={noise_db}dB:d={min_silence_s}",
|
||||
"-f", "null", "-",
|
||||
],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=120,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
logger.debug("audio guard ffmpeg run failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace") if proc.stderr else ""
|
||||
|
||||
# Truncation check first (real audio far shorter than the container) — but
|
||||
# NOT for DSD: the astats sample-count ÷ DSD-rate math is invalid there and
|
||||
# would always false-positive (#939). Silence detection below still applies.
|
||||
if not is_dsd_path(file_path):
|
||||
measured_s = measured_duration_from_astats(stderr, sample_rate)
|
||||
reason = incomplete_audio_reason(measured_s, container_s, min_ratio=min_ratio)
|
||||
if reason:
|
||||
return reason
|
||||
|
||||
# Then silence-padding (mostly-silent file).
|
||||
return is_mostly_silent_reason(stderr, container_s, threshold=threshold)
|
||||
|
|
@ -1,124 +0,0 @@
|
|||
"""Single -> parent-album resolution.
|
||||
|
||||
When a track is matched to a SINGLE release (album_type 'single', the single's
|
||||
name usually equal to the track title), it carries the single's name + the
|
||||
single's source album id. The canonical grouping in
|
||||
[core/imports/album_grouping.py] then files it under a different album row than
|
||||
its album-mates, and the album-grouped repair jobs dress that row in the
|
||||
single's art — songs of one album end up with different covers (Sokhi).
|
||||
|
||||
This module re-homes such a track onto the ALBUM it actually belongs to, so it
|
||||
carries the album's name/id and groups with the rest of the album.
|
||||
|
||||
Design: the SELECTION is a pure, conservative function (no I/O), and the lookup
|
||||
loop takes INJECTED fetchers, so both are unit-testable without a live metadata
|
||||
client. CONSERVATIVE by intent — it only re-homes a track when a real
|
||||
``album``-type release's tracklist *contains that exact track*. It never
|
||||
promotes a genuine standalone single and never guesses, because a wrong
|
||||
promotion would mis-home a real single onto an album (the inverse bug).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
_WS = re.compile(r"\s+")
|
||||
# Trailing version qualifiers that differ between a single and its album cut but
|
||||
# don't change track identity (kept conservative — only the obvious ones).
|
||||
_QUALIFIER = re.compile(
|
||||
r"\s*[\(\[]\s*(album version|single version|radio edit|remaster(ed)?( \d{4})?)\s*[\)\]]\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _norm(s: Any) -> str:
|
||||
"""Lowercase, strip a trailing '(Album Version)'-style qualifier, collapse
|
||||
whitespace — so 'Song' matches 'Song (Album Version)'."""
|
||||
t = str(s or "").strip().lower()
|
||||
t = _QUALIFIER.sub("", t)
|
||||
return _WS.sub(" ", t).strip()
|
||||
|
||||
|
||||
def _get(obj: Any, *keys: str, default=None):
|
||||
for k in keys:
|
||||
if isinstance(obj, dict):
|
||||
if obj.get(k) is not None:
|
||||
return obj.get(k)
|
||||
else:
|
||||
v = getattr(obj, k, None)
|
||||
if v is not None:
|
||||
return v
|
||||
return default
|
||||
|
||||
|
||||
def select_parent_album(track_title: str, candidate_albums: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||
"""Pick the parent ALBUM for ``track_title`` from normalized candidates, or
|
||||
None. Each candidate is ``{name, album_type, tracks: [title, ...], ...}``.
|
||||
|
||||
Conservative rules — a candidate qualifies ONLY when:
|
||||
* it is an ``album`` release (never single / ep / compilation), and
|
||||
* its name is not just the track title (that IS the single), and
|
||||
* its tracklist contains the track by exact normalized title.
|
||||
Returns the FIRST qualifying candidate (caller passes them in priority
|
||||
order, so the result is deterministic).
|
||||
"""
|
||||
tgt = _norm(track_title)
|
||||
if not tgt:
|
||||
return None
|
||||
for alb in candidate_albums or []:
|
||||
if str(_get(alb, "album_type", default="album")).lower() != "album":
|
||||
continue
|
||||
if _norm(_get(alb, "name", "title", default="")) == tgt:
|
||||
continue
|
||||
tracks = _get(alb, "tracks", default=[]) or []
|
||||
if any(_norm(t) == tgt for t in tracks):
|
||||
return alb
|
||||
return None
|
||||
|
||||
|
||||
def resolve_single_to_album(
|
||||
track_title: str,
|
||||
*,
|
||||
fetch_album_candidates: Callable[[], List[Dict[str, Any]]],
|
||||
fetch_album_tracks: Callable[[Dict[str, Any]], List[str]],
|
||||
max_albums: int = 8,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Find the parent album for a single-matched track. I/O is INJECTED so this
|
||||
is testable without a live client:
|
||||
* ``fetch_album_candidates()`` -> the artist's ALBUM-type releases (dicts
|
||||
with name/album_type/id/source), in priority order.
|
||||
* ``fetch_album_tracks(album)`` -> that album's track titles.
|
||||
Probes at most ``max_albums`` albums, lazily (stops at the first that
|
||||
contains the track). Fail-safe: any error / no confident match -> None
|
||||
(the track stays as it was matched). Returns the normalized winning album
|
||||
``{name, album_type, album_id, source, tracks}`` or None.
|
||||
"""
|
||||
if not _norm(track_title):
|
||||
return None
|
||||
try:
|
||||
albums = fetch_album_candidates() or []
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
probed = 0
|
||||
for alb in albums:
|
||||
if str(_get(alb, "album_type", default="album")).lower() != "album":
|
||||
continue
|
||||
if probed >= max_albums:
|
||||
break
|
||||
probed += 1
|
||||
try:
|
||||
tracks = fetch_album_tracks(alb) or []
|
||||
except Exception:
|
||||
continue
|
||||
normalized = {
|
||||
"name": _get(alb, "name", "title", default=""),
|
||||
"album_type": "album",
|
||||
"album_id": _get(alb, "id", "album_id"),
|
||||
"source": _get(alb, "source"),
|
||||
"tracks": list(tracks),
|
||||
}
|
||||
if select_parent_album(track_title, [normalized]):
|
||||
return normalized
|
||||
return None
|
||||
|
|
@ -56,12 +56,6 @@ def get_primary_source() -> str:
|
|||
return _get_primary_source()
|
||||
|
||||
|
||||
def get_primary_source_label() -> str:
|
||||
from core.metadata_service import get_primary_source_label as _get_primary_source_label
|
||||
|
||||
return _get_primary_source_label()
|
||||
|
||||
|
||||
def get_source_priority(preferred_source: str):
|
||||
from core.metadata_service import get_source_priority as _get_source_priority
|
||||
|
||||
|
|
|
|||
|
|
@ -65,64 +65,17 @@ def _coerce_spotify_data(track_info: Any) -> dict:
|
|||
return {}
|
||||
|
||||
|
||||
def read_embedded_track_number(file_path: str) -> Optional[int]:
|
||||
"""Read the track position from a downloaded audio file's own tags.
|
||||
|
||||
Streaming sources (Deezer/deemix, Qobuz, Tidal) and most Soulseek
|
||||
uploads write the correct album position into the file itself. That
|
||||
tag is authoritative for the *source's* idea of the track's place on
|
||||
its album — more reliable than a filename guess — so the resolver
|
||||
consults it before falling back to the filename / default-1 floor.
|
||||
|
||||
Issue #874-adjacent / "Track 01" bug: a single Deezer track is matched
|
||||
via Deezer's ``/search/track`` endpoint, which omits ``track_position``
|
||||
(core/deezer_client.py), so the metadata context never carried the
|
||||
real number — but the downloaded file *does* (deemix wrote it). This
|
||||
recovers it with no network call.
|
||||
|
||||
Returns a positive int, or None when the file has no usable
|
||||
tracknumber tag / can't be read. Never raises. Handles the common
|
||||
``"2/15"`` (number/total) form by taking the leading number.
|
||||
"""
|
||||
if not file_path:
|
||||
return None
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(file_path, easy=True)
|
||||
if audio is None:
|
||||
return None
|
||||
raw = audio.get('tracknumber')
|
||||
if isinstance(raw, list):
|
||||
raw = raw[0] if raw else None
|
||||
if raw is None:
|
||||
return None
|
||||
# "2/15" -> "2"; bare "2" -> "2".
|
||||
text = str(raw).split('/', 1)[0].strip()
|
||||
return _coerce_positive(text)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def resolve_track_number(
|
||||
album_info: Any,
|
||||
track_info: Any,
|
||||
file_path: str,
|
||||
embedded_track_number: Any = None,
|
||||
) -> Optional[int]:
|
||||
"""Walk the resolution chain and return the first valid positive
|
||||
int found, or None when every source is missing / unusable.
|
||||
|
||||
Order: album_info -> track_info -> nested spotify_data -> filename ->
|
||||
``embedded_track_number`` (the source-written file tag, when the caller
|
||||
supplies it). Caller is responsible for the final default-1 floor —
|
||||
leaving that out of this function so tests can pin "everything missing
|
||||
Caller is responsible for the final default-1 floor — leaving
|
||||
that out of this function so tests can pin "everything missing
|
||||
returns None" separate from the floor behaviour.
|
||||
|
||||
``embedded_track_number`` is passed in (not read here) so this stays a
|
||||
pure function — the file I/O lives in :func:`read_embedded_track_number`.
|
||||
It is consulted **last**, only when every other source came up empty, so
|
||||
it can never override a value the pre-fix resolver already produced — it
|
||||
only fills the gap that would otherwise hit the default-1 floor.
|
||||
"""
|
||||
album_info = album_info if isinstance(album_info, dict) else {}
|
||||
track_info = track_info if isinstance(track_info, dict) else {}
|
||||
|
|
@ -143,69 +96,10 @@ def resolve_track_number(
|
|||
# default-1 floor is the single source of that fallback —
|
||||
# otherwise this resolver would silently fill 1 and the
|
||||
# downstream floor logic would have no effect.
|
||||
if file_path:
|
||||
try:
|
||||
from_filename = extract_explicit_track_number(file_path)
|
||||
except Exception:
|
||||
from_filename = None
|
||||
ff = _coerce_positive(from_filename)
|
||||
if ff is not None:
|
||||
return ff
|
||||
|
||||
# Embedded source-written file tag is consulted LAST — only when every
|
||||
# other source (metadata + the ripped-album "NN - Title" filename) came
|
||||
# up empty. This is deliberate: it can ONLY fill the gap that would
|
||||
# otherwise hit the caller's default-1 floor, so it never overrides a
|
||||
# value the pre-fix resolver would have used. A correctly-named file
|
||||
# with a stale/wrong embedded tag is therefore never regressed.
|
||||
return _coerce_positive(embedded_track_number)
|
||||
|
||||
|
||||
def normalize_disc_number(value) -> int:
|
||||
"""Coerce a disc value to a positive int, defaulting to 1.
|
||||
|
||||
Every track in a multi-disc album MUST carry a disc number, or Jellyfin/Plex
|
||||
leave the disc-less ones floating ungrouped above the disc sections (Sokhi's
|
||||
"tracks 3/9/15 at the top"). Upstream sources can hand back 0, None, '', or a
|
||||
non-numeric string for some tracks — especially when a track resolved to a
|
||||
different edition than its siblings — and the tag-writer only wrote the disc
|
||||
tag when it was truthy, so those tracks lost it entirely on the clear-then-
|
||||
rewrite. Flooring to >=1 here means a track is never written disc-less.
|
||||
"""
|
||||
if not file_path:
|
||||
return None
|
||||
try:
|
||||
n = int(value) # int, float, or clean int-string
|
||||
except (TypeError, ValueError):
|
||||
try:
|
||||
n = int(float(str(value).strip())) # tolerate "2.0"
|
||||
except (TypeError, ValueError):
|
||||
return 1
|
||||
return n if n >= 1 else 1
|
||||
|
||||
|
||||
def resolve_disc_for_track(original_search, album_info) -> int:
|
||||
"""The disc number for a track — resolved IDENTICALLY for the 'Disc N' folder
|
||||
(import pipeline) and the embedded tag (metadata.source), so the two can never
|
||||
disagree.
|
||||
|
||||
Sokhi: the pipeline synced the resolved track_number into album_info (so the
|
||||
folder matched the tag) but never did the same for disc — the folder used
|
||||
album_info's original disc (often 1) while the tag took the per-track disc
|
||||
(e.g. 2/3 from a MusicBrainz multi-medium release). Result: a disc-2/3 track
|
||||
landed in the Disc 1 folder, collapsing every disc's tracks into one folder.
|
||||
|
||||
Returns the first VALID positive disc — the per-track search's, else the album
|
||||
context's — else 1. A falsy/unknown (0/None/'') per-track disc falls through to
|
||||
the album rather than flooring early. Single source of truth so both call sites
|
||||
stay in lockstep."""
|
||||
for src in ((original_search or {}), (album_info or {})):
|
||||
raw = src.get("disc_number")
|
||||
try:
|
||||
n = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
try:
|
||||
n = int(float(str(raw).strip()))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if n >= 1:
|
||||
return n
|
||||
return 1
|
||||
from_filename = extract_explicit_track_number(file_path)
|
||||
except Exception:
|
||||
from_filename = None
|
||||
return _coerce_positive(from_filename)
|
||||
|
|
|
|||
|
|
@ -103,12 +103,9 @@ def select_version_mismatch_fallback(
|
|||
# don't guess which the user wants.
|
||||
return None
|
||||
|
||||
# First tried = oldest = best (the retry walks candidates best-first; what
|
||||
# "best" means follows the active ordering — confidence-first by default, or
|
||||
# ranked-target quality when best_quality mode / the rank_candidates_by_quality
|
||||
# toggle is on, so this naturally accepts the highest-quality candidate then).
|
||||
# The id is a "<date>_<time>_<name>" timestamp prefix, so the lexicographically
|
||||
# smallest id is the earliest attempt.
|
||||
# First tried = oldest = highest-confidence (the retry walks candidates
|
||||
# best-first). The id is a "<date>_<time>_<name>" timestamp prefix, so the
|
||||
# lexicographically smallest id is the earliest attempt.
|
||||
return min((e for _, e in candidates), key=lambda e: e["id"])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -739,27 +739,9 @@ class iTunesClient:
|
|||
cache = get_metadata_cache()
|
||||
cached = cache.get_entity('itunes', 'album', f"{album_id}_tracks")
|
||||
if cached and cached.get('items'):
|
||||
# #918 follow-up: a tracks entry cached BEFORE the limit=200 fix is truncated
|
||||
# to 50 and survives in the persistent cache (30-day TTL), so every window that
|
||||
# loads this album from cache still shows 50 — not just the one path that was
|
||||
# re-fetched fresh. Self-heal: entries written by the fixed fetch carry
|
||||
# `_complete`; a legacy entry without it is re-validated against the album's
|
||||
# known trackCount and re-fetched if it's short. (trackCount comes from the
|
||||
# collection metadata and is unaffected by the tracks-limit bug.)
|
||||
if cached.get('_complete'):
|
||||
return cached
|
||||
album_meta = cache.get_entity('itunes', 'album', str(album_id))
|
||||
expected = (album_meta or {}).get('trackCount')
|
||||
if not (isinstance(expected, int) and expected > len(cached['items'])):
|
||||
return cached
|
||||
logger.info(
|
||||
"iTunes album %s tracks cache looks truncated (%d cached < %d trackCount) — refetching",
|
||||
album_id, len(cached['items']), expected,
|
||||
)
|
||||
return cached
|
||||
|
||||
# #918: the iTunes Lookup API returns only 50 related entities unless `limit` is
|
||||
# passed (max 200), so albums >50 tracks were truncated in the download window.
|
||||
results = self._lookup(id=album_id, entity='song', limit=200)
|
||||
results = self._lookup(id=album_id, entity='song')
|
||||
|
||||
if not results:
|
||||
return None
|
||||
|
|
@ -781,7 +763,7 @@ class iTunesClient:
|
|||
try:
|
||||
fb_results = self.session.get(
|
||||
self.LOOKUP_URL,
|
||||
params={'id': album_id, 'entity': 'song', 'country': fallback, 'limit': 200}, # #918
|
||||
params={'id': album_id, 'entity': 'song', 'country': fallback},
|
||||
timeout=15
|
||||
)
|
||||
if fb_results.status_code == 200:
|
||||
|
|
@ -867,11 +849,7 @@ class iTunesClient:
|
|||
'items': tracks,
|
||||
'total': len(tracks),
|
||||
'limit': len(tracks),
|
||||
'next': None,
|
||||
# Marks this entry as fetched with the limit=200 query (#918) so the
|
||||
# stale-cache self-heal above trusts it and never re-fetches in a loop —
|
||||
# important for region-restricted albums where len(tracks) < trackCount.
|
||||
'_complete': True,
|
||||
'next': None
|
||||
}
|
||||
|
||||
# Cache the album tracks listing
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ from datetime import datetime
|
|||
import json
|
||||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
from core.library.bulk_paginate import paginate_all_items
|
||||
|
||||
# Shared dataclasses live in the neutral media_server package — every
|
||||
# server client used to define a near-identical XTrackInfo /
|
||||
|
|
@ -106,7 +105,6 @@ class JellyfinTrack:
|
|||
self.title = jellyfin_data.get('Name', 'Unknown Track')
|
||||
self.duration = jellyfin_data.get('RunTimeTicks', 0) // 10000 # Convert from ticks to milliseconds
|
||||
self.trackNumber = jellyfin_data.get('IndexNumber')
|
||||
self.discNumber = jellyfin_data.get('ParentIndexNumber') # multi-disc: disc number
|
||||
self.year = jellyfin_data.get('ProductionYear')
|
||||
self.userRating = jellyfin_data.get('UserData', {}).get('Rating')
|
||||
self.addedAt = self._parse_date(jellyfin_data.get('DateCreated'))
|
||||
|
|
@ -513,8 +511,12 @@ class JellyfinClient(MediaServerClient):
|
|||
try:
|
||||
# SIMPLIFIED APPROACH: Fetch all tracks, then all albums separately (robust and fast)
|
||||
logger.info("Fetching all tracks in bulk...")
|
||||
|
||||
def _fetch_tracks_page(start_index, limit):
|
||||
all_tracks = []
|
||||
start_index = 0
|
||||
limit = 10000
|
||||
consecutive_failures = 0
|
||||
|
||||
while True:
|
||||
params = {
|
||||
'ParentId': self.music_library_id,
|
||||
'IncludeItemTypes': 'Audio',
|
||||
|
|
@ -525,19 +527,41 @@ class JellyfinClient(MediaServerClient):
|
|||
'StartIndex': start_index,
|
||||
'Limit': limit
|
||||
}
|
||||
|
||||
response = self._make_request(f'/Users/{self.user_id}/Items', params)
|
||||
return response.get('Items', []) if response else None # None = failed page
|
||||
|
||||
# Page in modest chunks so progress is reported every page — a single
|
||||
# huge silent request used to trip the 300s no-progress watchdog on
|
||||
# slow servers even though it was alive (see bulk_paginate docstring).
|
||||
all_tracks = paginate_all_items(
|
||||
_fetch_tracks_page,
|
||||
report_progress=self._progress_callback,
|
||||
label="tracks",
|
||||
on_retry_wait=lambda: time.sleep(5),
|
||||
)
|
||||
|
||||
if not response:
|
||||
consecutive_failures += 1
|
||||
# Wait before retrying — the server may still be processing the timed-out request
|
||||
time.sleep(5)
|
||||
if limit > 1000:
|
||||
limit = limit // 2
|
||||
consecutive_failures = 0 # Reset — give the smaller batch a fair chance
|
||||
logger.warning(f"Track fetch failed - reducing batch size to {limit}")
|
||||
continue
|
||||
elif consecutive_failures >= 2:
|
||||
logger.warning("Multiple track fetch failures at minimum batch size - stopping")
|
||||
break
|
||||
else:
|
||||
logger.warning("Track fetch failed at minimum batch size - retrying once")
|
||||
continue
|
||||
|
||||
consecutive_failures = 0
|
||||
batch_tracks = response.get('Items', [])
|
||||
if not batch_tracks:
|
||||
break
|
||||
|
||||
all_tracks.extend(batch_tracks)
|
||||
|
||||
if len(batch_tracks) < limit:
|
||||
break
|
||||
|
||||
start_index += limit
|
||||
progress_msg = f"Fetched {len(all_tracks)} tracks so far..."
|
||||
logger.info(f" {progress_msg} (batch size: {limit})")
|
||||
if self._progress_callback:
|
||||
self._progress_callback(progress_msg)
|
||||
|
||||
# Group tracks by album ID for instant lookup
|
||||
self._track_cache = {}
|
||||
for track_data in all_tracks:
|
||||
|
|
@ -553,8 +577,12 @@ class JellyfinClient(MediaServerClient):
|
|||
|
||||
# STEP 2: Fetch all albums in bulk (same proven pattern)
|
||||
logger.info("Fetching all albums in bulk...")
|
||||
|
||||
def _fetch_albums_page(start_index, limit):
|
||||
all_albums = []
|
||||
start_index = 0
|
||||
limit = 10000
|
||||
consecutive_failures = 0
|
||||
|
||||
while True:
|
||||
params = {
|
||||
'ParentId': self.music_library_id,
|
||||
'IncludeItemTypes': 'MusicAlbum',
|
||||
|
|
@ -565,16 +593,41 @@ class JellyfinClient(MediaServerClient):
|
|||
'StartIndex': start_index,
|
||||
'Limit': limit
|
||||
}
|
||||
|
||||
response = self._make_request(f'/Users/{self.user_id}/Items', params)
|
||||
return response.get('Items', []) if response else None # None = failed page
|
||||
|
||||
all_albums = paginate_all_items(
|
||||
_fetch_albums_page,
|
||||
report_progress=self._progress_callback,
|
||||
label="albums",
|
||||
on_retry_wait=lambda: time.sleep(5),
|
||||
)
|
||||
|
||||
if not response:
|
||||
consecutive_failures += 1
|
||||
# Wait before retrying — the server may still be processing the timed-out request
|
||||
time.sleep(5)
|
||||
if limit > 1000:
|
||||
limit = limit // 2
|
||||
consecutive_failures = 0 # Reset — give the smaller batch a fair chance
|
||||
logger.warning(f"Album fetch failed - reducing batch size to {limit}")
|
||||
continue
|
||||
elif consecutive_failures >= 2:
|
||||
logger.warning("Multiple album fetch failures at minimum batch size - stopping")
|
||||
break
|
||||
else:
|
||||
logger.warning("Album fetch failed at minimum batch size - retrying once")
|
||||
continue
|
||||
|
||||
consecutive_failures = 0
|
||||
batch_albums = response.get('Items', [])
|
||||
if not batch_albums:
|
||||
break
|
||||
|
||||
all_albums.extend(batch_albums)
|
||||
|
||||
if len(batch_albums) < limit:
|
||||
break
|
||||
|
||||
start_index += limit
|
||||
progress_msg = f"Fetched {len(all_albums)} albums so far..."
|
||||
logger.info(f" {progress_msg} (batch size: {limit})")
|
||||
if self._progress_callback:
|
||||
self._progress_callback(progress_msg)
|
||||
|
||||
# Group albums by artist ID for instant lookup
|
||||
self._album_cache = {}
|
||||
for album_data in all_albums:
|
||||
|
|
@ -1658,81 +1711,6 @@ class JellyfinClient(MediaServerClient):
|
|||
logger.error(f"Error reconciling Jellyfin playlist '{playlist_name}': {e}")
|
||||
return False
|
||||
|
||||
def get_playlist_track_ids(self, playlist_id: str) -> List[str]:
|
||||
"""The playlist's current track ids (Item Ids), in current order. [] on miss."""
|
||||
if not self.ensure_connection():
|
||||
return []
|
||||
try:
|
||||
resp = self._make_request(f'/Playlists/{playlist_id}/Items', {'UserId': self.user_id})
|
||||
if not resp:
|
||||
return []
|
||||
return [str(i.get('Id')) for i in resp.get('Items', []) if i.get('Id')]
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting Jellyfin playlist track ids '{playlist_id}': {e}")
|
||||
return []
|
||||
|
||||
def reorder_playlist(self, playlist_id, playlist_name: str, ordered_ids) -> bool:
|
||||
"""In-place reorder a playlist to an exact ordered track-id list ('Align
|
||||
playlists'). Removes any current entry whose track NOT in ``ordered_ids``
|
||||
('Mirror source' drops extras; 'Keep extras' keeps them in the list), then
|
||||
moves each desired track to its target index via the Jellyfin Move endpoint.
|
||||
Operates on the existing playlist (DELETE EntryIds + Items/{entryId}/Move/{i})
|
||||
so its poster, name and Id survive — no delete/recreate."""
|
||||
if not self.ensure_connection():
|
||||
return False
|
||||
try:
|
||||
import requests
|
||||
ordered = [str(i) for i in (ordered_ids or []) if str(i)]
|
||||
ordered_set = set(ordered)
|
||||
|
||||
# Entries carry both the track Id and the PlaylistItemId (entry id);
|
||||
# move/remove operate on the entry id.
|
||||
entries = [] # (track_id, entry_id) in current order
|
||||
resp = self._make_request(f'/Playlists/{playlist_id}/Items', {'UserId': self.user_id})
|
||||
if resp:
|
||||
for item in resp.get('Items', []):
|
||||
tid = str(item.get('Id') or '')
|
||||
eid = str(item.get('PlaylistItemId') or '')
|
||||
if tid:
|
||||
entries.append((tid, eid))
|
||||
if not entries:
|
||||
logger.error(f"Jellyfin reorder: no entries for playlist {playlist_id}")
|
||||
return False
|
||||
|
||||
by_tid = {tid: eid for tid, eid in entries}
|
||||
hdr = {'X-Emby-Token': self.api_key}
|
||||
|
||||
# Drop entries not in the desired list (extras, for Mirror source).
|
||||
extra_eids = [eid for tid, eid in entries if tid not in ordered_set and eid]
|
||||
for i in range(0, len(extra_eids), 100):
|
||||
batch = extra_eids[i:i + 100]
|
||||
r = requests.delete(
|
||||
f"{self.base_url}/Playlists/{playlist_id}/Items",
|
||||
params={'EntryIds': ','.join(batch)}, headers=hdr, timeout=30,
|
||||
)
|
||||
if r.status_code not in (200, 204):
|
||||
logger.error(f"Jellyfin reorder remove failed: HTTP {r.status_code}")
|
||||
return False
|
||||
|
||||
# Move each desired track to its target index, ascending — each move
|
||||
# lands the item exactly at index i without disturbing 0..i-1.
|
||||
for idx, tid in enumerate(ordered):
|
||||
eid = by_tid.get(tid)
|
||||
if not eid:
|
||||
continue
|
||||
r = requests.post(
|
||||
f"{self.base_url}/Playlists/{playlist_id}/Items/{eid}/Move/{idx}",
|
||||
headers=hdr, timeout=30,
|
||||
)
|
||||
if r.status_code not in (200, 204):
|
||||
logger.warning(f"Jellyfin reorder move failed for {tid}: HTTP {r.status_code}")
|
||||
return False
|
||||
logger.info(f"Aligned Jellyfin playlist '{playlist_name}' order ({len(ordered)} tracks)")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error reordering Jellyfin playlist '{playlist_name}': {e}")
|
||||
return False
|
||||
|
||||
def update_playlist(self, playlist_name: str, tracks) -> bool:
|
||||
"""Update an existing playlist or create it if it doesn't exist"""
|
||||
if not self.ensure_connection():
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
"""Paginate a bulk media-server fetch while feeding the no-progress watchdog.
|
||||
|
||||
The library scan fetches every track/album by paging a server API. The DB-update
|
||||
watchdog (``core/database_update_health.py``) kills a job that reports no progress
|
||||
for 300s. The old Jellyfin fetch used a single 10 000-item page, so a whole
|
||||
library came back in ONE request that emitted NO progress while it was in flight —
|
||||
on a slow server that single request exceeded 300s and the watchdog declared the
|
||||
job "stuck" even though it was alive, not hung (Discord: DXP4800 NAS, 7148 tracks,
|
||||
"Fetching all tracks in bulk…").
|
||||
|
||||
``paginate_all_items`` pages at a size chosen so progress is emitted on a cadence
|
||||
set by the PAGE SIZE, not the library size — the watchdog is fed every page, so it
|
||||
can never starve mid-fetch regardless of how big the library is. It is pure: all
|
||||
I/O lives in the injected ``fetch_page``, so the pagination + progress + failure-
|
||||
shrink logic is unit-testable without a server.
|
||||
|
||||
This does NOT change WHAT is fetched (same query, same fields, same items) — only
|
||||
how it's paged and that every page reports progress (the old loop skipped progress
|
||||
on the final/only page, which is the entire bug for a sub-page-size library).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, List, Optional
|
||||
|
||||
# Page size for bulk library fetches. Small enough that a single request stays
|
||||
# well under the 300s no-progress watchdog even on a slow NAS, and that progress
|
||||
# is reported every page. NOT a performance knob — a resilience/observability one.
|
||||
DEFAULT_PAGE_SIZE = 1000
|
||||
|
||||
# Floor the failure-shrink can reach before giving up — a server that can't return
|
||||
# even this many items in one request is genuinely struggling.
|
||||
DEFAULT_MIN_PAGE_SIZE = 250
|
||||
|
||||
|
||||
def paginate_all_items(
|
||||
fetch_page: Callable[[int, int], Optional[List[Any]]],
|
||||
*,
|
||||
report_progress: Optional[Callable[[str], None]] = None,
|
||||
label: str = "items",
|
||||
page_size: int = DEFAULT_PAGE_SIZE,
|
||||
min_page_size: int = DEFAULT_MIN_PAGE_SIZE,
|
||||
on_retry_wait: Optional[Callable[[], None]] = None,
|
||||
) -> List[Any]:
|
||||
"""Page through ``fetch_page(start_index, limit)`` until the server is drained.
|
||||
|
||||
``fetch_page`` returns the page's items (a list, possibly empty = end), or
|
||||
``None`` to signal a FAILED request (timeout/error) — on failure the page size
|
||||
is halved down to ``min_page_size`` and retried, then abandoned after two
|
||||
consecutive failures at the floor.
|
||||
|
||||
Progress is reported after EVERY non-empty page (including the final/only one),
|
||||
so a no-progress watchdog is fed on a cadence set by ``page_size`` — never by
|
||||
the total library size. Returns every item gathered.
|
||||
"""
|
||||
items: List[Any] = []
|
||||
start_index = 0
|
||||
limit = page_size
|
||||
consecutive_failures = 0
|
||||
|
||||
while True:
|
||||
batch = fetch_page(start_index, limit)
|
||||
|
||||
if batch is None: # failed request
|
||||
consecutive_failures += 1
|
||||
if on_retry_wait is not None:
|
||||
on_retry_wait()
|
||||
if limit > min_page_size:
|
||||
limit = max(min_page_size, limit // 2)
|
||||
consecutive_failures = 0 # give the smaller batch a fair chance
|
||||
continue
|
||||
if consecutive_failures >= 2:
|
||||
break # struggling at the floor — stop with what we have
|
||||
continue
|
||||
|
||||
consecutive_failures = 0
|
||||
if not batch:
|
||||
break # drained
|
||||
|
||||
items.extend(batch)
|
||||
# Feed the watchdog on EVERY page — this is the line the old loop only ran
|
||||
# when there was a *next* page, so a sub-page-size library reported nothing.
|
||||
if report_progress is not None:
|
||||
report_progress(f"Fetched {len(items)} {label} so far...")
|
||||
|
||||
if len(batch) < limit:
|
||||
break # last (partial) page
|
||||
start_index += limit
|
||||
|
||||
return items
|
||||
|
||||
|
||||
__all__ = ["paginate_all_items", "DEFAULT_PAGE_SIZE", "DEFAULT_MIN_PAGE_SIZE"]
|
||||
|
|
@ -10,7 +10,6 @@ from __future__ import annotations
|
|||
from dataclasses import dataclass
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
|
@ -122,20 +121,6 @@ def import_existing_track_for_album_slot(album_id: str, payload: dict, deps: Mis
|
|||
if album_data.get("server_source") and source_track.get("server_source") and album_data["server_source"] != source_track["server_source"]:
|
||||
raise MissingTrackImportError("Selected track belongs to a different library source", 400)
|
||||
|
||||
# #917: "I have this" rebuilds the destination path from album metadata. When the album row
|
||||
# has no year, the rebuilt path drops the $year and the copied file lands in a NEW, yearless
|
||||
# directory instead of the album's existing folder. Recover the year from a sibling track so
|
||||
# the import reuses the same directory.
|
||||
if not album_data.get("year"):
|
||||
recovered_year = _existing_album_year_from_sibling(
|
||||
database, album_id, deps.resolve_library_file_path_fn,
|
||||
int(expected.get("disc_number") or 1), int(expected.get("track_number") or 1),
|
||||
)
|
||||
if recovered_year:
|
||||
album_data["year"] = recovered_year
|
||||
logger.info("[I Have This] recovered album year %s from existing folder for album %s",
|
||||
recovered_year, album_id)
|
||||
|
||||
source_path = deps.resolve_library_file_path_fn(source_track.get("file_path"))
|
||||
if not source_path:
|
||||
raise MissingTrackImportError(_file_not_found_message(source_track.get("file_path")), 404)
|
||||
|
|
@ -441,50 +426,6 @@ def _sync_imported_track(deps: MissingTrackImportDeps, track_id, expected_title:
|
|||
logger.debug("Existing-track import server sync skipped/failed: %s", sync_err)
|
||||
|
||||
|
||||
def _existing_album_year_from_sibling(
|
||||
database,
|
||||
album_id: str,
|
||||
resolve_library_file_path_fn: Callable[[Optional[str]], Optional[str]],
|
||||
target_disc: int,
|
||||
target_track: int,
|
||||
) -> Optional[str]:
|
||||
"""Find the release year already baked into this album's on-disk folder (#917).
|
||||
|
||||
Read from a sibling track — its own ``year`` column first, else a ``(YYYY)`` /
|
||||
``[YYYY]`` in the album folder name — so an "I have this" import reuses the album's
|
||||
existing directory instead of rebuilding a yearless one. Returns the 4-digit year
|
||||
string, or None when no signal exists.
|
||||
"""
|
||||
try:
|
||||
with database._get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT file_path, year FROM tracks
|
||||
WHERE album_id = ?
|
||||
AND file_path IS NOT NULL AND file_path != ''
|
||||
AND NOT (COALESCE(disc_number, 1) = ? AND track_number = ?)
|
||||
ORDER BY COALESCE(disc_number, 1), track_number
|
||||
LIMIT 12
|
||||
""",
|
||||
(album_id, target_disc, target_track),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
for row in rows:
|
||||
year = row["year"]
|
||||
if year is not None and str(year).strip()[:4].isdigit():
|
||||
return str(year).strip()[:4]
|
||||
resolved = resolve_library_file_path_fn(row["file_path"])
|
||||
if resolved:
|
||||
folder = os.path.basename(os.path.dirname(resolved))
|
||||
match = re.search(r"[(\[](\d{4})[)\]]", folder) # "Album (2019)" / "Album [2019]"
|
||||
if match:
|
||||
return match.group(1)
|
||||
except Exception as exc:
|
||||
logger.debug("Could not recover album year from sibling for %s: %s", album_id, exc)
|
||||
return None
|
||||
|
||||
|
||||
def copy_album_identity_from_target_sibling(
|
||||
database,
|
||||
album_id: str,
|
||||
|
|
|
|||
|
|
@ -138,25 +138,10 @@ def _collect_base_dirs(
|
|||
except Exception as e:
|
||||
logger.debug("music paths read failed: %s", e)
|
||||
|
||||
# Normalize to absolute forms so resolution does NOT depend on the calling
|
||||
# thread's CWD. A relative config like "./Transfer" otherwise only resolves
|
||||
# when os.path.isdir("./Transfer") happens to be true from the current CWD —
|
||||
# which fails in background workers whose CWD isn't the app root, leaving
|
||||
# base_dirs empty and every track "unresolved". For each candidate we try
|
||||
# the raw form first (cheap, preserves an already-absolute path), then its
|
||||
# os.path.abspath() form so "./Transfer" → "/app/Transfer".
|
||||
expanded: list[str] = []
|
||||
for c in candidates:
|
||||
if not c:
|
||||
continue
|
||||
expanded.append(c)
|
||||
if not os.path.isabs(c):
|
||||
expanded.append(os.path.abspath(c))
|
||||
|
||||
# De-duplicate while preserving order, drop empties / non-existent dirs.
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for c in expanded:
|
||||
for c in candidates:
|
||||
if not c or c in seen:
|
||||
continue
|
||||
seen.add(c)
|
||||
|
|
@ -239,23 +224,10 @@ def resolve_library_file_path_with_diagnostic(
|
|||
if not base_dirs:
|
||||
return None, attempt
|
||||
|
||||
# Try progressively shorter path suffixes against each base dir.
|
||||
#
|
||||
# Start at index 0 so a clean RELATIVE library path is tried in FULL first.
|
||||
# SoulSync's own library scanner stores paths like
|
||||
# "Asketa/Another Side/01 - Track.flac" (no leading slash) — index 0 is the
|
||||
# artist folder and dropping it (the old range(1, ...)) meant the artist
|
||||
# segment was never joined, so nothing under transfer/ ever resolved and
|
||||
# every track looked unreadable to the quality scanner.
|
||||
#
|
||||
# For ABSOLUTE media-server paths ("/music/Artist/Album/track.flac") index 0
|
||||
# is the empty leading segment and i=0 yields os.path.join(base, "", ...) ==
|
||||
# base/Artist/... which simply won't exist and harmlessly falls through to
|
||||
# i=1 ("music/...") etc. A Windows drive part ("E:") at i=0 likewise just
|
||||
# fails on POSIX and falls through. So starting at 0 is safe for every form
|
||||
# and only ADDS the relative-full-path match that was missing.
|
||||
# Skip index 0 to avoid drive-letter / leading-slash artifacts
|
||||
# (e.g. "E:" or "" from a leading "/").
|
||||
for base in base_dirs:
|
||||
for i in range(0, len(path_parts)):
|
||||
for i in range(1, len(path_parts)):
|
||||
candidate = os.path.join(base, *path_parts[i:])
|
||||
if os.path.exists(candidate):
|
||||
return candidate, attempt
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ from core.runtime_state import (
|
|||
download_tasks,
|
||||
tasks_lock,
|
||||
)
|
||||
from core.metadata.album_tracks import get_album_for_source
|
||||
from core.metadata.registry import (
|
||||
get_deezer_client,
|
||||
get_itunes_client,
|
||||
|
|
@ -27,27 +26,6 @@ from database.music_database import get_database
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _album_data_from_source(full: dict, album_id: str, fallback_name: str) -> dict:
|
||||
"""Build the redownload `album_data` from a primary-source get_album result (#915).
|
||||
|
||||
Mirrors the Spotify branch's album_data shape so iTunes/Deezer redownloads carry the
|
||||
real release_date / album_type / total_tracks — instead of a lean {'name': ...} that
|
||||
drops the $year and forces a manual reorganize afterwards."""
|
||||
images = full.get('images') or []
|
||||
image_url = full.get('image_url') or ''
|
||||
if not image_url and images and isinstance(images[0], dict):
|
||||
image_url = images[0].get('url', '')
|
||||
return {
|
||||
'id': str(full.get('id') or album_id),
|
||||
'name': full.get('name') or fallback_name,
|
||||
'release_date': full.get('release_date', ''),
|
||||
'album_type': full.get('album_type', 'album'),
|
||||
'total_tracks': full.get('total_tracks', 0),
|
||||
'images': images,
|
||||
'image_url': image_url,
|
||||
}
|
||||
|
||||
|
||||
def _get_itunes_client():
|
||||
"""Mirror of web_server._get_itunes_client — delegates to registry."""
|
||||
return get_itunes_client()
|
||||
|
|
@ -163,26 +141,12 @@ def redownload_start(track_id):
|
|||
'images': album_images,
|
||||
'image_url': album_images[0]['url'] if album_images else '',
|
||||
}
|
||||
elif meta_source in ('itunes', 'deezer'):
|
||||
# #915: parity with the Spotify branch + Reorganize — pull the full album from
|
||||
# the primary source so album_data carries release_date/album_type/total_tracks
|
||||
# (was lean {'name': ...}, which dropped the $year on iTunes/Deezer redownloads).
|
||||
if meta_source == 'itunes':
|
||||
track_number = full_track_details.get('trackNumber')
|
||||
disc_number = full_track_details.get('discNumber', 1)
|
||||
_alb_id = full_track_details.get('collectionId')
|
||||
else:
|
||||
track_number = full_track_details.get('track_position')
|
||||
disc_number = full_track_details.get('disk_number', 1)
|
||||
_alb_id = (full_track_details.get('album') or {}).get('id')
|
||||
if _alb_id:
|
||||
try:
|
||||
_full_album = get_album_for_source(meta_source, str(_alb_id))
|
||||
except Exception as _alb_err: # noqa: BLE001 — never let metadata break redownload
|
||||
logger.debug("[Redownload] %s album fetch failed: %s", meta_source, _alb_err)
|
||||
_full_album = None
|
||||
if isinstance(_full_album, dict):
|
||||
album_data = _album_data_from_source(_full_album, str(_alb_id), metadata.get('album', ''))
|
||||
elif meta_source == 'itunes':
|
||||
track_number = full_track_details.get('trackNumber')
|
||||
disc_number = full_track_details.get('discNumber', 1)
|
||||
elif meta_source == 'deezer':
|
||||
track_number = full_track_details.get('track_position')
|
||||
disc_number = full_track_details.get('disk_number', 1)
|
||||
|
||||
track_data = {
|
||||
'id': meta_id,
|
||||
|
|
|
|||
|
|
@ -1,53 +0,0 @@
|
|||
"""What counts as a *residual* file — a leftover with no value once the audio it
|
||||
accompanied is gone: OS junk, cover/scan images, and lyric/metadata sidecars.
|
||||
|
||||
Single source of truth shared by:
|
||||
* the **Reorganize** cleanup, which strips these from a source dir after every
|
||||
track has moved out (so the empty-dir pruner can take the folder), and
|
||||
* the **Empty Folder Cleaner** job, which can optionally treat a folder holding
|
||||
ONLY residual files as removable (#891).
|
||||
|
||||
Defining "disposable" in one place keeps the two features agreeing on what a "dead
|
||||
folder" is. Pure predicates — no filesystem access — so they're unit-tested in
|
||||
isolation. The whitelist is deliberately conservative: anything NOT recognized here
|
||||
(a booklet ``.pdf``, a video, a ``.txt`` note) is treated as real content and kept.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
# OS / tooling junk.
|
||||
JUNK_FILES = {'.ds_store', 'thumbs.db', 'desktop.ini', '.directory', 'album.nfo~'}
|
||||
# Cover art + booklet scans.
|
||||
IMAGE_EXTS = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.tiff', '.tif'}
|
||||
# Lyric / metadata / playlist sidecars that are worthless without their audio.
|
||||
SIDECAR_EXTS = {'.lrc', '.nfo', '.cue', '.m3u', '.m3u8'}
|
||||
|
||||
|
||||
def _ext(name: str) -> str:
|
||||
return os.path.splitext(name or '')[1].lower()
|
||||
|
||||
|
||||
def is_junk(name: str) -> bool:
|
||||
return (name or '').lower() in JUNK_FILES
|
||||
|
||||
|
||||
def is_image(name: str) -> bool:
|
||||
return _ext(name) in IMAGE_EXTS
|
||||
|
||||
|
||||
def is_sidecar(name: str) -> bool:
|
||||
return _ext(name) in SIDECAR_EXTS
|
||||
|
||||
|
||||
def is_disposable(name: str) -> bool:
|
||||
"""True if this file is junk, a cover/scan image, or a lyric/metadata sidecar —
|
||||
i.e. safe to delete from a folder that has no audio left."""
|
||||
return is_junk(name) or is_image(name) or is_sidecar(name)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'JUNK_FILES', 'IMAGE_EXTS', 'SIDECAR_EXTS',
|
||||
'is_junk', 'is_image', 'is_sidecar', 'is_disposable',
|
||||
]
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
"""Decision logic for the SoulSync standalone Deep Scan's untracked → Staging move.
|
||||
|
||||
The standalone deep scan (``_run_soulsync_deep_scan`` in web_server) walks the
|
||||
Transfer folder, diffs it against the ``soulsync`` rows in the DB, and relocates
|
||||
every file it can't find a DB record for into Staging for auto-import. That's fine
|
||||
when Transfer is a scratch/landing area — files arrive, get moved, imported, and
|
||||
recorded, so a later scan only ever sees a few genuinely-new arrivals.
|
||||
|
||||
It is a DATA-LOSS trap when the DB is empty or out of sync with disk (a volume
|
||||
swap, a DB reset, external tag edits) while Transfer holds the user's real library:
|
||||
a path-only diff then flags the *entire* library as "untracked" and the scan
|
||||
relocates all of it (issue #904). The same failure mode the orphan detector and the
|
||||
media-server deep scan already guard against (``core.library.stale_guard``) — this
|
||||
path just never used the guard.
|
||||
|
||||
This module is the pure, testable decision: given the Transfer file set, the DB's
|
||||
known paths, and the user's "Transfer is my permanent library" preference, decide
|
||||
WHICH files are untracked and WHETHER it's safe to relocate them. The web layer does
|
||||
only the I/O (walk/move/delete) based on the returned plan.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable, Set
|
||||
|
||||
from core.library.stale_guard import (
|
||||
DEFAULT_MAX_ORPHAN_FRACTION,
|
||||
DEFAULT_MIN_ORPHANS,
|
||||
is_implausible_orphan_flood,
|
||||
)
|
||||
|
||||
# Block reason codes (web layer turns these into a user-facing warning).
|
||||
BLOCK_NONE = ""
|
||||
BLOCK_TRANSFER_PERMANENT = "transfer_permanent"
|
||||
BLOCK_DESYNC = "desync"
|
||||
|
||||
|
||||
def _norm(path: str) -> str:
|
||||
"""Normalize a path for cross-platform comparison (Windows vs Unix separators)."""
|
||||
return str(path).replace("\\", "/")
|
||||
|
||||
|
||||
def diff_untracked(transfer_files: Iterable[str], db_paths: Iterable[str]) -> Set[str]:
|
||||
"""Files present in ``transfer_files`` but with no matching ``db_paths`` record.
|
||||
|
||||
Comparison is separator-normalized, so a DB path stored with one separator style
|
||||
still matches the on-disk path. Pure — no I/O. Returns the original (un-normalized)
|
||||
transfer paths so the caller can act on the real filesystem entries.
|
||||
"""
|
||||
db_norm = {_norm(p) for p in db_paths if p}
|
||||
return {f for f in transfer_files if _norm(f) not in db_norm}
|
||||
|
||||
|
||||
def plan_standalone_deep_scan(
|
||||
transfer_files: Iterable[str],
|
||||
db_paths: Iterable[str],
|
||||
*,
|
||||
never_move: bool = False,
|
||||
min_untracked: int = DEFAULT_MIN_ORPHANS,
|
||||
max_fraction: float = DEFAULT_MAX_ORPHAN_FRACTION,
|
||||
) -> dict:
|
||||
"""Plan the untracked → Staging move for a standalone deep scan. Pure — no I/O.
|
||||
|
||||
Returns a dict:
|
||||
* ``untracked`` (set[str]) — Transfer files with no DB record.
|
||||
* ``move_blocked`` (bool) — True when the untracked files must NOT be relocated.
|
||||
* ``block_reason`` (str) — ``BLOCK_TRANSFER_PERMANENT`` / ``BLOCK_DESYNC`` / "".
|
||||
|
||||
The move is blocked when either:
|
||||
* ``never_move`` is set (the user marked Transfer as their permanent library), or
|
||||
* the untracked share is implausibly large (> ``min_untracked`` files AND
|
||||
> ``max_fraction`` of the folder) — the empty/desynced-DB signature, where a
|
||||
path-only diff would relocate the whole library. Below that floor a normal
|
||||
batch of new arrivals still moves as before.
|
||||
|
||||
``move_blocked`` is only ever True when there ARE untracked files; an empty scan
|
||||
or a clean library returns ``move_blocked=False`` with no reason.
|
||||
"""
|
||||
transfer_set = set(transfer_files) # concrete (handles generators) + dedups
|
||||
untracked = diff_untracked(transfer_set, db_paths)
|
||||
total = len(transfer_set)
|
||||
n_untracked = len(untracked)
|
||||
|
||||
if n_untracked == 0:
|
||||
return {"untracked": untracked, "move_blocked": False, "block_reason": BLOCK_NONE}
|
||||
|
||||
if never_move:
|
||||
return {"untracked": untracked, "move_blocked": True, "block_reason": BLOCK_TRANSFER_PERMANENT}
|
||||
|
||||
if is_implausible_orphan_flood(
|
||||
n_untracked, total, min_orphans=min_untracked, max_fraction=max_fraction
|
||||
):
|
||||
return {"untracked": untracked, "move_blocked": True, "block_reason": BLOCK_DESYNC}
|
||||
|
||||
return {"untracked": untracked, "move_blocked": False, "block_reason": BLOCK_NONE}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"diff_untracked",
|
||||
"plan_standalone_deep_scan",
|
||||
"BLOCK_NONE",
|
||||
"BLOCK_TRANSFER_PERMANENT",
|
||||
"BLOCK_DESYNC",
|
||||
]
|
||||
|
|
@ -26,16 +26,14 @@ without a source ID are reported back to the caller and skipped
|
|||
entirely.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, List, Optional, Set, Tuple
|
||||
from typing import Any, Callable, Dict, List, Optional, Set
|
||||
|
||||
# Per-album track concurrency. Matches the download workers' per-batch
|
||||
# concurrency (3) so reorganize feels comparable to a fresh download.
|
||||
|
|
@ -101,7 +99,6 @@ _ALBUM_ID_COLUMNS = {
|
|||
'deezer': 'deezer_id',
|
||||
'discogs': 'discogs_id',
|
||||
'hydrabase': 'soul_id',
|
||||
'musicbrainz': 'musicbrainz_release_id',
|
||||
}
|
||||
|
||||
# Human-facing label for each source.
|
||||
|
|
@ -410,18 +407,6 @@ def _differentiators_in(norm_title: str) -> frozenset:
|
|||
return frozenset(t for t in norm_title.split() if t in _VERSION_DIFFERENTIATORS)
|
||||
|
||||
|
||||
# Featured-artist credit: "(feat. X)" / "[ft X]" / a trailing "feat. X". The
|
||||
# parenthesised form is stripped wherever it appears; the bare form only when
|
||||
# something follows it (so a song literally named "The Feat" is left alone, and
|
||||
# "Defeat"/"Lift" never trip the word-boundary). Case-insensitive.
|
||||
_FEAT_RE = re.compile(
|
||||
r"""\s*[\(\[]\s*(?:feat|ft|featuring)\b\.?[^)\]]*[\)\]] # (feat. X) / [ft. X]
|
||||
| \s+(?:feat|ft|featuring)\b\.?\s+\S.*$ # trailing feat. X ...
|
||||
""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_title(value) -> str:
|
||||
"""Lowercase + strip cosmetic punctuation and treat brackets / dashes
|
||||
/ slashes as word separators so the same track named slightly
|
||||
|
|
@ -433,17 +418,10 @@ def _normalize_title(value) -> str:
|
|||
- ``Don't Stop Believin'`` ↔ ``Don’t Stop Believin’``
|
||||
- ``Swimming Pools (Drank) - Extended Version``
|
||||
↔ ``Swimming Pools (Drank) (Extended Version)``
|
||||
- ``The Chase (feat. Big Artist)`` ↔ ``The Chase`` (#914)
|
||||
"""
|
||||
if value is None:
|
||||
return ''
|
||||
out = str(value).strip()
|
||||
# #914: drop featured-artist credits FIRST (while the parens are still here to
|
||||
# bound the group). iTunes appends "(feat. X)" to track titles while a user's
|
||||
# file is often just "The Chase" — the credit is metadata, not the song's
|
||||
# identity, and leaving it in dropped the match ratio below the threshold so
|
||||
# correctly-identified tracks reported as "not in the tracklist".
|
||||
out = _FEAT_RE.sub('', out).lower()
|
||||
out = str(value).strip().lower()
|
||||
# Strip characters that don't carry meaning across providers.
|
||||
for ch in ('"', "'", '‘', '’', '“', '”', '.', ',', '!', '?',
|
||||
'(', ')', '[', ']', '{', '}'):
|
||||
|
|
@ -1102,12 +1080,6 @@ def preview_album_reorganize(
|
|||
'track_number': track.get('track_number', 0),
|
||||
'current_path': _trim_to_transfer(db_path, resolved, transfer_dir),
|
||||
'new_path': '',
|
||||
# Absolute on-disk paths (additive). `current_path`/`new_path` above are
|
||||
# display-trimmed; these carry the real paths so the rename-only executor
|
||||
# acts on EXACTLY what the preview computed — no separate path logic that
|
||||
# could drift from what the user saw (#875).
|
||||
'current_path_abs': resolved or '',
|
||||
'new_path_abs': '',
|
||||
'file_exists': resolved is not None,
|
||||
'unchanged': False,
|
||||
'collision': False,
|
||||
|
|
@ -1155,7 +1127,6 @@ def preview_album_reorganize(
|
|||
new_full, _ok = build_final_path_fn(
|
||||
context, spotify_artist, album_info, file_ext, create_dirs=False
|
||||
)
|
||||
item['new_path_abs'] = new_full or ''
|
||||
item['new_path'] = (
|
||||
os.path.relpath(new_full, transfer_dir)
|
||||
if transfer_dir and new_full and new_full.startswith(transfer_dir)
|
||||
|
|
@ -1815,150 +1786,6 @@ def reorganize_album(
|
|||
return summary
|
||||
|
||||
|
||||
def _rename_track_in_place(current_abs: str, new_abs: str) -> Tuple[bool, Optional[str]]:
|
||||
"""Move ONE file from ``current_abs`` to ``new_abs`` in place — no copy, no re-tag,
|
||||
no post-processing. Creates the destination folder, carries sibling-format files
|
||||
(e.g. a lossy ``.opus`` alongside the ``.flac``) along with the renamed stem, and
|
||||
falls back to a cross-device move when the rename crosses a filesystem boundary.
|
||||
|
||||
Refuses to overwrite a DIFFERENT existing file at the destination (returns an error
|
||||
instead) — never silent data loss. Returns ``(ok, error_message)``.
|
||||
"""
|
||||
try:
|
||||
if current_abs and not os.path.exists(current_abs):
|
||||
return False, 'source file no longer on disk'
|
||||
same = os.path.normpath(current_abs) == os.path.normpath(new_abs)
|
||||
if os.path.exists(new_abs) and not same:
|
||||
return False, 'destination already exists'
|
||||
os.makedirs(os.path.dirname(new_abs), exist_ok=True)
|
||||
# Carry sibling-format audio to the same destination with the renamed stem —
|
||||
# mirrors _finalize_track so lossy-copy pairs don't get orphaned.
|
||||
for sibling_src in _find_sibling_audio_files(current_abs):
|
||||
_move_sibling_to_destination(sibling_src, new_abs)
|
||||
try:
|
||||
os.rename(current_abs, new_abs)
|
||||
except OSError as e:
|
||||
if getattr(e, 'errno', None) == errno.EXDEV:
|
||||
shutil.move(current_abs, new_abs) # crosses a filesystem boundary
|
||||
else:
|
||||
raise
|
||||
return True, None
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def reorganize_album_rename_only(
|
||||
*,
|
||||
album_id: str,
|
||||
db,
|
||||
transfer_dir: str,
|
||||
resolve_file_path_fn: Callable[[Optional[str]], Optional[str]],
|
||||
build_final_path_fn: Callable,
|
||||
update_track_path_fn: Optional[Callable[[object, str], None]] = None,
|
||||
cleanup_empty_dir_fn: Optional[Callable[[str], None]] = None,
|
||||
on_progress: Optional[Callable[[dict], None]] = None,
|
||||
primary_source: Optional[str] = None,
|
||||
strict_source: bool = False,
|
||||
metadata_source: str = 'api',
|
||||
stop_check: Optional[Callable[[], bool]] = None,
|
||||
preview_fn: Optional[Callable] = None,
|
||||
) -> dict:
|
||||
"""RENAME-ONLY reorganize (#875): move each track's file to the path the current
|
||||
naming scheme dictates, and nothing else — no copy-to-staging, no re-tag, no
|
||||
quality/AcoustID checks.
|
||||
|
||||
It acts on EXACTLY what :func:`preview_album_reorganize` computed (injected via
|
||||
``preview_fn`` for testability), so the apply can never disagree with what the user
|
||||
saw, and ONLY files whose path actually changes are touched — files marked
|
||||
``unchanged`` are skipped, which is what keeps a rename from rewriting the whole
|
||||
album (the #875 complaint). Tags and audio are left byte-for-byte alone.
|
||||
|
||||
Returns the same summary shape as :func:`reorganize_album`.
|
||||
"""
|
||||
preview_fn = preview_fn or preview_album_reorganize
|
||||
summary = {
|
||||
'status': 'completed', 'source': None, 'total': 0,
|
||||
'moved': 0, 'skipped': 0, 'failed': 0, 'errors': [],
|
||||
}
|
||||
|
||||
def _emit(**updates):
|
||||
if on_progress is None:
|
||||
return
|
||||
try:
|
||||
on_progress(updates)
|
||||
except Exception as e:
|
||||
logger.debug("[Reorganize/rename] progress emit failed: %s", e)
|
||||
|
||||
preview = preview_fn(
|
||||
album_id=album_id, db=db, transfer_dir=transfer_dir,
|
||||
resolve_file_path_fn=resolve_file_path_fn,
|
||||
build_final_path_fn=build_final_path_fn,
|
||||
primary_source=primary_source, strict_source=strict_source,
|
||||
metadata_source=metadata_source,
|
||||
)
|
||||
summary['source'] = preview.get('source')
|
||||
if not preview.get('success'):
|
||||
summary['status'] = preview.get('status', 'error')
|
||||
return summary
|
||||
|
||||
tracks = preview.get('tracks', [])
|
||||
summary['total'] = len(tracks)
|
||||
src_dirs_touched: Set[str] = set()
|
||||
|
||||
for t in tracks:
|
||||
if stop_check and stop_check():
|
||||
break
|
||||
title = t.get('title', 'Unknown')
|
||||
_emit(current_track=title)
|
||||
|
||||
# Skip anything that isn't a real, changing move. `unchanged` is the key one —
|
||||
# it's why a rename no longer rewrites files whose name didn't change.
|
||||
if (not t.get('matched') or t.get('unchanged')
|
||||
or t.get('collision') or not t.get('new_path_abs')):
|
||||
summary['skipped'] += 1
|
||||
_emit(skipped=summary['skipped'])
|
||||
continue
|
||||
|
||||
current_abs = t.get('current_path_abs')
|
||||
new_abs = t.get('new_path_abs')
|
||||
ok, err = _rename_track_in_place(current_abs, new_abs)
|
||||
if not ok:
|
||||
summary['failed'] += 1
|
||||
summary['errors'].append({
|
||||
'track_id': t.get('track_id'), 'title': title,
|
||||
'error': err or 'rename failed',
|
||||
})
|
||||
_emit(failed=summary['failed'], errors=list(summary['errors']))
|
||||
continue
|
||||
|
||||
# File is at its new home — update the DB directly (authoritative; no need to
|
||||
# round-trip through a server scan to learn what we just did). On DB failure the
|
||||
# file still moved; a library scan reconciles it, so we don't fail the track.
|
||||
if update_track_path_fn:
|
||||
try:
|
||||
update_track_path_fn(t.get('track_id'), new_abs)
|
||||
except Exception as db_err:
|
||||
logger.warning(
|
||||
"[Reorganize/rename] DB path update failed for %s: %s "
|
||||
"(file moved to %s; a scan will reconcile)",
|
||||
t.get('track_id'), db_err, new_abs,
|
||||
)
|
||||
if current_abs:
|
||||
src_dirs_touched.add(os.path.dirname(current_abs))
|
||||
summary['moved'] += 1
|
||||
_emit(moved=summary['moved'],
|
||||
processed=summary['moved'] + summary['skipped'] + summary['failed'])
|
||||
|
||||
if cleanup_empty_dir_fn:
|
||||
for src_dir in src_dirs_touched:
|
||||
try:
|
||||
cleanup_empty_dir_fn(src_dir)
|
||||
except Exception as e:
|
||||
logger.debug("[Reorganize/rename] cleanup of %s failed: %s", src_dir, e)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def _find_artist_dir(dest_path: str, transfer_dir: str) -> Optional[str]:
|
||||
"""Walk up from ``dest_path`` until the parent equals ``transfer_dir``;
|
||||
the directory at that point is the artist folder. Returns None if
|
||||
|
|
@ -2020,8 +1847,14 @@ def _prune_empty_album_dirs(artist_dir: str) -> None:
|
|||
# Sidecars that live alongside ONE audio file (same filename stem).
|
||||
_TRACK_SIDECAR_EXTS = ('.lrc', '.nfo', '.txt', '.cue', '.json')
|
||||
|
||||
# Album-level leftovers (cover images, .lrc, etc.) are classified by the shared
|
||||
# `core.library.residual_files.is_disposable` predicate — see `_delete_album_sidecars`.
|
||||
# Sidecars that live at the ALBUM level (one per directory).
|
||||
_ALBUM_SIDECARS = (
|
||||
'cover.jpg', 'cover.jpeg', 'cover.png',
|
||||
'folder.jpg', 'folder.png',
|
||||
'front.jpg', 'front.png',
|
||||
'album.jpg', 'album.png',
|
||||
'artwork.jpg', 'artwork.png',
|
||||
)
|
||||
|
||||
# Audio extensions used to decide whether a source directory still has
|
||||
# tracks the user might care about (i.e. a per-track failure left audio
|
||||
|
|
@ -2121,30 +1954,16 @@ def _delete_track_sidecars(audio_path: str) -> None:
|
|||
|
||||
|
||||
def _delete_album_sidecars(src_dir: str) -> None:
|
||||
"""Delete album-level *residual* files from ``src_dir`` — any cover/scan image,
|
||||
lyric/metadata sidecar (.lrc/.nfo/.cue/.m3u), or OS junk. Called during
|
||||
end-of-run cleanup ONLY when no audio remains in the directory, so everything
|
||||
here is leftover from the album that just moved (#891 — previously this only
|
||||
removed a fixed list of cover names, so ``back.jpg`` / ``disc.jpg`` / ``.webp``
|
||||
survived and kept the folder un-prunable).
|
||||
|
||||
Uses the shared ``is_disposable`` predicate so it agrees with the Empty Folder
|
||||
Cleaner on what's a dead leftover; anything unrecognized (a booklet ``.pdf``, a
|
||||
video) is deliberately LEFT. Best-effort — individual failures are debug-logged."""
|
||||
from core.library.residual_files import is_disposable
|
||||
try:
|
||||
entries = os.listdir(src_dir)
|
||||
except OSError:
|
||||
return
|
||||
for name in entries:
|
||||
if not is_disposable(name):
|
||||
continue
|
||||
full = os.path.join(src_dir, name)
|
||||
if os.path.isfile(full):
|
||||
"""Delete album-level sidecars (cover.jpg, folder.jpg, etc.) from
|
||||
`src_dir`. Used during end-of-run cleanup when no audio files remain
|
||||
in the directory. Best-effort — individual failures are debug-logged."""
|
||||
for name in _ALBUM_SIDECARS:
|
||||
sidecar = os.path.join(src_dir, name)
|
||||
if os.path.isfile(sidecar):
|
||||
try:
|
||||
os.remove(full)
|
||||
os.remove(sidecar)
|
||||
except OSError as e:
|
||||
logger.debug(f"[Reorganize] Couldn't remove residual file {full}: {e}")
|
||||
logger.debug(f"[Reorganize] Couldn't remove album sidecar {sidecar}: {e}")
|
||||
|
||||
|
||||
def _has_remaining_audio(directory: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -47,10 +47,7 @@ class LidarrDownloadClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
self.download_path = Path(download_path)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.active_downloads: Dict[str, Dict[str, Any]] = {}
|
||||
self._download_lock = threading.Lock()
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue