Compare commits
333 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3ab9301a18 | ||
|
|
4b8ddad9ff | ||
|
|
68d52a1b3f | ||
|
|
81bd85d639 | ||
|
|
f746fb4fc4 | ||
|
|
8f1ffa7632 | ||
|
|
a8520bc9b2 | ||
|
|
79648a4f5f | ||
|
|
9fc3628062 | ||
|
|
9b18e99419 | ||
|
|
b05b641521 | ||
|
|
efefdd64ff | ||
|
|
f3672c7ab4 | ||
|
|
6f451a34e1 | ||
|
|
633aa82b22 | ||
|
|
bd6db37624 | ||
|
|
8d549eb4fa | ||
|
|
1e68e339ca | ||
|
|
84d6208bc9 | ||
|
|
5d5aac9be3 | ||
|
|
60e7193539 | ||
|
|
9bb73c4cb4 | ||
|
|
050ce79d51 | ||
|
|
4743dfd644 | ||
|
|
91eaaeabb2 | ||
|
|
37c8b06a27 | ||
|
|
6e1a377f37 | ||
|
|
614deba0af | ||
|
|
465a77f01d | ||
|
|
0db72bf48d | ||
|
|
37c2b9b569 | ||
|
|
ede62824ad | ||
|
|
c593a17ac2 | ||
|
|
62aa2bef2d | ||
|
|
5df873655f | ||
|
|
92c101e300 | ||
|
|
ad715dddda | ||
|
|
34adb6fb32 | ||
|
|
79383df6d8 | ||
|
|
a62d2d4310 | ||
|
|
3207310448 | ||
|
|
6a388c43fc | ||
|
|
33fe92a525 | ||
|
|
512ec227be | ||
|
|
b07359cdb5 | ||
|
|
caee0fc3e2 | ||
|
|
969700674c | ||
|
|
8abf470018 | ||
|
|
551df0c3ca | ||
|
|
1382cb6117 | ||
|
|
8149f35fae | ||
|
|
1da677ee2d | ||
|
|
b43e44219a | ||
|
|
0276aa8764 | ||
|
|
60dee1b4d8 | ||
|
|
2fb142dded | ||
|
|
e2317de0a4 | ||
|
|
70dba77711 | ||
|
|
a1b0e787bc | ||
|
|
d4e2dccd73 | ||
|
|
d871899451 | ||
|
|
b72febbf1c | ||
|
|
8aa8fcb94a | ||
|
|
4af7600fd5 | ||
|
|
c96135ee60 | ||
|
|
b62d9b5b08 | ||
|
|
df0b4d3595 | ||
|
|
579617f861 | ||
|
|
bcf99d76d3 | ||
|
|
77e3673c9c | ||
|
|
d62afbb2df | ||
|
|
36986e4668 | ||
|
|
d72f6396f0 | ||
|
|
2b8f6f8611 | ||
|
|
b543db147a | ||
|
|
d30273985f | ||
|
|
4487a9d2dc | ||
|
|
d84fba14ba | ||
|
|
50c17ec9f7 | ||
|
|
88b68b8073 | ||
|
|
80d84c6de3 | ||
|
|
60b0022122 | ||
|
|
b683c7fb50 | ||
|
|
4c53a8f3a2 | ||
|
|
0be1952222 | ||
|
|
27ea2ee735 | ||
|
|
034a19b24f | ||
|
|
bd3c5860f6 | ||
|
|
116edeb477 | ||
|
|
319b6483a2 | ||
|
|
b85e9b40d9 | ||
|
|
b4583f23be | ||
|
|
f383a62c4f | ||
|
|
de9e78cf95 | ||
|
|
5a54ffe14a | ||
|
|
6802399805 | ||
|
|
240dce0c1b | ||
|
|
c6caaaa599 | ||
|
|
9faaf5c50c | ||
|
|
bce6a91aa2 | ||
|
|
28a539a840 | ||
|
|
027cf74d47 | ||
|
|
a42cf3b865 | ||
|
|
3a92571c71 | ||
|
|
866c8a6400 | ||
|
|
eddaea2f93 | ||
|
|
b1f061a2a8 | ||
|
|
7e2d2db08d | ||
|
|
e96d62432f | ||
|
|
3a95fc45a4 | ||
|
|
31637e0096 | ||
|
|
18f390b879 | ||
|
|
85d5904846 | ||
|
|
a6364ac283 | ||
|
|
8a6b02b3fd | ||
|
|
c8accf4e28 | ||
|
|
580d9eb0f5 | ||
|
|
92025f5fb3 | ||
|
|
c033656fdf | ||
|
|
30ff0bde49 | ||
|
|
086d153d77 | ||
|
|
3c33e31985 | ||
|
|
dc813d67c1 | ||
|
|
9847d6f0a9 | ||
|
|
602b035bad | ||
|
|
e3915b63e6 | ||
|
|
528aedbdfe | ||
|
|
e29cc641cb | ||
|
|
7a8b66fd2e | ||
|
|
2668980872 | ||
|
|
ab05508acc | ||
|
|
05b36704c3 | ||
|
|
88ff47e115 | ||
|
|
9cb5c4d40d | ||
|
|
9c91ba29bf | ||
|
|
71aa3397bf | ||
|
|
ad657f02a8 | ||
|
|
729a06c6d7 | ||
|
|
ed0a2079cf | ||
|
|
b4dde43b45 | ||
|
|
f010fbc487 | ||
|
|
79101e1847 | ||
|
|
5b7f99c30b | ||
|
|
d0966ec262 | ||
|
|
5cad2c02b0 | ||
|
|
e148f859e7 | ||
|
|
81a8b57dba | ||
|
|
df4ef99389 | ||
|
|
4d287c9699 | ||
|
|
551d12dba3 | ||
|
|
124e8bb21c | ||
|
|
774d0f6c65 | ||
|
|
49d3c77808 | ||
|
|
bac5da9177 | ||
|
|
8afbfbfeab | ||
|
|
606d1f951d | ||
|
|
17137cea5b | ||
|
|
2ae9ad3c6f | ||
|
|
d310b090e2 | ||
|
|
13bfdaeda2 | ||
|
|
8934c6418c | ||
|
|
5d5ed486c4 | ||
|
|
ecd2500c39 | ||
|
|
462a722cce | ||
|
|
0f2d6ddb63 | ||
|
|
4c47c01076 | ||
|
|
1005c7e306 | ||
|
|
ff12d8bbf2 | ||
|
|
94637cbe6f | ||
|
|
1f14fb4d5e | ||
|
|
ef0b68a973 | ||
|
|
e4ba27d8b3 | ||
|
|
bdbeff89d6 | ||
|
|
5f5bf4b24e | ||
|
|
83f21b248d | ||
|
|
8f66432592 | ||
|
|
1770cae04f | ||
|
|
842bc0ab34 | ||
|
|
55f7176c34 | ||
|
|
ec99d686cc | ||
|
|
d647fc8ad1 | ||
|
|
9d091207f6 | ||
|
|
03926bd6e2 | ||
|
|
c21031b9bc | ||
|
|
9ad5188610 | ||
|
|
d4e80fdaa0 | ||
|
|
2b17ed8451 | ||
|
|
cce7df4f3d | ||
|
|
600a744f7f | ||
|
|
2934903874 | ||
|
|
e32b4ec727 | ||
|
|
e8cc7ca2c8 | ||
|
|
16cb29c9ee | ||
|
|
15fa64248c | ||
|
|
10a1e1337f | ||
|
|
9d16abf952 | ||
|
|
fc2c38ad97 | ||
|
|
9f5bc0de89 | ||
|
|
e301877e64 | ||
|
|
65f73fae92 | ||
|
|
69cb51cc13 | ||
|
|
f2f4f8ccee | ||
|
|
b42ce3e0ca | ||
|
|
674b80972a | ||
|
|
d169818043 | ||
|
|
7186d24120 | ||
|
|
63374b32f1 | ||
|
|
dcd68e3631 | ||
|
|
cefddd73b7 | ||
|
|
ab8f82af2e | ||
|
|
5f1ec9ed7e | ||
|
|
c06ef6bb34 | ||
|
|
5ca88d96c1 | ||
|
|
d430dd8b71 | ||
|
|
42ff13d517 | ||
|
|
c6b5cd9763 | ||
|
|
fb9d88ea6a | ||
|
|
01c51a3c0e | ||
|
|
df6815c2cc | ||
|
|
b73389adea | ||
|
|
4b58d8079e | ||
|
|
b5b71df3fa | ||
|
|
9454970a83 | ||
|
|
49592f898c | ||
|
|
b95a8f539e | ||
|
|
89018bb6b3 | ||
|
|
203142c4a9 | ||
|
|
c11a742e58 | ||
|
|
1ad80d77a6 | ||
|
|
6e622d30f1 | ||
|
|
0df1d55ed6 | ||
|
|
41f5b29c27 | ||
|
|
ad0eda3575 | ||
|
|
2a7259b296 | ||
|
|
458658de86 | ||
|
|
9aaafaf341 | ||
|
|
15ea87a154 | ||
|
|
1c93a5640d | ||
|
|
3aaf19a357 | ||
|
|
d58da7a3db | ||
|
|
7159b3dc35 | ||
|
|
3496bb1800 | ||
|
|
1782e36c78 | ||
|
|
cf2b4d7d38 | ||
|
|
b6cb244cbd | ||
|
|
4bce1932ba | ||
|
|
1249160ff5 | ||
|
|
7bbd069147 | ||
|
|
75f3b0aa71 | ||
|
|
f9dfbd8fd5 | ||
|
|
bfc207f21a | ||
|
|
b761229a00 | ||
|
|
71cc6618b3 | ||
|
|
cc7d48b736 | ||
|
|
298d825757 | ||
|
|
213592821b | ||
|
|
8e218303be | ||
|
|
1ccc7b5e15 | ||
|
|
1367108e02 | ||
|
|
9a36d6f70b | ||
|
|
c4c112d17e | ||
|
|
f4c16ecc22 | ||
|
|
3e554f8274 | ||
|
|
08fb21fb13 | ||
|
|
dbd8278a14 | ||
|
|
70ea7eabf6 | ||
|
|
b04010a037 | ||
|
|
400b35d655 | ||
|
|
56da2e105c | ||
|
|
a5267ee8cf | ||
|
|
2ecbd8badc | ||
|
|
820ff20139 | ||
|
|
58363ae510 | ||
|
|
00b26fc5f1 | ||
|
|
b216233658 | ||
|
|
3b0394dbc6 | ||
|
|
7e175fec02 | ||
|
|
7948a90f09 | ||
|
|
6c0d79a84a | ||
|
|
4bdde1248e | ||
|
|
d15b3a185d | ||
|
|
48e86a1a58 | ||
|
|
46be97b195 | ||
|
|
93af95d865 | ||
|
|
e7814e0acf | ||
|
|
f2f0f5d849 | ||
|
|
02d6af29ed | ||
|
|
2905fe0853 | ||
|
|
b928f4df43 | ||
|
|
0f7e15363b | ||
|
|
526ed227c7 | ||
|
|
bb1a1222f8 | ||
|
|
1734a1b3c4 | ||
|
|
16673f4559 | ||
|
|
0e14ff03ee | ||
|
|
8bb749de9c | ||
|
|
973d28f61d | ||
|
|
5b3061ee2e | ||
|
|
4cb1937810 | ||
|
|
59858b033b | ||
|
|
bb632f6564 | ||
|
|
e82f3ab04f | ||
|
|
cf8f562e61 | ||
|
|
8a22b3f8ba | ||
|
|
ff061324ba | ||
|
|
9af31c3706 | ||
|
|
446465a833 | ||
|
|
501868d9c9 | ||
|
|
d2abec4a92 | ||
|
|
55f3dd427c | ||
|
|
949070ce73 | ||
|
|
f55a4fcf72 | ||
|
|
e594ac9799 | ||
|
|
d484046523 | ||
|
|
31d85e59c9 | ||
|
|
11d2fa9ad6 | ||
|
|
62d5821d26 | ||
|
|
975cf4cf3d | ||
|
|
b6be680d23 | ||
|
|
7421839da5 | ||
|
|
c32fe219fe | ||
|
|
fe78a3cdc3 | ||
|
|
6046a814cb | ||
|
|
d717f06afe | ||
|
|
95a7b51966 | ||
|
|
e0c55342bc | ||
|
|
71f7c4a5e1 | ||
|
|
310a5fe1bd | ||
|
|
12341f006b | ||
|
|
97242cbd8d | ||
|
|
afa07690f5 | ||
|
|
4cc2401332 | ||
|
|
2c91af0062 |
294 changed files with 30501 additions and 3501 deletions
4
.github/workflows/docker-publish.yml
vendored
4
.github/workflows/docker-publish.yml
vendored
|
|
@ -9,9 +9,9 @@ on:
|
|||
workflow_dispatch:
|
||||
inputs:
|
||||
version_tag:
|
||||
description: 'Version tag (e.g. 2.7.2)'
|
||||
description: 'Version tag (e.g. 2.8.2)'
|
||||
required: true
|
||||
default: '2.7.2'
|
||||
default: '2.8.2'
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
|
|
|
|||
10
.gitignore
vendored
10
.gitignore
vendored
|
|
@ -12,10 +12,12 @@ __pycache__/
|
|||
|
||||
# User-specific files (auto-created by the app if missing)
|
||||
config/config.json
|
||||
database/music_library.db
|
||||
database/music_library.db-shm
|
||||
database/music_library.db-wal
|
||||
database/music_library.db.backup_*
|
||||
config/youtube_cookies.txt
|
||||
# All app databases are live user data — never commit (music_library, video_library, …)
|
||||
database/*.db
|
||||
database/*.db-shm
|
||||
database/*.db-wal
|
||||
database/*.db.backup_*
|
||||
database/api_call_history.json
|
||||
storage/image_cache/
|
||||
logs/*.log
|
||||
|
|
|
|||
74
DISCOVER_BEST_IN_CLASS_PLAN.md
Normal file
74
DISCOVER_BEST_IN_CLASS_PLAN.md
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
# discover page — best in class plan (#913 + full generator audit)
|
||||
|
||||
morning notes. did the work overnight. tl;dr at top, details below, all of it `break nothing` + tested.
|
||||
|
||||
## what i shipped tonight (done, tested, safe)
|
||||
|
||||
### 1. listening recommendations (#913) — went from BROKEN to best-in-class
|
||||
|
||||
the feature was silently producing **zero** recs on real data. dug in and found three stacked bugs in the generation:
|
||||
|
||||
- **wrong id key (the killer).** `similar_artists.source_artist_id` is a *source* id (spotify/itunes/deezer), but the scanner built its id→name map from `artists.id` (the internal row id). so every edge resolved to nothing → 0 recs. proved it on your live db: internal-id join = 0 rows, spotify-id join = 71,636 rows.
|
||||
- **consensus could never fire.** it fed the ranker `get_top_similar_artists`, which does `GROUP BY similar_artist_name` + `MAX(source_artist_id)` — collapsing every similar artist down to a *single* seed. the whole point of the ranker is "artist X is similar to 3 of your seeds = strong signal," and that signal was being flattened away before it ever reached the ranker.
|
||||
- **similarity strength thrown away.** each edge stores a 1-10 closeness rank; it was ignored (everything weighted equally).
|
||||
|
||||
the fix (all in the pure, tested core + thin scan wiring):
|
||||
- build id→name from the **source-id columns**, query the **raw per-seed edges** (consensus preserved), and thread **similarity_rank** into the score so a seed's closest matches count for more.
|
||||
- **recency-weighted seeds**: `weight = lifetime_plays + 1.5 × recent_30d_plays`. picks now track what you're into *now*, not just all-time totals.
|
||||
|
||||
result on your actual library (simulated through the real code path): **40 recommendations, 13 with multi-seed consensus, all 40 with cached art.** top picks: Arcangel (Bad Bunny + Ozuna + J Balvin), Melanie Martinez (Ariana + Billie), Maluma, De La Ghetto — all coherent, all explainable.
|
||||
|
||||
### 2. its own row on the discover page
|
||||
|
||||
new row **"Based On Your Listening"** — play-weighted, consensus-ranked artist cards with a **"Because you listen to X, Y"** line. sits right above the library-driven "Recommended For You" row. purely additive: new endpoint `/api/discover/listening-recommendations`, new loader, hides itself when empty.
|
||||
|
||||
**you need to run one watchlist scan** for the row to populate (the data regenerates during the scan — i did NOT touch your live db). before that scan the row just stays hidden; after it, it fills in.
|
||||
|
||||
> note: this is deliberately different from the existing "Recommended For You" row. that one is driven by your *whole library / watchlist*. this one is driven by your *actual listening intensity* — the ~30 artists you really play, not the thousands you happen to own.
|
||||
|
||||
### 3. Fresh Tape "only 5-10 tracks" — fixed
|
||||
|
||||
root cause: `get_discovery_recent_albums` orders `release_date DESC`, so announced-but-unreleased albums sort to the *top* and ate the 50-album budget. the scanner skipped them *after* the budget was already spent → only a handful of released albums left → 5-10 tracks. fixed by fetching a generous budget (300) **and** excluding next-year albums at the query, so released albums fill the budget. the precise same-year `is_future_release` skip stays as a second guard. downstream caps (6/artist, top 75, take 50) unchanged.
|
||||
|
||||
**tests:** 25 pure-core cases (consensus/similarity/recency) + 2 Fresh Tape regression tests, all green. full discovery suite (255) green. nothing else touched.
|
||||
|
||||
---
|
||||
|
||||
## best-in-class roadmap for listening recs (next phases — your call)
|
||||
|
||||
these are the levers to take it further. ordered by value-to-risk. none are required; tonight's work stands on its own.
|
||||
|
||||
| phase | what | value | risk | notes |
|
||||
|---|---|---|---|---|
|
||||
| **3** | **playable track row** ✅ DONE | high | low-med | shipped: "🎧 Your Listening Mix" row — a track playlist (play/queue/download/sync) right under the artist row. stored as full render-ready dicts (not pool-hydrated, so it can't shrink on pool rotation like Fresh Tape does). |
|
||||
| **4** | **direct top-tracks fetch** ✅ DONE | high | med | shipped: scan fetches each recommended artist's top tracks (Spotify/Deezer), resolving the artist id by name-search when the similar-artist row lacks one — guarded by a strict name-match so it never pulls the wrong artist. bounded (top 20 recs), per-call guarded, fail-soft to the pool. iTunes has no top-tracks API → pool-only there. needs a live scan to populate. |
|
||||
| **5** | **genre-affinity boost** | med | low | we already compute your genre breakdown. boost recs whose genres match your top genres → tighter taste alignment. pure scoring add. |
|
||||
| **6** | **adventurousness dial** | med | low | the ranker already supports `min_seed_count` (consensus floor). expose it as a "Safe ↔ Adventurous" slider on the row. |
|
||||
| **7** | **diversity pass** | low-med | low | avoid 40 recs all orbiting your single heaviest seed — cap picks-per-seed so the row spans your taste. |
|
||||
|
||||
the core is built to absorb all of these without re-plumbing — `similarity_from_rank`, `build_recency_weighted_seeds`, and the scoring formula are all pure + tested.
|
||||
|
||||
---
|
||||
|
||||
## full discover-page generator audit (every soulsync-built row, excluding last.fm + listenbrainz)
|
||||
|
||||
how each one is generated today, and whether it can be elevated. "clear win" = safe + additive. "product call" = needs your decision (changes the row's character).
|
||||
|
||||
### curated (built during the scan, then hydrated)
|
||||
- **Fresh Tape / Release Radar** — new releases from watchlist+similar artists. **FIXED tonight** (see above). one more *clear win* available: hydration silently drops any curated id no longer in the discovery pool — could fall back to the stored `track_data_json` blob so the row can't shrink at read time.
|
||||
- **The Archives / Discovery Weekly** — strong already. nice 3-tier popularity split + serendipity scoring (boost never-played artists, penalize overplayed). same hydration-drop caveat as Fresh Tape; same cheap fallback fix.
|
||||
- **Seasonal Mix** — cleanest of the bunch. hydrates from a dedicated `seasonal_tracks` table (carries its own data), so it doesn't suffer the pool-drop problem. no bug.
|
||||
|
||||
### discovery-pool generators (live queries)
|
||||
- **Popular Picks** — ranks by popularity DESC. solid. only nit: on iTunes (no popularity scale) it silently degrades to random — indistinguishable from Shuffle there. UI-label thing at most.
|
||||
- **Hidden Gems** — *clear win*. currently `ORDER BY RANDOM()` over low-popularity tracks — so it's "random obscure," not "*best* obscure." a light ranking (popularity just under the threshold, or genre-affinity to you) would make it feel curated instead of arbitrary. (a deeper *product call*: add personalization like Archives has — bigger lift, changes its "pure underground" character.)
|
||||
- **Genre Playlists** — good. pushes the genre match into SQL. `RANDOM()` ordering is fine for a browse; a popularity/affinity tiebreak (*clear win*) would make thin genres feel less arbitrary.
|
||||
- **Discovery Shuffle** — random by design, correct. only possible add: exclude tracks already shown in other rows this refresh (needs a cross-section seen-set — medium plumbing).
|
||||
- **Time Machine (by decade)** — *clear win, low risk*: decades are hardcoded, so a modern-only library shows 7 decade tabs, 5 empty. filter the tabs to decades that actually have pool data.
|
||||
- **Daily Mix** — the weakest row. the "50% your library" half permanently returns nothing (library tracks have no source ids to play), so each Daily Mix is really just a relabeled Genre Playlist. real fix = backfill source ids into library rows (*schema-level, higher risk*) — worth a dedicated pass, not a quick tweak. also silently falls back to "top artists as pseudo-genres" when genre data is missing → "Daily Mix 1" becomes mislabeled artist-radio. gate/label that (*clear win*).
|
||||
|
||||
### cross-cutting
|
||||
- **hydration fragility** (Fresh Tape + Archives): both depend on curated ids still living in the pool at read time; misses are dropped silently. Seasonal already solved this with a dedicated table. giving the two spotify-style rows the same data-blob fallback is the single most robust cross-cutting fix. low risk, clear win.
|
||||
- **RANDOM-ordering pattern** (Hidden Gems, Shuffle, Genre, Decade): intentional for variety, but leaves quality signal on the table for the non-shuffle rows. adding a light ranking pass to Hidden Gems + Genre is the biggest "best-in-class" lever after tonight's work.
|
||||
|
||||
want me to take any of these? the Hidden Gems ranking + Time Machine empty-decade filter + the Fresh Tape/Archives hydration fallback are all safe, additive, same-shape-as-tonight wins i can knock out next.
|
||||
13
RELEASE_2.7.9_discord.md
Normal file
13
RELEASE_2.7.9_discord.md
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
**SoulSync 2.7.9** is out 🎉 a big one.
|
||||
|
||||
🎚️ **Best-quality downloads** — downloads now follow a ranked quality profile you drag to order (FLAC 24/192 → mp3). best-quality mode grabs the highest-quality copy across *every* source; priority mode gets an opt-in rank-based order toggle. quarantine is folded into the Downloads page + safer imports (AcoustID fail-closed, silence/truncation guards).
|
||||
|
||||
🎧 **Discover got smart** — "Based On Your Listening" ranks artists from who you *actually play*, and "Your Listening Mix" is a playable track playlist of their top tracks (works on any source). Fresh Tape fills properly now.
|
||||
|
||||
⚡ **Wing It Pool** — a new spot next to Discovery Pool to review + re-match the tracks Wing It guessed at (they used to be invisible).
|
||||
|
||||
🔁 **Auto-Sync redesign** — the scheduling board is now clean horizontal lanes instead of a side-scrolling column wall.
|
||||
|
||||
🐛 **Fixes** — multi-disc albums no longer show disc-2 as "missing" (#927), playlists no longer stuck on "Never Synced" (#925), and tracks can't import while quarantined (#928). thanks @ramonskie + @nick2000713 🙏
|
||||
|
||||
⚠️ **re-scan your library once** so the multi-disc fix can backfill disc numbers on existing tracks. enjoy! 🎶
|
||||
13
RELEASE_2.8.0_discord.md
Normal file
13
RELEASE_2.8.0_discord.md
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
**SoulSync 2.8.0** is out 🎉 a quality + reliability release.
|
||||
|
||||
🧹 **The Unverified queue, finally under control** — if you saw thousands of "unverified" rows piling up, this is for you. the AcoustID scan stops duplicating history rows, a one-time reconcile on startup clears the existing backlog from your library (no re-scan), and a new 🧹 *Clean orphaned* button sweeps dead rows whose file is gone. (#934 — thanks @nick2000713 for #938)
|
||||
|
||||
✂️ **Preview Clip Cleanup** — a new Tools job that finds the ~30s preview clips the HiFi source sometimes hands back instead of the full song, then deletes them and re-wishlists the real version. each finding has a ▶ Play button so you can confirm before approving.
|
||||
|
||||
💿 **Album Completeness handles split albums** — an album split across multiple library rows no longer shows every fragment as falsely "incomplete"; it groups the validated fragments into one correct finding. (#936 — thanks @ragnarlotus)
|
||||
|
||||
🐛 **Fixes** — pasted YouTube cookies no longer throw `unsupported browser: "custom"` on Docker (thanks HellRa1SeR); longer remasters aren't quarantined as "truncated" anymore (#937, thanks @diegocade1); "Add to Wishlist" from a discography went from ~15–30s *per track* to instant; wishlist art renders for re-downloads; and **Clear Completed** is back on the Downloads page.
|
||||
|
||||
⚡ **Performance** — trimmed the dashboard GPU usage that was hammering Firefox/Zen (and Background Particles are OFF by default now), plus bounded the runaway memory growth that could lock the app up on big libraries. (#935 / #802)
|
||||
|
||||
enjoy! 🎶
|
||||
15
RELEASE_2.8.1_discord.md
Normal file
15
RELEASE_2.8.1_discord.md
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
**SoulSync 2.8.1** is out 🎉 a feature + reliability release.
|
||||
|
||||
🎧 **Export playlists to Spotify & Deezer** — the mirrored-playlist export now has **Sync to Spotify** and **Sync to Deezer** next to the ListenBrainz / JSPF options. it builds a playlist in your account from the IDs soulsync already has (the discovery cache first, then your library), so an already-discovered playlist exports **instantly with zero API calls**. re-exporting updates the same playlist instead of duplicating it, and an optional *"match missing tracks"* toggle confidently searches for the stragglers — a wrong-artist or karaoke version is left out, never guessed. the first Spotify export asks permission once. (#945)
|
||||
|
||||
🏷️ **Library Reorganize — Rename only** — a lighter action that just **renames your files** to your naming scheme: no re-tag, no quality/AcoustID re-check, no copy-to-staging. much faster on a NAS, and only touches files whose path actually changes. pick it from the new Action dropdown. (#875 — thanks @tsoulard / @Tacobell444)
|
||||
|
||||
💿 **Broader lossless handling** — lossy-copy now covers **all lossless formats**, not just FLAC (#941); and **DSD** (`.dsf`/`.dff`) is recognized as lossless instead of false-flagged "truncated" (#939).
|
||||
|
||||
🐛 **Download + search fixes** — an unbalanced bracket no longer false-fails as "file not found"; a file we couldn't quarantine is left for retry instead of deleted; "file not found" errors are actionable now; pasted Qobuz/Tidal links inject the exact track into manual search (#932); the Wing It pool "Fix Match" works again.
|
||||
|
||||
⚡ **Reduce visual effects, refined** — it no longer freezes functional motion (spinners, progress), only the expensive GPU stuff (blur, shadows, glow). worker orbs default OFF on Firefox and run at ~30fps under reduce-effects. plus a jellyfin scan watchdog fix for big libraries.
|
||||
|
||||
🔧 **Under the hood** — settings cleanup (#943, @nick2000713), spotify oauth hardening (#942) + npm security fixes (#944, HellRa1SeR).
|
||||
|
||||
enjoy! 🎶
|
||||
9
RELEASE_2.8.2_discord.md
Normal file
9
RELEASE_2.8.2_discord.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
**SoulSync 2.8.2** is out 🎉 a stability + performance release.
|
||||
|
||||
🎧 **Spotify, reliably** — the Docker boot hang is fixed: with Spotify as your primary source, an unreachable Spotify API could block startup so the container bound `:8008` but never served the UI. auth probes are now deferred during boot + capped with a timeout. the "re-auth didn't stick" bug is fixed too (the OAuth callback and the app were reading different token caches), and **Sync to Spotify** now works — it asks for playlist-write permission once, on-demand, leaving your normal login untouched. (#949 — thanks HellRa1SeR)
|
||||
|
||||
⚡ **The "slow after update" fix** — the post-update lag wasn't SoulSync, it was browser password managers (Bitwarden/1Password/etc.) rebuilding their autofill overlay on *every* DOM change. non-credential fields are now marked so they skip them — **~110× less main-thread blocking** in the reporter's benchmark. plus a new **Max Performance** mode (Settings → Appearance) that kills every effect for no-GPU / Docker setups. (#948 — thanks @nick2000713)
|
||||
|
||||
📥 **Large-library imports no longer time out** — dropping a whole library into staging used to make the import page scan every file synchronously and never load. the scan runs in the background now with a live "Scanning N of M…" progress, and fills in when done. (#947 — thanks @ramonskie)
|
||||
|
||||
enjoy! 🎶
|
||||
|
|
@ -44,7 +44,8 @@
|
|||
},
|
||||
"metadata_enhancement": {
|
||||
"enabled": true,
|
||||
"embed_album_art": true
|
||||
"embed_album_art": true,
|
||||
"single_to_album": false
|
||||
},
|
||||
"file_organization": {
|
||||
"enabled": true,
|
||||
|
|
|
|||
|
|
@ -662,7 +662,12 @@ class ConfigManager:
|
|||
# source whose art is smaller is skipped so the next source is
|
||||
# tried — stops a low-res Cover Art Archive upload from winning.
|
||||
# 0 disables the size gate.
|
||||
"min_art_size": 1000
|
||||
"min_art_size": 1000,
|
||||
# When a track matches a SINGLE release, look up the parent ALBUM
|
||||
# that contains it and tag it as that album, so it groups with its
|
||||
# album-mates and gets the album cover (not the single's). Off by
|
||||
# default — it's an extra per-import metadata lookup.
|
||||
"single_to_album": False
|
||||
},
|
||||
"musicbrainz": {
|
||||
"embed_tags": True
|
||||
|
|
@ -700,6 +705,12 @@ class ConfigManager:
|
|||
},
|
||||
"import": {
|
||||
"staging_path": "./Staging",
|
||||
# Master toggle for quality-filtering on import. On by default:
|
||||
# downloaded files that don't meet the quality profile are
|
||||
# quarantined instead of imported (same gate the download
|
||||
# pipeline uses). Off → import everything regardless of quality;
|
||||
# the library Quality Upgrade Scanner still flags them.
|
||||
"quality_filter_enabled": True,
|
||||
"replace_lower_quality": False,
|
||||
# Use the top Staging folder as the artist (Artist/Album layouts,
|
||||
# mixtapes). On by default to preserve the long-standing import
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ from config.settings import config_manager
|
|||
from core.amazon_client import AmazonClient, AmazonClientError
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
from core.quality.model import AudioQuality
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("amazon_download_client")
|
||||
|
|
@ -76,9 +78,12 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get("soulseek.download_path", "./downloads")
|
||||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
|
||||
self._quality = config_manager.get("amazon_download.quality", "flac")
|
||||
self._quality = quality_tier_for_source("amazon", default="flac")
|
||||
self._allow_fallback = config_manager.get("amazon_download.allow_fallback", True)
|
||||
|
||||
self._client = AmazonClient(preferred_codec=self._quality)
|
||||
|
|
@ -133,11 +138,17 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
album_map: Dict[str, AlbumResult] = {}
|
||||
album_order: List[str] = []
|
||||
preferred = self._client.preferred_codec
|
||||
# Search results only carry the codec (real sample_rate arrives at
|
||||
# stream time). Claim the format honestly — FLAC for the lossless
|
||||
# codec, lossy otherwise — so audio_quality derives a real format
|
||||
# instead of the display label ("Lossless"), and the post-download
|
||||
# probe pins the actual sample_rate/bit_depth.
|
||||
amazon_q = AudioQuality(format='flac' if _codec_key(preferred) == 'flac' else 'aac')
|
||||
|
||||
for item in items:
|
||||
quality = _quality_label(preferred)
|
||||
if item.is_track:
|
||||
track_results.append(TrackResult(
|
||||
tr = TrackResult(
|
||||
username="amazon",
|
||||
filename=f"{item.asin}||{item.artist_name} - {item.title}",
|
||||
size=0,
|
||||
|
|
@ -155,7 +166,9 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
"album_asin": item.album_asin,
|
||||
"isrc": item.isrc,
|
||||
},
|
||||
))
|
||||
)
|
||||
tr.set_quality(amazon_q)
|
||||
track_results.append(tr)
|
||||
elif item.is_album:
|
||||
album_asin = item.album_asin or item.asin
|
||||
if album_asin not in album_map:
|
||||
|
|
@ -173,6 +186,7 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
title=item.title,
|
||||
album=item.album_name,
|
||||
)
|
||||
placeholder.set_quality(amazon_q)
|
||||
album_map[album_asin] = AlbumResult(
|
||||
username="amazon",
|
||||
album_path=album_asin,
|
||||
|
|
|
|||
|
|
@ -660,8 +660,13 @@ class AutoImportWorker:
|
|||
auto_process = self._config_manager.get('auto_import.auto_process', True)
|
||||
|
||||
try:
|
||||
# Phase 3: Identify
|
||||
identification = self._identify_folder(candidate)
|
||||
# Phase 3: Identify.
|
||||
# Re-identify (#889): if the user designated this exact file's release in
|
||||
# the Re-identify modal, a hint short-circuits the guessing — we match
|
||||
# straight against the chosen album. No hint → byte-identical to before.
|
||||
rematch_hint, identification = self._resolve_rematch_hint(candidate)
|
||||
if identification is None:
|
||||
identification = self._identify_folder(candidate)
|
||||
if not identification:
|
||||
self._record_result(candidate, 'needs_identification', 0.0,
|
||||
error_message='Could not identify album from tags, folder name, or fingerprint')
|
||||
|
|
@ -690,7 +695,10 @@ class AutoImportWorker:
|
|||
high_conf_matches = [m for m in match_result.get('matches', []) if m['confidence'] >= 0.8]
|
||||
has_strong_individual_matches = len(high_conf_matches) > 0
|
||||
|
||||
if (confidence >= threshold or has_strong_individual_matches) and auto_process:
|
||||
# A re-identify is an explicit user choice — let it auto-process like a
|
||||
# strong match (still gated on the global auto_process preference).
|
||||
if (confidence >= threshold or has_strong_individual_matches
|
||||
or rematch_hint is not None) and auto_process:
|
||||
# Phase 5: Auto-process — insert an in-progress row
|
||||
# so the UI sees the import the moment it starts,
|
||||
# then update it with the final status when done.
|
||||
|
|
@ -709,6 +717,13 @@ class AutoImportWorker:
|
|||
confidence = max(confidence, effective_conf)
|
||||
if success:
|
||||
self._bump_stat('auto_processed')
|
||||
# Re-identify (#889): only NOW that the new home exists do we
|
||||
# consume the hint and (if replace was chosen) delete the old
|
||||
# row + file — so a failed import never loses the original. Pass
|
||||
# the landing paths so we never delete a file the re-import landed
|
||||
# at the SAME place (picking the release it's already in).
|
||||
if rematch_hint is not None:
|
||||
self._finalize_rematch_hint(rematch_hint, getattr(candidate, '_reid_final_paths', None))
|
||||
else:
|
||||
self._bump_stat('failed')
|
||||
|
||||
|
|
@ -1003,6 +1018,75 @@ class AutoImportWorker:
|
|||
except Exception:
|
||||
return False
|
||||
|
||||
# ── Re-identify hints (#889) ──
|
||||
|
||||
def _resolve_rematch_hint(self, candidate: 'FolderCandidate'):
|
||||
"""If this staged file carries a user-designated re-identify hint, return
|
||||
``(hint, identification)`` so matching skips the guessing tiers; otherwise
|
||||
``(None, None)`` and the caller falls back to normal identification.
|
||||
|
||||
Fail-safe: ANY error (no table, DB hiccup) returns ``(None, None)`` so a
|
||||
re-identify problem can never break ordinary auto-import. Only single-file
|
||||
candidates are eligible — a re-identify always stages exactly one track."""
|
||||
try:
|
||||
files = candidate.audio_files or []
|
||||
if len(files) != 1:
|
||||
return None, None
|
||||
from core.imports.rematch_hints import (
|
||||
build_identification_from_hint,
|
||||
find_hint_for_file,
|
||||
quick_file_signature,
|
||||
)
|
||||
file_path = files[0]
|
||||
sig = quick_file_signature(file_path)
|
||||
conn = self.database._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
hint = find_hint_for_file(cursor, file_path, sig)
|
||||
finally:
|
||||
conn.close()
|
||||
if hint is None:
|
||||
return None, None
|
||||
logger.info("[Auto-Import] Re-identify hint for %s → %s '%s' (%s)",
|
||||
candidate.name, hint.album_type or 'release',
|
||||
hint.album_name or '?', hint.source)
|
||||
return hint, build_identification_from_hint(hint)
|
||||
except Exception as e:
|
||||
logger.debug("[Auto-Import] rematch-hint lookup skipped: %s", e)
|
||||
return None, None
|
||||
|
||||
def _finalize_rematch_hint(self, hint, new_paths=None) -> None:
|
||||
"""Post-success: delete the replaced library row + file (if the user chose
|
||||
replace) and consume the hint so it's single-use. ``new_paths`` are where the
|
||||
re-import landed — passed through so the same-home guard never deletes a file
|
||||
the import wrote at the old location. Best-effort — a cleanup failure is
|
||||
logged, never raised, since the re-import already succeeded."""
|
||||
try:
|
||||
from core.imports.rematch_hints import consume_hint, delete_replaced_track
|
||||
|
||||
def _resolve_old(stored):
|
||||
# The old row's path is a STORED path (Docker/media-server view) — map
|
||||
# it to a file this process can actually unlink, same as everywhere else.
|
||||
try:
|
||||
from core.library.path_resolver import resolve_library_file_path
|
||||
return resolve_library_file_path(stored, config_manager=getattr(self, '_config_manager', None))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
conn = self.database._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
removed = delete_replaced_track(cursor, hint.replace_track_id,
|
||||
resolve_fn=_resolve_old, new_paths=new_paths)
|
||||
consume_hint(cursor, hint.id)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
if removed:
|
||||
logger.info("[Auto-Import] Re-identify replaced old track — removed %s", removed)
|
||||
except Exception as e:
|
||||
logger.warning("[Auto-Import] rematch-hint finalize failed (import still OK): %s", e)
|
||||
|
||||
# ── Identification ──
|
||||
|
||||
def _identify_folder(self, candidate: FolderCandidate) -> Optional[Dict]:
|
||||
|
|
@ -1431,8 +1515,11 @@ class AutoImportWorker:
|
|||
|
||||
def _match_tracks(self, candidate: FolderCandidate, identification: Dict) -> Optional[Dict]:
|
||||
"""Match staging files to the identified album's tracklist."""
|
||||
# Singles: no album tracklist to match against — the file IS the match
|
||||
if candidate.is_single or identification.get('is_single'):
|
||||
# Singles: no album tracklist to match against — the file IS the match.
|
||||
# force_album_match (set by a re-identify hint) overrides this: even a lone
|
||||
# staged file is matched INTO the chosen album, so it inherits the album's
|
||||
# year / track number / art instead of the bare singles stub (#889).
|
||||
if not identification.get('force_album_match') and (candidate.is_single or identification.get('is_single')):
|
||||
conf = identification.get('identification_confidence', 0.7)
|
||||
track_data = {
|
||||
'name': identification.get('track_name', identification.get('album_name', '')),
|
||||
|
|
@ -1600,6 +1687,7 @@ class AutoImportWorker:
|
|||
|
||||
processed = 0
|
||||
errors = []
|
||||
reid_final_paths = [] # #889: where the pipeline landed each file (same-home guard)
|
||||
all_matches = list(match_result.get('matches', []))
|
||||
|
||||
# Album total duration — sum of every matched track's duration.
|
||||
|
|
@ -1780,6 +1868,11 @@ class AutoImportWorker:
|
|||
|
||||
self._process_callback(context_key, context, file_path)
|
||||
processed += 1
|
||||
# Capture where the pipeline actually landed the file (#889 same-home
|
||||
# guard) — the pipeline writes it back into the mutable context.
|
||||
_landed = context.get('_final_processed_path')
|
||||
if _landed:
|
||||
reid_final_paths.append(_landed)
|
||||
logger.info(f"[Auto-Import] Processed: {track_number}. {track_name}")
|
||||
|
||||
except Exception as e:
|
||||
|
|
@ -1803,6 +1896,13 @@ class AutoImportWorker:
|
|||
except Exception as e:
|
||||
logger.debug("automation emit failed: %s", e)
|
||||
|
||||
# Stash landing paths on the candidate so _finalize_rematch_hint can avoid
|
||||
# deleting a file the re-import landed at the SAME place (#889).
|
||||
try:
|
||||
candidate._reid_final_paths = reid_final_paths
|
||||
except Exception as e:
|
||||
logger.debug("could not stash reid final paths: %s", e)
|
||||
|
||||
return processed > 0
|
||||
|
||||
# ── Database ──
|
||||
|
|
|
|||
27
core/boot_phase.py
Normal file
27
core/boot_phase.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
"""Boot-phase guard for non-blocking container startup.
|
||||
|
||||
While the gunicorn worker is importing ``web_server`` (module-level client and
|
||||
worker initialization), external provider API probes must not block startup.
|
||||
Network validation is deferred until ``mark_boot_complete()`` runs at the end
|
||||
of that import pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
_boot_lock = threading.Lock()
|
||||
_boot_active = True
|
||||
|
||||
|
||||
def is_boot_phase() -> bool:
|
||||
"""Return True while module import must avoid blocking provider API calls."""
|
||||
with _boot_lock:
|
||||
return _boot_active
|
||||
|
||||
|
||||
def mark_boot_complete() -> None:
|
||||
"""End the boot phase — provider clients may perform network probes again."""
|
||||
global _boot_active
|
||||
with _boot_lock:
|
||||
_boot_active = False
|
||||
|
|
@ -117,6 +117,48 @@ def _is_full_track_payload(payload: Optional[Dict[str, Any]]) -> bool:
|
|||
return 'track_position' in payload and 'contributors' in payload
|
||||
|
||||
|
||||
def resolve_album_track_positions(session, base_url, album_ids, cache=None, sleep_s=0.2):
|
||||
"""Build ``{str(track_id): track_position}`` for a set of Deezer album ids.
|
||||
|
||||
Deezer PLAYLIST and SEARCH track objects (and even the album object's embedded
|
||||
``tracks.data``) omit ``track_position`` — only ``/album/<id>/tracks`` and
|
||||
``/track/<id>`` carry it. So numbering playlist tracks by their playlist index
|
||||
silently poisons the real album track number, which then rides onto the
|
||||
downloaded file's tag. This resolves the authoritative position per album
|
||||
(cache-first, best-effort — a failed album just isn't in the map)."""
|
||||
import time as _time
|
||||
positions: Dict[str, int] = {}
|
||||
for aid in album_ids:
|
||||
aid = str(aid)
|
||||
at_list = None
|
||||
if cache:
|
||||
try:
|
||||
ct = cache.get_entity('deezer', 'album_tracks', aid)
|
||||
if ct and ct.get('data'):
|
||||
at_list = ct['data']
|
||||
except Exception: # noqa: BLE001 - cache is best-effort
|
||||
at_list = None
|
||||
if at_list is None:
|
||||
try:
|
||||
if sleep_s:
|
||||
_time.sleep(sleep_s) # respect Deezer rate limits
|
||||
r = session.get(f"{base_url}/album/{aid}/tracks", params={'limit': 500}, timeout=10)
|
||||
if getattr(r, 'ok', False):
|
||||
at_list = (r.json() or {}).get('data', [])
|
||||
if cache and at_list is not None:
|
||||
try:
|
||||
cache.store_entity('deezer', 'album_tracks', aid, {'data': at_list})
|
||||
except Exception as _cache_err: # noqa: BLE001
|
||||
logger.debug("album_tracks cache store failed for %s: %s", aid, _cache_err)
|
||||
except Exception: # noqa: BLE001 - never let metadata resolution break the fetch
|
||||
at_list = None
|
||||
for at in (at_list or []):
|
||||
tp = at.get('track_position')
|
||||
if at.get('id') and tp:
|
||||
positions[str(at['id'])] = tp
|
||||
return positions
|
||||
|
||||
|
||||
# ==================== Dataclasses (match iTunesClient / SpotifyClient format) ====================
|
||||
|
||||
@dataclass
|
||||
|
|
@ -1357,6 +1399,16 @@ class DeezerClient:
|
|||
|
||||
raw_tracks.extend(page_tracks)
|
||||
|
||||
# Real album track positions — playlist tracks don't carry track_position,
|
||||
# so numbering by playlist index would poison the downloaded file's tag.
|
||||
album_ids = {str(t.get('album', {}).get('id')) for t in raw_tracks if t.get('album', {}).get('id')}
|
||||
try:
|
||||
from core.metadata.cache import get_metadata_cache
|
||||
_cache = get_metadata_cache()
|
||||
except Exception:
|
||||
_cache = None
|
||||
track_positions = resolve_album_track_positions(self.session, self.BASE_URL, album_ids, _cache)
|
||||
|
||||
# Normalize tracks
|
||||
tracks: List[Dict[str, Any]] = []
|
||||
for i, t in enumerate(raw_tracks, start=1):
|
||||
|
|
@ -1368,7 +1420,8 @@ class DeezerClient:
|
|||
'artists': [artist_name],
|
||||
'album': t.get('album', {}).get('title', ''),
|
||||
'duration_ms': t.get('duration', 0) * 1000,
|
||||
'track_number': i,
|
||||
# REAL album position; the playlist index is a last resort only.
|
||||
'track_number': track_positions.get(str(t.get('id'))) or i,
|
||||
})
|
||||
|
||||
result = {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
|||
import requests
|
||||
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
from core.quality.source_map import quality_from_deezer, quality_tier_for_source
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("deezer_download")
|
||||
|
|
@ -92,7 +93,10 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
|
||||
# Engine reference is populated by set_engine() at registration
|
||||
# time. None until orchestrator wires the registry.
|
||||
|
|
@ -117,14 +121,20 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
self._license_token = None
|
||||
self._user_data = None
|
||||
self._authenticated = False
|
||||
self._pending_arl: Optional[str] = None
|
||||
|
||||
# Quality preference
|
||||
self._quality = config_manager.get('deezer_download.quality', 'flac')
|
||||
self._quality = quality_tier_for_source('deezer', default='flac')
|
||||
|
||||
# Try to authenticate on init if ARL is configured
|
||||
arl = config_manager.get('deezer_download.arl', '')
|
||||
if arl:
|
||||
self._authenticate(arl)
|
||||
from core.boot_phase import is_boot_phase
|
||||
if is_boot_phase():
|
||||
self._pending_arl = arl
|
||||
logger.debug("Deezer ARL present — authentication deferred until after boot")
|
||||
else:
|
||||
self._authenticate(arl)
|
||||
|
||||
logger.info(f"Deezer download client initialized (download path: {self.download_path})")
|
||||
|
||||
|
|
@ -223,12 +233,66 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
return self._authenticated
|
||||
|
||||
def is_authenticated(self) -> bool:
|
||||
if self._pending_arl and not self._authenticated:
|
||||
from core.boot_phase import is_boot_phase
|
||||
if not is_boot_phase():
|
||||
self._authenticate(self._pending_arl)
|
||||
self._pending_arl = None
|
||||
return self._authenticated
|
||||
|
||||
async def check_connection(self) -> bool:
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(None, self.is_available)
|
||||
|
||||
# ─── Playlist export (#945) ──────────────────────────────────
|
||||
#
|
||||
# UNOFFICIAL: rides the private gw-light gateway with the ARL session already used
|
||||
# for downloads. Deezer shut their public developer API, so this is the only write
|
||||
# path — and it's fragile by nature (breaks when Deezer changes internals).
|
||||
|
||||
def create_or_update_playlist(self, name, track_ids, *, existing_id=None,
|
||||
public=False, description=""):
|
||||
"""Create a Deezer playlist (or append to an existing one) from a mirrored
|
||||
playlist's tracks. ``track_ids`` are stored ``deezer_id`` values per library track.
|
||||
``existing_id`` set → add to that playlist (idempotent re-export reuses the stored
|
||||
target); unset → create a new one. Returns
|
||||
``{success, playlist_id, url, added, error}``."""
|
||||
if not self._authenticated:
|
||||
return {"success": False, "error": "Deezer is not connected (ARL)"}
|
||||
song_ids = [str(t) for t in (track_ids or []) if t]
|
||||
if not song_ids:
|
||||
return {"success": False, "error": "No matching Deezer tracks to export"}
|
||||
try:
|
||||
songs = [[sid, i] for i, sid in enumerate(song_ids)]
|
||||
if existing_id:
|
||||
res = self._gw_call("playlist.addSongs",
|
||||
{"playlist_id": int(existing_id), "songs": songs})
|
||||
if res is None:
|
||||
return {"success": False, "error": "Deezer rejected the playlist update"}
|
||||
playlist_id = existing_id
|
||||
else:
|
||||
res = self._gw_call("playlist.create", {
|
||||
"title": name, "description": description,
|
||||
"is_public": bool(public), "songs": songs,
|
||||
})
|
||||
if res is None:
|
||||
return {"success": False, "error": "Deezer rejected the playlist create"}
|
||||
# gw 'playlist.create' returns the new playlist id (int) as `results`.
|
||||
if isinstance(res, dict):
|
||||
playlist_id = res.get("PLAYLIST_ID") or res.get("id")
|
||||
else:
|
||||
playlist_id = res
|
||||
if not playlist_id:
|
||||
return {"success": False, "error": "Deezer did not return a playlist id"}
|
||||
return {
|
||||
"success": True,
|
||||
"playlist_id": str(playlist_id),
|
||||
"url": f"https://www.deezer.com/playlist/{playlist_id}",
|
||||
"added": len(song_ids),
|
||||
}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def reconnect(self, arl: str = None) -> bool:
|
||||
"""Re-authenticate with a new or existing ARL."""
|
||||
if arl is None:
|
||||
|
|
@ -417,6 +481,12 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
if aid:
|
||||
album_ids.add(str(aid))
|
||||
album_release_dates = {}
|
||||
# Deezer PLAYLIST tracks do NOT carry `track_position` (only `/track/<id>`
|
||||
# and `/album/<id>/tracks` do), so numbering them by their playlist index
|
||||
# poisons the real album track number — which then rides into the wishlist
|
||||
# and onto the downloaded file's tag (e.g. 'Apologize' tagged track 1 instead
|
||||
# of 16). Resolve the REAL position from each album's track list (cache-first).
|
||||
track_positions: Dict[str, int] = {} # str(track_id) -> album track_position
|
||||
try:
|
||||
from core.metadata.cache import get_metadata_cache
|
||||
cache = get_metadata_cache()
|
||||
|
|
@ -429,24 +499,32 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
cached = cache.get_entity('deezer', 'album', aid)
|
||||
if cached and cached.get('release_date'):
|
||||
album_release_dates[aid] = cached['release_date']
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("cache get_entity album release_date: %s", e)
|
||||
# Cache miss — fetch from API
|
||||
try:
|
||||
time.sleep(0.3) # Respect rate limits
|
||||
a_resp = self._session.get(f'https://api.deezer.com/album/{aid}', timeout=10)
|
||||
if a_resp.ok:
|
||||
a_data = a_resp.json()
|
||||
album_release_dates[aid] = a_data.get('release_date', '')
|
||||
# Store in metadata cache for future use
|
||||
if cache:
|
||||
try:
|
||||
cache.store_entity('deezer', 'album', aid, a_data)
|
||||
except Exception as e:
|
||||
logger.debug("cache store_entity album release_date: %s", e)
|
||||
except Exception as e:
|
||||
logger.debug("fetch deezer album release_date %s: %s", aid, e)
|
||||
if aid not in album_release_dates:
|
||||
try:
|
||||
time.sleep(0.3) # Respect rate limits
|
||||
a_resp = self._session.get(f'https://api.deezer.com/album/{aid}', timeout=10)
|
||||
if a_resp.ok:
|
||||
a_data = a_resp.json()
|
||||
album_release_dates[aid] = a_data.get('release_date', '')
|
||||
# Store in metadata cache for future use
|
||||
if cache:
|
||||
try:
|
||||
cache.store_entity('deezer', 'album', aid, a_data)
|
||||
except Exception as e:
|
||||
logger.debug("cache store_entity album release_date: %s", e)
|
||||
except Exception as e:
|
||||
logger.debug("fetch deezer album release_date %s: %s", aid, e)
|
||||
# Real album track positions (separate endpoint — playlist tracks AND the
|
||||
# album object's embedded tracks both omit track_position). Cache-first.
|
||||
try:
|
||||
from core.deezer_client import resolve_album_track_positions
|
||||
track_positions = resolve_album_track_positions(
|
||||
self._session, 'https://api.deezer.com', album_ids, cache)
|
||||
except Exception as e:
|
||||
logger.debug("resolve deezer album track positions: %s", e)
|
||||
|
||||
tracks = []
|
||||
for i, t in enumerate(raw_tracks, start=1):
|
||||
|
|
@ -467,7 +545,9 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
'id': album_id,
|
||||
},
|
||||
'duration_ms': t.get('duration', 0) * 1000,
|
||||
'track_number': i,
|
||||
# REAL album position (resolved above); the playlist index is a last
|
||||
# resort only when the album lookup failed, never the default.
|
||||
'track_number': track_positions.get(str(t.get('id'))) or t.get('track_position') or i,
|
||||
})
|
||||
|
||||
return {
|
||||
|
|
@ -581,7 +661,7 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
bitrate = 128
|
||||
quality = 'mp3'
|
||||
|
||||
results.append(TrackResult(
|
||||
tr = TrackResult(
|
||||
username='deezer_dl',
|
||||
filename=f"{track_id}||{artist} - {title}",
|
||||
size=est_size,
|
||||
|
|
@ -595,7 +675,10 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
title=title,
|
||||
album=album,
|
||||
track_number=item.get('track_position'),
|
||||
))
|
||||
)
|
||||
# Stamp CD-quality FLAC (16/44.1) so lossless ranks correctly.
|
||||
tr.set_quality(quality_from_deezer(self._quality))
|
||||
results.append(tr)
|
||||
|
||||
logger.info(f"Deezer search for '{query}' returned {len(results)} results")
|
||||
return results, []
|
||||
|
|
|
|||
340
core/discovery/listening_recommendations.py
Normal file
340
core/discovery/listening_recommendations.py
Normal file
|
|
@ -0,0 +1,340 @@
|
|||
"""Listening-driven recommendation core (#913).
|
||||
|
||||
PURE, side-effect-free ranking that turns "the artists you listen to most" plus
|
||||
"who's similar to each" into:
|
||||
|
||||
1. a consensus-ranked list of artists you'd probably love but don't own, and
|
||||
2. an aggregated candidate-track list for a generated playlist.
|
||||
|
||||
No DB / network / config here. The caller (the watchlist scanner) supplies the
|
||||
seeds (top-played artists), the ``similar_artists`` rows per seed, and the
|
||||
owned-artist set, then fetches top tracks for the winners. Keeping the decision
|
||||
logic in one pure place makes it fully unit-testable without the live stack and
|
||||
keeps the scan wiring thin — and additive, so it can't disturb existing flows.
|
||||
|
||||
Scoring rationale (the "best in class" bit): a recommended artist's score is
|
||||
``Σ over the seeds that recommend it of (seed_weight × similarity)``. That single
|
||||
sum rewards all three signals at once — **consensus** (an artist endorsed by many
|
||||
of your seeds accumulates more terms), your **play weight** (heavier seeds push
|
||||
harder), and **similarity strength** — instead of a flat "appears in N lists".
|
||||
``seed_count`` is exposed separately for display ("because you like A, B, C") and
|
||||
as the adventurousness dial's lever (``min_seed_count``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Sequence, Set
|
||||
|
||||
|
||||
def _norm(name: object) -> str:
|
||||
return str(name or "").strip().lower()
|
||||
|
||||
|
||||
def _positive_float(value: object, default: float = 1.0) -> float:
|
||||
try:
|
||||
f = float(value) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return f if f > 0 else default
|
||||
|
||||
|
||||
def _get(row: object, attr: str):
|
||||
"""Read a field from a dataclass row or a dict row."""
|
||||
if isinstance(row, dict):
|
||||
return row.get(attr)
|
||||
return getattr(row, attr, None)
|
||||
|
||||
|
||||
def choose_mix_fetch_source(active_source: object, active_can_fetch: bool) -> str:
|
||||
"""Pick which source to fetch the "Listening Mix" top tracks from.
|
||||
|
||||
The mix is a list of (artist, title) pairs acquired via Soulseek, so the fetch source need
|
||||
NOT match the user's active metadata source. Use the active source when it can fetch top
|
||||
tracks itself (Spotify/Deezer); otherwise fall back to Deezer, whose public ``artist/{id}/top``
|
||||
needs no auth and is available to every user — so iTunes / Discogs / MusicBrainz users still
|
||||
get a full mix without switching sources. Pure.
|
||||
"""
|
||||
if str(active_source or "").lower() in ("spotify", "deezer") and active_can_fetch:
|
||||
return str(active_source).lower()
|
||||
return "deezer"
|
||||
|
||||
|
||||
def names_match(a: object, b: object) -> bool:
|
||||
"""Strict artist-name equality after stripping case + non-alphanumerics.
|
||||
|
||||
Used to verify a name-search result before fetching that artist's top tracks, so the
|
||||
"Listening Mix" can never pull the WRONG artist's songs (e.g. a same-name act). Exact
|
||||
alphanumeric match: "Tyler, The Creator" == "Tyler The Creator", but "Drake" != "Drake Bell".
|
||||
Pure.
|
||||
"""
|
||||
def _alnum(x: object) -> str:
|
||||
return "".join(ch for ch in str(x or "").lower() if ch.isalnum())
|
||||
na, nb = _alnum(a), _alnum(b)
|
||||
return bool(na) and na == nb
|
||||
|
||||
|
||||
def similarity_from_rank(rank: object, max_rank: int = 10) -> float:
|
||||
"""Turn a stored ``similarity_rank`` (1 = most similar … 10 = least) into a 0–1 weight.
|
||||
|
||||
SoulSync stores each ``(seed → similar)`` edge with a 1–10 rank (``1`` is the closest
|
||||
match). The ranker multiplies this into the score so a seed's *closest* matches count
|
||||
for more than its long-tail ones. Linear decay over the documented range: rank 1 → 1.0,
|
||||
rank 5 → 0.6, rank 10 → 0.1, with a 0.1 floor so a far match still contributes. A
|
||||
missing/garbage rank falls back to 1.0 (treat as "no rank info, full weight"). Pure.
|
||||
"""
|
||||
try:
|
||||
r = int(rank)
|
||||
except (TypeError, ValueError):
|
||||
return 1.0
|
||||
floor = round(1.0 / max_rank, 4)
|
||||
if r <= 1:
|
||||
return 1.0
|
||||
if r >= max_rank:
|
||||
return floor
|
||||
return round((max_rank - r + 1) / max_rank, 4)
|
||||
|
||||
|
||||
def build_recency_weighted_seeds(
|
||||
top_artists: Sequence[dict],
|
||||
recent_play_counts: Optional[Dict[str, float]] = None,
|
||||
*,
|
||||
recency_factor: float = 1.5,
|
||||
) -> List[dict]:
|
||||
"""Blend lifetime + recent play counts into seed weights — "what you're into NOW".
|
||||
|
||||
``weight = lifetime_plays + recency_factor × recent_plays``. An artist you've played a
|
||||
lot *recently* outranks one you played a lot years ago, so the recommendations track
|
||||
your current taste instead of your all-time history. ``recency_factor`` is the dial
|
||||
(0 = pure lifetime). Returns ``[{'name', 'weight'}]`` for :func:`rank_recommended_artists`.
|
||||
Pure — the caller supplies both play-count maps from the listening history.
|
||||
"""
|
||||
recent = {_norm(k): _positive_float(v, 0.0) for k, v in (recent_play_counts or {}).items()}
|
||||
out: List[dict] = []
|
||||
for a in top_artists or ():
|
||||
name = str(a.get("name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
lifetime = _positive_float(a.get("play_count", a.get("weight", 1.0)))
|
||||
boost = recency_factor * recent.get(_norm(name), 0.0)
|
||||
out.append({"name": name, "weight": lifetime + boost})
|
||||
return out
|
||||
|
||||
|
||||
def group_similars_by_seed(
|
||||
seeds: Sequence[dict],
|
||||
similar_rows: Sequence,
|
||||
id_to_name: Dict[str, str],
|
||||
*,
|
||||
source_id_attr: str = "source_artist_id",
|
||||
similar_name_attr: str = "similar_artist_name",
|
||||
rank_attr: Optional[str] = None,
|
||||
) -> Dict[str, List[dict]]:
|
||||
"""Reshape flat ``similar_artists`` rows into ``{seed_name_lower: [{'name', 'score'?}]}``.
|
||||
|
||||
The stored rows key the similar artist by the SEED's source id (``source_artist_id``),
|
||||
not its name, so :func:`rank_recommended_artists` can't consume them directly. This
|
||||
resolves each row's source id to a name via ``id_to_name`` (``{source_artist_id:
|
||||
artist_name}`` for the library, built by the caller) and keeps only rows that resolve
|
||||
to one of the ``seeds``. Rows may be dataclass objects or dicts. Pure — no I/O.
|
||||
|
||||
``id_to_name`` MUST be keyed by whatever id the edges actually store — for SoulSync that
|
||||
is the artist's SOURCE id (Spotify/iTunes/Deezer/MusicBrainz), NOT the internal row id.
|
||||
When ``rank_attr`` is given, each row's rank is converted via :func:`similarity_from_rank`
|
||||
and carried as ``score`` so closer matches weigh more; without it every similar comes out
|
||||
score-less (the ranker then treats similarity as 1.0 — original behavior).
|
||||
"""
|
||||
seed_names = {_norm(s.get("name")) for s in seeds}
|
||||
seed_names.discard("")
|
||||
id_to_norm = {str(k): _norm(v) for k, v in (id_to_name or {}).items()}
|
||||
|
||||
out: Dict[str, List[dict]] = {}
|
||||
for row in similar_rows or ():
|
||||
seed_name = id_to_norm.get(str(_get(row, source_id_attr) or ""), "")
|
||||
if not seed_name or seed_name not in seed_names:
|
||||
continue
|
||||
sim_name = str(_get(row, similar_name_attr) or "").strip()
|
||||
if not sim_name:
|
||||
continue
|
||||
entry = {"name": sim_name}
|
||||
if rank_attr is not None:
|
||||
entry["score"] = similarity_from_rank(_get(row, rank_attr))
|
||||
out.setdefault(seed_name, []).append(entry)
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecommendedArtist:
|
||||
"""One artist recommended from your listening, with the why."""
|
||||
name: str # display name (first-seen casing)
|
||||
score: float # Σ seed_weight × similarity
|
||||
seed_count: int # distinct seeds endorsing it (consensus)
|
||||
seeds: List[str] = field(default_factory=list) # display names of those seeds
|
||||
|
||||
|
||||
def rank_recommended_artists(
|
||||
seeds: Sequence[dict],
|
||||
similars_by_seed: Dict[str, Sequence[dict]],
|
||||
owned_artist_names: Optional[Set[str]] = None,
|
||||
*,
|
||||
limit: int = 30,
|
||||
min_seed_count: int = 1,
|
||||
) -> List[RecommendedArtist]:
|
||||
"""Rank artists similar to your most-played by consensus + play weight + similarity.
|
||||
|
||||
Args:
|
||||
seeds: ``[{'name': str, 'weight': float}]`` — your top-played artists.
|
||||
``weight`` (play count or any positive number) defaults to 1.0.
|
||||
similars_by_seed: ``{seed_name_lower: [{'name': str, 'score': float}]}`` — the
|
||||
similar-artist rows for each seed. ``score`` is optional (defaults 1.0).
|
||||
owned_artist_names: lowercased names already in the library — excluded so the
|
||||
result is artists you DON'T have. The seeds themselves are always excluded.
|
||||
limit: max results.
|
||||
min_seed_count: drop recommendations endorsed by fewer than N seeds — the
|
||||
adventurousness dial's "Safer" end raises this for higher-confidence picks.
|
||||
|
||||
Returns up to ``limit`` :class:`RecommendedArtist`, highest score first.
|
||||
"""
|
||||
owned = {_norm(a) for a in (owned_artist_names or set())}
|
||||
seed_norms = {_norm(s.get("name")) for s in seeds}
|
||||
seed_norms.discard("")
|
||||
exclude = owned | seed_norms
|
||||
|
||||
acc: Dict[str, dict] = {}
|
||||
for seed in seeds:
|
||||
s_name = _norm(seed.get("name"))
|
||||
if not s_name:
|
||||
continue
|
||||
s_display = str(seed.get("name") or "").strip()
|
||||
weight = _positive_float(seed.get("weight", 1.0))
|
||||
for sim in similars_by_seed.get(s_name, ()) or ():
|
||||
a_norm = _norm(sim.get("name"))
|
||||
if not a_norm or a_norm in exclude:
|
||||
continue
|
||||
sim_score = _positive_float(sim.get("score", 1.0))
|
||||
row = acc.setdefault(
|
||||
a_norm, {"name": str(sim.get("name") or "").strip(), "score": 0.0, "seeds": {}}
|
||||
)
|
||||
row["score"] += weight * sim_score
|
||||
row["seeds"].setdefault(s_name, s_display) # one seed counts once
|
||||
|
||||
out: List[RecommendedArtist] = []
|
||||
floor = max(1, int(min_seed_count))
|
||||
for row in acc.values():
|
||||
seed_count = len(row["seeds"])
|
||||
if seed_count < floor:
|
||||
continue
|
||||
out.append(RecommendedArtist(
|
||||
name=row["name"],
|
||||
score=round(row["score"], 6),
|
||||
seed_count=seed_count,
|
||||
seeds=list(row["seeds"].values()),
|
||||
))
|
||||
out.sort(key=lambda r: (-r.score, -r.seed_count, r.name.lower()))
|
||||
return out[:limit]
|
||||
|
||||
|
||||
def aggregate_candidate_tracks(
|
||||
recommended_artists: Sequence[RecommendedArtist],
|
||||
top_tracks_by_artist: Dict[str, Sequence[dict]],
|
||||
owned_track_keys: Optional[Set] = None,
|
||||
*,
|
||||
per_artist: int = 3,
|
||||
limit: int = 50,
|
||||
exclude_owned: bool = True,
|
||||
) -> List[dict]:
|
||||
"""Build the candidate track list for the generated playlist.
|
||||
|
||||
Takes the top ``per_artist`` tracks from each recommended artist **in artist-rank
|
||||
order**, dedups by ``(artist, title)``, optionally drops owned tracks (the
|
||||
"discovery" flavor) and caps at ``limit``. Each returned track dict is the source
|
||||
track plus ``_seed_artist`` (which recommended artist it came from).
|
||||
|
||||
Args:
|
||||
recommended_artists: ranked output of :func:`rank_recommended_artists`.
|
||||
top_tracks_by_artist: ``{artist_name_lower: [track_dict, ...]}`` — fetched by
|
||||
the caller (Last.fm / source top tracks), NOT limited to a curated pool.
|
||||
owned_track_keys: set of ``(artist_lower, title_lower)`` already in the library.
|
||||
exclude_owned: drop tracks in ``owned_track_keys`` (discovery flavor). Set False
|
||||
for a "replay" playlist of tracks you already own.
|
||||
"""
|
||||
owned = owned_track_keys or set()
|
||||
seen: Set = set()
|
||||
out: List[dict] = []
|
||||
for art in recommended_artists:
|
||||
tracks = top_tracks_by_artist.get(_norm(art.name), ()) or ()
|
||||
taken = 0
|
||||
for t in tracks:
|
||||
if taken >= per_artist:
|
||||
break
|
||||
title = str(t.get("name") or t.get("title") or "").strip()
|
||||
if not title:
|
||||
continue
|
||||
key = (_norm(art.name), _norm(title))
|
||||
if key in seen:
|
||||
continue
|
||||
if exclude_owned and key in owned:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append({**t, "_seed_artist": art.name})
|
||||
taken += 1
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out[:limit]
|
||||
|
||||
|
||||
def to_mix_track(track: object, source: str) -> Optional[dict]:
|
||||
"""Shape one source "top tracks" API dict into the flat dict the Discover compact
|
||||
playlist row renders + syncs (the "Listening Mix" #913 playlist).
|
||||
|
||||
Spotify's ``artist_top_tracks`` and Deezer's ``get_artist_top_tracks`` both return the
|
||||
same Spotify-shape object (``id, name, artists[], album{name,images[]}, duration_ms``).
|
||||
This flattens that into the renderer's field names (``track_name/artist_name/album_name/
|
||||
album_cover_url/duration_ms``), keeps the original under ``track_data_json`` for sync, and
|
||||
sets the source-specific id field. Returns None for anything without a usable id/title so
|
||||
the caller can filter. A ``name`` key is kept so :func:`aggregate_candidate_tracks` can
|
||||
dedup by title. Pure — no I/O.
|
||||
"""
|
||||
if not isinstance(track, dict):
|
||||
return None
|
||||
tid = track.get("id")
|
||||
name = str(track.get("name") or "").strip()
|
||||
if not tid or not name:
|
||||
return None
|
||||
artists = track.get("artists") or []
|
||||
artist_name = ""
|
||||
if artists and isinstance(artists[0], dict):
|
||||
artist_name = str(artists[0].get("name") or "").strip()
|
||||
album = track.get("album") if isinstance(track.get("album"), dict) else {}
|
||||
album_name = str(album.get("name") or "").strip()
|
||||
images = album.get("images") or []
|
||||
cover = images[0].get("url") if images and isinstance(images[0], dict) else None
|
||||
out = {
|
||||
"track_id": str(tid),
|
||||
"name": name, # for aggregate_candidate_tracks dedup
|
||||
"track_name": name, # for the renderer
|
||||
"artist_name": artist_name,
|
||||
"album_name": album_name,
|
||||
"album_cover_url": cover,
|
||||
"duration_ms": track.get("duration_ms") or 0,
|
||||
"track_data_json": track, # full payload for sync/download
|
||||
"source": source,
|
||||
}
|
||||
id_field = {"spotify": "spotify_track_id", "deezer": "deezer_track_id",
|
||||
"itunes": "itunes_track_id"}.get(source)
|
||||
if id_field:
|
||||
out[id_field] = str(tid)
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RecommendedArtist",
|
||||
"choose_mix_fetch_source",
|
||||
"names_match",
|
||||
"similarity_from_rank",
|
||||
"build_recency_weighted_seeds",
|
||||
"to_mix_track",
|
||||
"group_similars_by_seed",
|
||||
"rank_recommended_artists",
|
||||
"aggregate_candidate_tracks",
|
||||
]
|
||||
|
|
@ -21,7 +21,11 @@ from core.metadata.registry import get_client_for_source
|
|||
from core.metadata.types import Album
|
||||
from core.wishlist.payloads import ensure_wishlist_track_format
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
# Use the project logger namespace ("soulsync.*") so the scanner's progress and
|
||||
# diagnostics actually surface in the app log — plain getLogger(__name__) lands
|
||||
# under "core.discovery.quality_scanner", which the app log view doesn't show.
|
||||
from utils.logging_config import get_logger
|
||||
logger = get_logger("discovery.quality_scanner")
|
||||
|
||||
|
||||
# Per-source typed converter dispatch — same registry pattern as
|
||||
|
|
|
|||
|
|
@ -173,6 +173,32 @@ async def _database_only_find_track(spotify_track, candidate_pool=None):
|
|||
logger.debug("sync match cache fast-path failed: %s", e)
|
||||
# --- End cache fast-path ---
|
||||
|
||||
# Durable manual library match (#787) — survives a library rescan (the
|
||||
# sync_match_cache above does not), so a user's Find & Add pairing keeps
|
||||
# sticking across auto-syncs instead of being re-matched from scratch (#895
|
||||
# follow-up). Self-heals a stale library id via the stored file path.
|
||||
if spotify_id:
|
||||
try:
|
||||
from core.artists.map import get_current_profile_id
|
||||
m = db.find_manual_library_match_by_source_track_id(
|
||||
get_current_profile_id(), str(spotify_id), active_server)
|
||||
if m:
|
||||
lib_id = m.get('library_track_id')
|
||||
dt = db.get_track_by_id(lib_id) if lib_id is not None else None
|
||||
if not dt and m.get('library_file_path'):
|
||||
new_id = db.find_track_id_by_file_path(m['library_file_path'])
|
||||
dt = db.get_track_by_id(new_id) if new_id else None
|
||||
if dt:
|
||||
class DatabaseTrackDurable:
|
||||
def __init__(self, db_t):
|
||||
self.ratingKey = db_t.id
|
||||
self.title = db_t.title
|
||||
self.id = db_t.id
|
||||
logger.debug(f"Durable manual match hit: '{original_title}' → {lib_id}")
|
||||
return DatabaseTrackDurable(dt), 1.0
|
||||
except Exception as e:
|
||||
logger.debug("durable manual match fast-path failed: %s", e)
|
||||
|
||||
# Try each artist
|
||||
for artist in spotify_track.artists:
|
||||
if isinstance(artist, str):
|
||||
|
|
@ -281,35 +307,11 @@ def run_sync_task(
|
|||
# This avoids needing to re-fetch it from Spotify
|
||||
logger.info("Converting JSON tracks to SpotifyTrack objects...")
|
||||
|
||||
# Store original track data with full album objects (for wishlist with cover art)
|
||||
# Normalize formats for wishlist: album must be dict {'name': ...}, artists must be [{'name': ...}]
|
||||
# Important: copy data — don't mutate tracks_json since SpotifyTrack expects List[str] artists
|
||||
original_tracks_map = {}
|
||||
for t in tracks_json:
|
||||
track_id = t.get('id', '')
|
||||
if track_id:
|
||||
normalized = dict(t)
|
||||
# Normalize album to dict format, preserving images and metadata
|
||||
raw_album = normalized.get('album', '')
|
||||
if isinstance(raw_album, str):
|
||||
normalized['album'] = {
|
||||
'name': raw_album or normalized.get('name', 'Unknown Album'),
|
||||
'images': [], 'album_type': 'single', 'total_tracks': 1, 'release_date': ''
|
||||
}
|
||||
elif not isinstance(raw_album, dict):
|
||||
normalized['album'] = {
|
||||
'name': str(raw_album) if raw_album else normalized.get('name', 'Unknown Album'),
|
||||
'images': [], 'album_type': 'single', 'total_tracks': 1, 'release_date': ''
|
||||
}
|
||||
else:
|
||||
# Dict — ensure required keys exist
|
||||
raw_album.setdefault('name', 'Unknown Album')
|
||||
raw_album.setdefault('images', [])
|
||||
# Normalize artists to list of dicts
|
||||
raw_artists = normalized.get('artists', [])
|
||||
if raw_artists and isinstance(raw_artists[0], str):
|
||||
normalized['artists'] = [{'name': a} for a in raw_artists]
|
||||
original_tracks_map[track_id] = normalized
|
||||
# Store original track data with full album objects (for wishlist with cover art).
|
||||
# Shared with the sync-detail "re-add to wishlist" action so both build the
|
||||
# IDENTICAL payload (album→dict + images, artists→dicts). Copy-safe.
|
||||
from core.sync.wishlist_readd import build_original_tracks_map
|
||||
original_tracks_map = build_original_tracks_map(tracks_json)
|
||||
|
||||
tracks = []
|
||||
for i, t in enumerate(tracks_json):
|
||||
|
|
|
|||
|
|
@ -32,6 +32,26 @@ from typing import Any, Callable
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_UNKNOWN_ARTIST = 'Unknown Artist'
|
||||
|
||||
|
||||
def resolve_display_artist(yt_artist: str, matched_artist: str) -> str:
|
||||
"""The artist to show in the 'YT Artist' column (#909).
|
||||
|
||||
YouTube's flat playlist data carries no artist, so a track starts as
|
||||
"Unknown Artist" and only gains a real name if per-video recovery succeeds.
|
||||
When recovery comes up empty but the track still matched confidently, show
|
||||
the matched artist instead of a misleading "Unknown Artist". Returns the
|
||||
original ``yt_artist`` whenever it's already a real name (recovery worked) or
|
||||
when there's no matched artist to fall back to — purely a display choice, the
|
||||
match itself is unaffected.
|
||||
"""
|
||||
current = (yt_artist or '').strip()
|
||||
if current and current != _UNKNOWN_ARTIST:
|
||||
return current # recovery already gave a real name — keep it
|
||||
fallback = (matched_artist or '').strip()
|
||||
return fallback or _UNKNOWN_ARTIST # backfill from the match, else honest Unknown
|
||||
|
||||
|
||||
@dataclass
|
||||
class YoutubeDiscoveryDeps:
|
||||
|
|
@ -131,14 +151,15 @@ def run_youtube_discovery_worker(url_hash, deps: YoutubeDiscoveryDeps):
|
|||
cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source)
|
||||
if cached_match and deps.validate_discovery_cache_artist(cleaned_artist, cached_match):
|
||||
logger.debug(f"CACHE HIT [{i+1}/{len(tracks)}]: {cleaned_artist} - {cleaned_title}")
|
||||
_match_artist = deps.extract_artist_name(cached_match.get('artists', [''])[0]) if cached_match.get('artists') else ''
|
||||
result = {
|
||||
'index': i,
|
||||
'yt_track': cleaned_title,
|
||||
'yt_artist': cleaned_artist,
|
||||
'yt_artist': resolve_display_artist(cleaned_artist, _match_artist),
|
||||
'status': 'Found',
|
||||
'status_class': 'found',
|
||||
'spotify_track': cached_match.get('name', ''),
|
||||
'spotify_artist': deps.extract_artist_name(cached_match.get('artists', [''])[0]) if cached_match.get('artists') else '',
|
||||
'spotify_artist': _match_artist,
|
||||
'spotify_album': cached_match.get('album', {}).get('name', '') if isinstance(cached_match.get('album'), dict) else cached_match.get('album', ''),
|
||||
'duration': f"{int(track['duration_ms']) // 60000}:{(int(track['duration_ms']) % 60000) // 1000:02d}" if track['duration_ms'] else '0:00',
|
||||
'discovery_source': discovery_source,
|
||||
|
|
@ -265,15 +286,17 @@ def run_youtube_discovery_worker(url_hash, deps: YoutubeDiscoveryDeps):
|
|||
best_confidence = confidence
|
||||
logger.info(f"Strategy 4 YouTube match (extended): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")
|
||||
|
||||
# Create result entry
|
||||
# Create result entry. yt_artist falls back to the matched artist when
|
||||
# YouTube/recovery left it "Unknown Artist" but we matched confidently (#909).
|
||||
_match_artist = deps.extract_artist_name(matched_track.artists[0]) if matched_track else ''
|
||||
result = {
|
||||
'index': i,
|
||||
'yt_track': cleaned_title,
|
||||
'yt_artist': cleaned_artist,
|
||||
'yt_artist': resolve_display_artist(cleaned_artist, _match_artist),
|
||||
'status': 'Found' if matched_track else 'Not Found',
|
||||
'status_class': 'found' if matched_track else 'not-found',
|
||||
'spotify_track': matched_track.name if matched_track else '',
|
||||
'spotify_artist': deps.extract_artist_name(matched_track.artists[0]) if matched_track else '',
|
||||
'spotify_artist': _match_artist,
|
||||
'spotify_album': matched_track.album if matched_track else '',
|
||||
'duration': f"{int(track['duration_ms']) // 60000}:{(int(track['duration_ms']) % 60000) // 1000:02d}" if track['duration_ms'] else '0:00',
|
||||
'discovery_source': discovery_source,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ big-bang switchover.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
||||
|
||||
|
|
@ -391,6 +392,16 @@ class DownloadEngine:
|
|||
(tracks, albums) tuple, or ``([], [])`` when every source
|
||||
in the chain is exhausted.
|
||||
|
||||
Priority mode is deliberately quality-AGNOSTIC at search time — source
|
||||
order is king and the first source that returns any tracks wins, exactly
|
||||
matching pre-quality-system behaviour byte-for-byte (#896 review #3).
|
||||
Quality-gating the priority path would deprioritise e.g. a soulseek
|
||||
mp3 whose bitrate slskd omitted (``bitrate=None`` → "unsatisfied"),
|
||||
changing which source wins and adding latency for users who never opted
|
||||
in. Cross-source quality pooling is the job of best_quality mode
|
||||
(``search_all_sources``); final per-result ranking still happens in the
|
||||
orchestrator's match/quality filter. RAW tracks are returned.
|
||||
|
||||
Replaces orchestrator's hand-rolled hybrid search loop. The
|
||||
chain is ordered (most-preferred first).
|
||||
"""
|
||||
|
|
@ -406,9 +417,10 @@ class DownloadEngine:
|
|||
try:
|
||||
logger.info(f"Trying {source_name} (priority {i+1}): {query}")
|
||||
tracks, albums = await plugin.search(query, timeout, progress_callback)
|
||||
if tracks:
|
||||
logger.info(f"{source_name} found {len(tracks)} tracks")
|
||||
return (tracks, albums)
|
||||
if not tracks:
|
||||
continue
|
||||
logger.info(f"{source_name} found {len(tracks)} tracks")
|
||||
return (tracks, albums)
|
||||
except Exception as e:
|
||||
logger.warning(f"{source_name} search failed: {e}")
|
||||
|
||||
|
|
@ -418,6 +430,75 @@ class DownloadEngine:
|
|||
)
|
||||
return ([], [])
|
||||
|
||||
async def search_all_sources(self, query: str, source_chain,
|
||||
timeout=None, progress_callback=None,
|
||||
exclude_sources=None):
|
||||
"""Best-quality mode: pool RAW tracks from EVERY configured source in
|
||||
``source_chain`` instead of stopping at the first satisfying one.
|
||||
|
||||
Unlike :meth:`search_with_fallback`, no source short-circuits the
|
||||
search — the caller (orchestrator/worker) ranks the combined pool
|
||||
best→worst by actual audio quality. ``exclude_sources`` drops sources
|
||||
whose per-source retry budget is already spent (so their candidates
|
||||
never re-enter the pool). Unconfigured / unregistered / raising sources
|
||||
are skipped exactly like the fallback path. Returns
|
||||
``(combined_tracks, combined_albums)``.
|
||||
"""
|
||||
excluded = {s.lower() for s in (exclude_sources or []) if s}
|
||||
pooled_tracks = []
|
||||
pooled_albums = []
|
||||
# Per-source contribution for an honest pool log — e.g. a release-level
|
||||
# source like usenet/torrent that returns nothing for a track-title
|
||||
# query should read "usenet=0", not silently hide behind the chain name.
|
||||
contributions = []
|
||||
|
||||
# Decide which sources to actually query, recording why the rest were
|
||||
# skipped. Searches then run CONCURRENTLY so the pool waits only for the
|
||||
# slowest source (e.g. usenet/Prowlarr, which can be slow) rather than
|
||||
# the sum of every source's latency.
|
||||
to_search = [] # (source_name, plugin)
|
||||
for source_name in source_chain:
|
||||
if source_name.lower() in excluded:
|
||||
contributions.append(f"{source_name}=excluded")
|
||||
continue
|
||||
plugin = self._plugins.get(source_name)
|
||||
if plugin is None:
|
||||
logger.info(f"Skipping {source_name} (not available)")
|
||||
contributions.append(f"{source_name}=unavailable")
|
||||
continue
|
||||
if hasattr(plugin, 'is_configured') and not plugin.is_configured():
|
||||
logger.info(f"Skipping {source_name} (not configured)")
|
||||
contributions.append(f"{source_name}=unconfigured")
|
||||
continue
|
||||
to_search.append((source_name, plugin))
|
||||
|
||||
async def _one(plugin):
|
||||
return await plugin.search(query, timeout, progress_callback)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*[_one(plugin) for _, plugin in to_search],
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
for (source_name, _), result in zip(to_search, results, strict=True):
|
||||
if isinstance(result, Exception):
|
||||
logger.warning(f"{source_name} search failed: {result}")
|
||||
contributions.append(f"{source_name}=error")
|
||||
continue
|
||||
tracks, albums = result
|
||||
n = len(tracks) if tracks else 0
|
||||
if tracks:
|
||||
pooled_tracks.extend(tracks)
|
||||
if albums:
|
||||
pooled_albums.extend(albums)
|
||||
contributions.append(f"{source_name}={n}")
|
||||
|
||||
logger.info(
|
||||
"Best-quality pool: %d candidates [%s] for: %s",
|
||||
len(pooled_tracks), ', '.join(contributions), query,
|
||||
)
|
||||
return (pooled_tracks, pooled_albums)
|
||||
|
||||
async def download_with_fallback(self, username: str, filename: str,
|
||||
file_size: int, source_chain) -> Optional[str]:
|
||||
"""Try each source in ``source_chain`` until one accepts the
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from config.settings import config_manager
|
|||
from core.download_engine import DownloadEngine
|
||||
from core.download_plugins.registry import DownloadPluginRegistry, build_default_registry
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.quality.selection import load_search_mode
|
||||
|
||||
logger = get_logger("download_orchestrator")
|
||||
|
||||
|
|
@ -102,12 +103,14 @@ class DownloadOrchestrator:
|
|||
deezer_dl = self.client('deezer_dl')
|
||||
if deezer_arl and deezer_dl:
|
||||
deezer_dl.reconnect(deezer_arl)
|
||||
deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac')
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
deezer_dl._quality = quality_tier_for_source('deezer', default='flac')
|
||||
|
||||
# Reload Amazon quality preference (T2Tunes needs no reconnect — public proxy)
|
||||
amazon = self.client('amazon')
|
||||
if amazon:
|
||||
quality = config_manager.get('amazon_download.quality', 'flac')
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
quality = quality_tier_for_source('amazon', default='flac')
|
||||
amazon._quality = quality
|
||||
amazon._allow_fallback = config_manager.get('amazon_download.allow_fallback', True)
|
||||
if hasattr(amazon, '_client') and amazon._client:
|
||||
|
|
@ -140,7 +143,10 @@ class DownloadOrchestrator:
|
|||
continue
|
||||
if hasattr(client, 'download_path') and client.download_path != new_path:
|
||||
client.download_path = new_path
|
||||
client.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
client.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {new_path}: {e}")
|
||||
# YouTube also caches path in yt-dlp opts
|
||||
if hasattr(client, 'download_opts') and 'outtmpl' in client.download_opts:
|
||||
client.download_opts['outtmpl'] = str(new_path / '%(title)s.%(ext)s')
|
||||
|
|
@ -342,6 +348,11 @@ class DownloadOrchestrator:
|
|||
if not chain:
|
||||
logger.warning("Hybrid search exhausted: no eligible sources after exclusion filter")
|
||||
return [], []
|
||||
if load_search_mode() == 'best_quality':
|
||||
logger.info(f"Best-quality search ({' → '.join(chain)}): {query}")
|
||||
return await self.engine.search_all_sources(
|
||||
query, chain, timeout, progress_callback,
|
||||
)
|
||||
logger.info(f"Hybrid search ({' → '.join(chain)}): {query}")
|
||||
return await self.engine.search_with_fallback(query, chain, timeout, progress_callback)
|
||||
|
||||
|
|
@ -412,9 +423,17 @@ class DownloadOrchestrator:
|
|||
|
||||
if scored:
|
||||
scored.sort(key=lambda x: x._match_confidence, reverse=True)
|
||||
filtered_results = scored
|
||||
# Match filter done (right track); now prefer the best quality
|
||||
# among the confidence-passing survivors so streaming isn't
|
||||
# quality-blind like Soulseek already isn't. Stable ranking
|
||||
# keeps confidence order within an equal quality tier; the
|
||||
# `or scored` fail-safe never leaves us with nothing to try.
|
||||
from core.quality.selection import rank_for_profile
|
||||
ranked, _ = rank_for_profile(scored)
|
||||
filtered_results = ranked or scored
|
||||
logger.info(f"Streaming validation: {len(scored)}/{len(tracks)} passed "
|
||||
f"(best: {scored[0]._match_confidence:.2f})")
|
||||
f"(best: {scored[0]._match_confidence:.2f}, "
|
||||
f"quality pick: {filtered_results[0].audio_quality.label()})")
|
||||
else:
|
||||
logger.warning(f"No streaming results passed validation for: {query}")
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -13,10 +13,11 @@ import from a neutral package per Cin's contract-first standard.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from core.imports.filename import parse_filename_metadata
|
||||
from core.quality.model import AudioQuality
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -32,6 +33,36 @@ class SearchResult:
|
|||
upload_speed: int
|
||||
queue_length: int
|
||||
result_type: str = "track" # "track" or "album"
|
||||
# Rich quality metadata — populated by sources that provide it.
|
||||
# None means "unknown", not "absent".
|
||||
sample_rate: Optional[int] = None # Hz (e.g. 44100, 96000, 192000)
|
||||
bit_depth: Optional[int] = None # bits per sample (16, 24)
|
||||
|
||||
@property
|
||||
def audio_quality(self) -> AudioQuality:
|
||||
"""Unified quality descriptor derived from this result's fields."""
|
||||
return AudioQuality(
|
||||
format=self.quality.lower() if self.quality else 'unknown',
|
||||
bitrate=self.bitrate,
|
||||
sample_rate=self.sample_rate,
|
||||
bit_depth=self.bit_depth,
|
||||
)
|
||||
|
||||
def set_quality(self, aq: AudioQuality) -> None:
|
||||
"""Merge a mapped :class:`AudioQuality` onto this result's fields.
|
||||
|
||||
Used by streaming sources to stamp their claimed tier (Tidal/HiFi
|
||||
tier strings, Qobuz API values, …) so ``audio_quality`` ranks
|
||||
correctly. Mapper-provided fields win; a ``None`` from the mapper
|
||||
leaves any already-reported value (e.g. a probed bitrate) intact.
|
||||
"""
|
||||
self.quality = aq.format
|
||||
if aq.bitrate is not None:
|
||||
self.bitrate = aq.bitrate
|
||||
if aq.sample_rate is not None:
|
||||
self.sample_rate = aq.sample_rate
|
||||
if aq.bit_depth is not None:
|
||||
self.bit_depth = aq.bit_depth
|
||||
|
||||
@property
|
||||
def quality_score(self) -> float:
|
||||
|
|
@ -127,6 +158,19 @@ class AlbumResult:
|
|||
queue_length: int = 0
|
||||
result_type: str = "album"
|
||||
|
||||
@property
|
||||
def audio_quality(self) -> AudioQuality:
|
||||
"""Unified quality descriptor derived from dominant track quality."""
|
||||
sample_rates = [t.sample_rate for t in self.tracks if t.sample_rate]
|
||||
bit_depths = [t.bit_depth for t in self.tracks if t.bit_depth]
|
||||
bitrates = [t.bitrate for t in self.tracks if t.bitrate]
|
||||
return AudioQuality(
|
||||
format=self.dominant_quality.lower() if self.dominant_quality else 'unknown',
|
||||
bitrate=max(bitrates) if bitrates else None,
|
||||
sample_rate=max(sample_rates) if sample_rates else None,
|
||||
bit_depth=max(bit_depths) if bit_depths else None,
|
||||
)
|
||||
|
||||
@property
|
||||
def quality_score(self) -> float:
|
||||
"""Calculate album quality score based on dominant quality and track count"""
|
||||
|
|
|
|||
|
|
@ -83,9 +83,26 @@ def clear_completed_local() -> int:
|
|||
"""
|
||||
cleared = 0
|
||||
with tasks_lock:
|
||||
# Protect tasks belonging to a still-active batch. A batch is "active"
|
||||
# while any of its queued tasks is non-terminal (still searching /
|
||||
# downloading / queued / post-processing). Pruning a batch's completed
|
||||
# or failed tasks mid-run would yank them out of the Downloads page —
|
||||
# and failed/cancelled rows aren't recoverable from library_history —
|
||||
# so the user would never see them until the batch ended. Keep the whole
|
||||
# active batch intact; it gets cleaned by a later run once it finishes.
|
||||
protected_task_ids: set = set()
|
||||
for batch in download_batches.values():
|
||||
queue = batch.get('queue', []) if isinstance(batch, dict) else []
|
||||
batch_active = any(
|
||||
download_tasks.get(tid, {}).get('status') not in _TERMINAL_STATUSES
|
||||
for tid in queue if tid in download_tasks
|
||||
)
|
||||
if batch_active:
|
||||
protected_task_ids.update(queue)
|
||||
|
||||
task_ids_to_remove = [
|
||||
tid for tid, task in download_tasks.items()
|
||||
if task.get('status') in _TERMINAL_STATUSES
|
||||
if task.get('status') in _TERMINAL_STATUSES and tid not in protected_task_ids
|
||||
]
|
||||
for tid in task_ids_to_remove:
|
||||
del download_tasks[tid]
|
||||
|
|
|
|||
|
|
@ -47,6 +47,55 @@ from core.runtime_state import (
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _priority_sort_key(r):
|
||||
"""Today's confidence-first key: never download a high-quality WRONG file."""
|
||||
return (
|
||||
getattr(r, 'confidence', 0) or 0,
|
||||
getattr(r, 'quality_score', 0) or 0,
|
||||
getattr(r, 'upload_speed', 0) or 0,
|
||||
-(getattr(r, 'queue_length', 0) or 0),
|
||||
getattr(r, 'free_upload_slots', 0) or 0,
|
||||
getattr(r, 'size', 0) or 0,
|
||||
)
|
||||
|
||||
|
||||
def _quality_first_sort_key(r, targets):
|
||||
"""Best-quality key: the user's profile quality rank dominates; all the
|
||||
priority-mode signals (confidence, speed, …) become tiebreakers.
|
||||
|
||||
Every candidate reaching this point already passed match filtering, so it
|
||||
is "correct enough" — ordering by quality among correct candidates is safe.
|
||||
Candidates with no usable quality info, or that match no target, sort last
|
||||
(never dropped). Lower target index = better target, so it's negated to fit
|
||||
the descending (reverse=True) sort.
|
||||
"""
|
||||
from core.quality.model import rank_candidate
|
||||
|
||||
aq = getattr(r, 'audio_quality', None)
|
||||
if aq is None or not targets:
|
||||
target_idx, tier = (len(targets) if targets else 0), 0.0
|
||||
else:
|
||||
try:
|
||||
target_idx, tier = rank_candidate(aq, targets)
|
||||
except Exception:
|
||||
target_idx, tier = len(targets), 0.0
|
||||
return (-target_idx, tier) + _priority_sort_key(r)
|
||||
|
||||
|
||||
def order_candidates(candidates, *, quality_first=False, targets=None):
|
||||
"""Return *candidates* ordered best-first for the download walk.
|
||||
|
||||
``quality_first=False`` (priority mode) → confidence-first, byte-for-byte
|
||||
today's behaviour. ``quality_first=True`` (best-quality mode) → the user's
|
||||
profile quality rank dominates, confidence/peer signals break ties.
|
||||
"""
|
||||
if quality_first:
|
||||
key = lambda r: _quality_first_sort_key(r, targets or [])
|
||||
else:
|
||||
key = _priority_sort_key
|
||||
return sorted(candidates, key=key, reverse=True)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CandidatesDeps:
|
||||
"""Bundle of cross-cutting deps the candidate-fallback logic needs."""
|
||||
|
|
@ -59,25 +108,25 @@ class CandidatesDeps:
|
|||
on_download_completed: Callable
|
||||
|
||||
|
||||
def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, deps: CandidatesDeps = None):
|
||||
def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
|
||||
deps: CandidatesDeps = None, *,
|
||||
quality_first=False, quality_targets=None):
|
||||
"""
|
||||
Attempts to download with fallback candidate logic (matches GUI's retry_parallel_download_with_fallback).
|
||||
Returns True if successful, False if all candidates fail.
|
||||
|
||||
``quality_first`` (best-quality search mode) orders the walk by the user's
|
||||
profile quality rank instead of confidence-first; ``quality_targets`` is the
|
||||
profile target list used for that ranking. Defaults preserve priority-mode
|
||||
behaviour exactly.
|
||||
"""
|
||||
# Sort candidates by match confidence first, then peer quality. Upstream
|
||||
# Soulseek validation already considers peer speed/slots/queue when scores
|
||||
# are close; preserve that signal here instead of flattening ties back to
|
||||
# arbitrary slskd response order.
|
||||
candidates.sort(
|
||||
key=lambda r: (
|
||||
getattr(r, 'confidence', 0) or 0,
|
||||
getattr(r, 'quality_score', 0) or 0,
|
||||
getattr(r, 'upload_speed', 0) or 0,
|
||||
-(getattr(r, 'queue_length', 0) or 0),
|
||||
getattr(r, 'free_upload_slots', 0) or 0,
|
||||
getattr(r, 'size', 0) or 0,
|
||||
),
|
||||
reverse=True,
|
||||
# Sort candidates. Priority mode: confidence-first, then peer quality —
|
||||
# upstream Soulseek validation already considers peer speed/slots/queue when
|
||||
# scores are close; preserve that signal instead of flattening ties back to
|
||||
# arbitrary slskd response order. Best-quality mode: profile quality rank
|
||||
# dominates (all candidates here already passed match filtering).
|
||||
candidates = order_candidates(
|
||||
candidates, quality_first=quality_first, targets=quality_targets,
|
||||
)
|
||||
|
||||
with tasks_lock:
|
||||
|
|
@ -205,6 +254,21 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
|
|||
'artists': _fallback_album_artists
|
||||
}
|
||||
|
||||
# #915: parity with Reorganize / manual Enrich. If the album context is lean
|
||||
# (no release_date) and the user's PRIMARY metadata source isn't Spotify, hydrate
|
||||
# it from that source — the same place a reorganize reads — so the download's
|
||||
# $year folder, release_date and album_type match instead of dropping the year /
|
||||
# defaulting to YYYY-01-01 and forcing a manual reorganize afterwards.
|
||||
try:
|
||||
from core.downloads.track_metadata_backfill import backfill_album_context_from_source
|
||||
from core.metadata import registry as _meta_registry
|
||||
from core.metadata.album_tracks import get_album_for_source as _get_album_for_source
|
||||
backfill_album_context_from_source(
|
||||
spotify_album_context, _meta_registry.get_primary_source(), _get_album_for_source,
|
||||
)
|
||||
except Exception as _bf_err: # noqa: BLE001 — never let backfill break a download
|
||||
logger.debug("[Context] primary-source album backfill skipped: %s", _bf_err)
|
||||
|
||||
download_payload = candidate.__dict__
|
||||
|
||||
username = download_payload.get('username')
|
||||
|
|
|
|||
|
|
@ -77,7 +77,16 @@ def _normalize_for_finding(text: str) -> str:
|
|||
return ""
|
||||
text = unidecode(text).lower()
|
||||
text = re.sub(r'[._/]', ' ', text)
|
||||
text = re.sub(r'[\[\(].*?[\]\)]', '', text)
|
||||
# Strip ONLY balanced bracket pairs (tags like "[FLAC]", "(Remastered 2016)").
|
||||
# The old combined pattern r'[\[\(].*?[\]\)]' allowed MISMATCHED delimiters, so a
|
||||
# lone unbalanced '[' — slskd reports "[34 - You & Me (Flume Remix)" but saves the
|
||||
# file as "34 - You & Me (Flume Remix)" — matched from that '[' all the way to the
|
||||
# next ')', eating the entire title and collapsing the search target to "flac". The
|
||||
# file then scored 0.40 against the real on-disk name and was reported "not found"
|
||||
# despite sitting right there. Per-delimiter pairs can't over-consume; a stray
|
||||
# unbalanced bracket simply survives to the alphanumeric strip below.
|
||||
text = re.sub(r'\[[^\]]*\]', '', text)
|
||||
text = re.sub(r'\([^)]*\)', '', text)
|
||||
text = re.sub(r'[^a-z0-9\s-]', '', text)
|
||||
return ' '.join(text.split()).strip()
|
||||
|
||||
|
|
|
|||
62
core/downloads/history_match.py
Normal file
62
core/downloads/history_match.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
"""Match a file back to its download-history row when its path has drifted (#934).
|
||||
|
||||
``library_history.file_path`` is frozen at import time, but the file moves afterward
|
||||
(media-server import, library reorganize) and ``tracks.file_path`` — what the AcoustID
|
||||
scanner reads — no longer equals it. Matching on the exact path alone then fails twice:
|
||||
the verification status never reaches the history row (verified tracks read "unverified"),
|
||||
and a fresh ``acoustid_scan`` row gets inserted every run (thousands of duplicates).
|
||||
|
||||
This module picks the canonical history row by exact path first, then by FILENAME guarded
|
||||
by a title check — so a shared filename ("01 - Intro.flac") can never heal the wrong song.
|
||||
Pure (no DB) so the matching rules are unit-testable; the caller does the SQL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Iterable, Optional, Sequence, Tuple
|
||||
|
||||
|
||||
def _norm_title(value) -> str:
|
||||
"""Alphanumeric-only lowercase form, so "Song (Remaster)" vs "song remaster"
|
||||
style drift between the download tag and the media-server tag still agrees."""
|
||||
return ''.join(ch for ch in str(value or '').lower() if ch.isalnum())
|
||||
|
||||
|
||||
def like_filename_filter(basename: str) -> str:
|
||||
"""A ``LIKE ... ESCAPE '\\'`` pattern that coarsely matches rows whose path ends
|
||||
in ``basename``. Escapes the LIKE metacharacters (``%`` ``_`` ``\\``) — filenames
|
||||
routinely contain underscores. Callers MUST still confirm with an exact basename
|
||||
compare (``pick_history_row`` does), since ``'%name'`` also matches ``'xname'``."""
|
||||
esc = basename.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_')
|
||||
return '%' + esc
|
||||
|
||||
|
||||
def pick_history_row(candidates: Sequence[Tuple], *, current_paths: Iterable[str],
|
||||
basename: str, title: str) -> Optional[int]:
|
||||
"""Return the id of the history row to update for this file, or None.
|
||||
|
||||
``candidates``: ``(id, file_path, title, download_source)`` rows the DB pre-filtered
|
||||
(exact path or filename LIKE). A row matches when its path equals the current path OR
|
||||
its filename matches AND its title agrees — the title guard prevents a shared filename
|
||||
("01 - Intro.flac") from healing a different song's row. Among matches a REAL download
|
||||
row is preferred over a synthetic ``acoustid_scan`` row, so the scanner heals the
|
||||
genuine record and the caller can delete the synthetic duplicate. None when nothing
|
||||
matches safely (caller then inserts a fresh row — the "file SoulSync never downloaded"
|
||||
intent)."""
|
||||
paths = {p for p in current_paths if p}
|
||||
want = _norm_title(title)
|
||||
matches: list = [] # (id, is_exact, is_real)
|
||||
for cid, cpath, ctitle, csource in candidates:
|
||||
is_real = csource != 'acoustid_scan'
|
||||
if cpath and cpath in paths:
|
||||
matches.append((cid, True, is_real))
|
||||
elif (basename and cpath and os.path.basename(cpath) == basename
|
||||
and (not want or not _norm_title(ctitle) or _norm_title(ctitle) == want)):
|
||||
matches.append((cid, False, is_real))
|
||||
if not matches:
|
||||
return None
|
||||
# Prefer a REAL download row over a synthetic acoustid_scan row; within that, prefer an
|
||||
# exact-path match over a filename match. Stable, so ties keep DB order (first/oldest id).
|
||||
matches.sort(key=lambda m: (m[2], m[1]), reverse=True)
|
||||
return matches[0][0]
|
||||
|
|
@ -26,6 +26,7 @@ Lifted verbatim from web_server.py. Dependencies injected via
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import traceback
|
||||
|
|
@ -44,6 +45,27 @@ from core.runtime_state import (
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# A task that has been in 'post_processing' longer than this is treated as stuck.
|
||||
# Post-processing (AcoustID + quality + import) is serialized, so a large batch
|
||||
# legitimately backs up — keep this generous so genuinely-slow imports aren't
|
||||
# cut off mid-flight (the old 5-min cutoff falsely "completed" queued tasks).
|
||||
_POST_PROCESSING_STUCK_TIMEOUT = 1800 # 30 minutes
|
||||
|
||||
|
||||
def _resolve_stuck_post_processing_status(task: dict) -> str:
|
||||
"""Decide the terminal status for a task stuck in post_processing.
|
||||
|
||||
Only call it 'completed' if the import actually produced a file on disk
|
||||
(``final_file_path`` is set at the end of successful post-processing). Without
|
||||
a real file, force-completing is a lie — the task shows as a downloaded track
|
||||
that isn't anywhere. Mark those 'failed' so they're retryable and honest.
|
||||
"""
|
||||
final_path = task.get('final_file_path')
|
||||
if final_path and os.path.exists(final_path):
|
||||
return 'completed'
|
||||
return 'failed'
|
||||
|
||||
|
||||
def _safe_batch_dirname(batch_id: str) -> str:
|
||||
safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch'))
|
||||
return safe or 'batch'
|
||||
|
|
@ -435,9 +457,15 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life
|
|||
retrying_count += 1
|
||||
elif task_status == 'post_processing':
|
||||
task_age = current_time - task.get('status_change_time', current_time)
|
||||
if task_age > 300: # 5 minutes (post-processing should be fast)
|
||||
logger.info(f"⏰ [Stuck Detection] Task {queue_task_id} stuck in post_processing for {task_age:.0f}s - forcing completion")
|
||||
task['status'] = 'completed' # Assume it worked if file verification is taking too long
|
||||
if task_age > _POST_PROCESSING_STUCK_TIMEOUT:
|
||||
new_status = _resolve_stuck_post_processing_status(task)
|
||||
if new_status == 'completed':
|
||||
logger.info(f"⏰ [Stuck Detection] Task {queue_task_id} stuck in post_processing for {task_age:.0f}s but file exists — completing")
|
||||
task['status'] = 'completed'
|
||||
else:
|
||||
logger.warning(f"⏰ [Stuck Detection] Task {queue_task_id} stuck in post_processing for {task_age:.0f}s with no output file — marking failed")
|
||||
task['status'] = 'failed'
|
||||
task['error_message'] = 'Post-processing timed out without producing a file'
|
||||
finished_count += 1
|
||||
else:
|
||||
retrying_count += 1
|
||||
|
|
@ -667,9 +695,15 @@ def check_batch_completion_v2(batch_id: str, deps: LifecycleDeps) -> Optional[bo
|
|||
retrying_count += 1
|
||||
elif task_status == 'post_processing':
|
||||
task_age = current_time - task.get('status_change_time', current_time)
|
||||
if task_age > 300: # 5 minutes (post-processing should be fast)
|
||||
logger.info(f"⏰ [Stuck Detection V2] Task {task_id} stuck in post_processing for {task_age:.0f}s - forcing completion")
|
||||
task['status'] = 'completed' # Assume it worked if file verification is taking too long
|
||||
if task_age > _POST_PROCESSING_STUCK_TIMEOUT:
|
||||
new_status = _resolve_stuck_post_processing_status(task)
|
||||
if new_status == 'completed':
|
||||
logger.info(f"⏰ [Stuck Detection V2] Task {task_id} stuck in post_processing for {task_age:.0f}s but file exists — completing")
|
||||
task['status'] = 'completed'
|
||||
else:
|
||||
logger.warning(f"⏰ [Stuck Detection V2] Task {task_id} stuck in post_processing for {task_age:.0f}s with no output file — marking failed")
|
||||
task['status'] = 'failed'
|
||||
task['error_message'] = 'Post-processing timed out without producing a file'
|
||||
finished_count += 1
|
||||
else:
|
||||
retrying_count += 1
|
||||
|
|
|
|||
|
|
@ -760,7 +760,8 @@ class WebUIDownloadMonitor:
|
|||
# used_sources keys are formatted as "{username}_{filename}", so startswith is exact.
|
||||
is_tidal = any(s.startswith('tidal_') for s in tried_sources)
|
||||
if is_tidal:
|
||||
tidal_quality = config_manager.get('tidal_download.quality', 'lossless')
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
tidal_quality = quality_tier_for_source('tidal', default='lossless')
|
||||
allow_fb = config_manager.get('tidal_download.allow_fallback', True)
|
||||
if tidal_quality == 'hires' and not allow_fb:
|
||||
task['error_message'] = (
|
||||
|
|
|
|||
55
core/downloads/orphan_history.py
Normal file
55
core/downloads/orphan_history.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
"""Identify dead review-queue history rows whose file is gone (#934 follow-up).
|
||||
|
||||
The Unverified/Quarantine review queue is fed from ``library_history`` — an
|
||||
append-only log that is never pruned. When a file is deleted, replaced, or
|
||||
re-downloaded elsewhere, its old ``unverified`` row lingers forever and can
|
||||
never be healed (there's no file left to confirm). Those are *orphans*.
|
||||
|
||||
This decides which rows are orphans, given a ``resolve(row) -> path | None``
|
||||
the caller wires to the real filesystem lookup. Pure (no DB, no filesystem) so
|
||||
the rules — including the safety gate — are unit-testable.
|
||||
|
||||
Safety gate: a filesystem check mass-false-positives when the library mount is
|
||||
down (every file looks missing). So if EVERY reviewed file is unreachable and
|
||||
there are enough rows to judge, we flag it ``suspicious`` and the caller refuses
|
||||
to delete — better to clean nothing than to wipe a healthy log during an outage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Sequence
|
||||
|
||||
|
||||
def find_orphan_history_ids(
|
||||
rows: Sequence[dict],
|
||||
resolve: Callable[[dict], Any],
|
||||
*,
|
||||
min_for_safety: int = 5,
|
||||
deletable: Callable[[dict], bool] | None = None,
|
||||
) -> dict:
|
||||
"""Return ``{'orphan_ids', 'checked', 'suspicious'}``.
|
||||
|
||||
A row is an orphan when it has a non-empty ``file_path`` but ``resolve`` can
|
||||
find no file for it. ``suspicious`` is True when every checked row is
|
||||
missing and there are at least ``min_for_safety`` of them — the mount-down
|
||||
signature; the caller should refuse to delete in that case.
|
||||
|
||||
``deletable`` (optional) protects rows from removal WITHOUT weakening the
|
||||
safety gate: a protected row still counts toward ``checked`` and the
|
||||
all-missing signal (so e.g. a few unverified orphans can't be swept during a
|
||||
mount outage just because protected rows were filtered out first), but it
|
||||
never appears in ``orphan_ids``. Default: every missing row is deletable.
|
||||
"""
|
||||
orphan_ids = []
|
||||
checked = 0
|
||||
missing = 0
|
||||
for row in rows:
|
||||
if not str((row.get('file_path') or '')).strip():
|
||||
continue
|
||||
checked += 1
|
||||
if resolve(row) is None:
|
||||
missing += 1
|
||||
if deletable is None or deletable(row):
|
||||
orphan_ids.append(row.get('id'))
|
||||
suspicious = checked >= min_for_safety and missing == checked
|
||||
return {'orphan_ids': orphan_ids, 'checked': checked, 'suspicious': suspicious}
|
||||
|
|
@ -171,6 +171,21 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
|
|||
logger.info(f"[Post-Processing] Task {task_id} already completed by stream processor, skipping verification")
|
||||
return
|
||||
|
||||
# RACE GUARD: the monitor sets status -> 'post_processing' immediately
|
||||
# before submitting this worker. If the status is now anything else, the
|
||||
# browser-poll post-processor already took ownership of this task — e.g.
|
||||
# it quarantined the file and requeued the next-best candidate (status
|
||||
# -> 'searching', source identity cleared). Bail WITHOUT marking failed
|
||||
# or notifying batch completion: otherwise we clobber that in-flight
|
||||
# retry with a bogus "missing file or source information" failure while a
|
||||
# parallel attempt is importing the song.
|
||||
if task['status'] != 'post_processing':
|
||||
logger.info(
|
||||
f"[Post-Processing] Task {task_id} no longer in 'post_processing' "
|
||||
f"(now '{task['status']}') — another path took over, skipping"
|
||||
)
|
||||
return
|
||||
|
||||
# Extract file information for verification
|
||||
track_info = task.get('track_info', {})
|
||||
task_filename = task.get('filename') or track_info.get('filename')
|
||||
|
|
@ -438,7 +453,19 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
|
|||
logger.error(f"[Post-Processing] Task {task_id} was completed by stream processor - not marking as failed")
|
||||
return
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f'File not found on disk after {_file_search_max_retries} search attempts. Expected: {os.path.basename(task_filename)}'
|
||||
# slskd reported the transfer complete, but the finder never located
|
||||
# the file under the configured download folder. Name the folder we
|
||||
# searched and the two real causes — "still being written" (timing)
|
||||
# or "SoulSync's download path doesn't match slskd's" (the classic
|
||||
# standalone config mismatch) — so the user can self-diagnose instead
|
||||
# of getting an opaque "not found". (Discord: Shdjfgatdif.)
|
||||
_searched_name = os.path.basename((task_filename or '').replace('\\', '/')) or task_filename
|
||||
download_tasks[task_id]['error_message'] = (
|
||||
f"slskd reported '{_searched_name}' downloaded, but it never appeared "
|
||||
f"under the download folder ({download_dir}) after {_file_search_max_retries} "
|
||||
f"checks. Either it's still being written, or SoulSync's download path "
|
||||
f"doesn't match slskd's download directory — they must point at the same folder."
|
||||
)
|
||||
deps.on_download_completed(batch_id, task_id, False)
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -94,6 +94,10 @@ class StatusDeps:
|
|||
run_async: Optional[Callable] = None
|
||||
on_download_completed: Optional[Callable[[str, str, bool], None]] = None
|
||||
get_persistent_download_history: Optional[Callable[[int], list[dict]]] = None
|
||||
# Returns ALL library_history rows with verification_status in
|
||||
# ('unverified', 'force_imported') — no recency limit, so historical
|
||||
# entries are never buried by the general history tail cap.
|
||||
get_unverified_download_history: Optional[Callable[[], list[dict]]] = None
|
||||
|
||||
|
||||
# Streaming sources the engine fallback applies to. Soulseek goes through
|
||||
|
|
@ -796,6 +800,13 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
'progress': progress,
|
||||
'error': task.get('error_message'),
|
||||
'verification_status': task.get('verification_status'),
|
||||
# library_history row id (set at import) so the Unverified review
|
||||
# queue can act on a still-live completed task before it becomes
|
||||
# a persistent-history row.
|
||||
'history_id': task.get('history_id'),
|
||||
# Real probed audio quality (mutagen-read from the actual file),
|
||||
# surfaced so the Downloads page can show what was downloaded.
|
||||
'quality': task.get('quality') or '',
|
||||
'retry_info': task.get('retry_info'),
|
||||
'retry_trigger': task.get('retry_trigger'),
|
||||
'batch_id': batch_id,
|
||||
|
|
@ -812,6 +823,32 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
'is_persistent_history': False,
|
||||
})
|
||||
|
||||
# --- Unverified history (unconditional, no limit) ---
|
||||
# Always load every library_history row that still needs human confirmation
|
||||
# (verification_status IN ('unverified', 'force_imported')). This is NOT
|
||||
# gated on len(items) < limit so that historical entries from past batches
|
||||
# are visible even during a large active batch that would otherwise exhaust
|
||||
# the limit before the history tail is read. Dedup against live tasks by
|
||||
# identity so a track currently in post-processing isn't shown twice.
|
||||
if deps.get_unverified_download_history is not None:
|
||||
try:
|
||||
unverified_entries = deps.get_unverified_download_history() or []
|
||||
except Exception as exc:
|
||||
logger.debug("[Downloads] unverified history lookup failed: %s", exc)
|
||||
unverified_entries = []
|
||||
|
||||
for entry in unverified_entries:
|
||||
item = _build_history_download_item(entry)
|
||||
identity = _download_identity(item.get('title'), item.get('artist'), item.get('album'))
|
||||
if identity in live_identities:
|
||||
continue
|
||||
items.append(item)
|
||||
live_identities.add(identity)
|
||||
|
||||
# --- General recent-history tail (capped, recency-ordered) ---
|
||||
# Fills in the completed/verified tail so the full Downloads list looks
|
||||
# populated. Gated on len(items) < limit so a busy batch doesn't trigger
|
||||
# an extra DB round-trip when we're already at capacity.
|
||||
if deps.get_persistent_download_history is not None and len(items) < limit:
|
||||
history_limit = min(limit - len(items), _PERSISTENT_HISTORY_TAIL_LIMIT)
|
||||
try:
|
||||
|
|
@ -832,7 +869,14 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
live_identities.add(identity)
|
||||
appended_history += 1
|
||||
|
||||
# Sort: active first (by priority), then by timestamp desc within each group
|
||||
# Sort: active first (by priority), then by timestamp desc within each group.
|
||||
# NOTE: the array order is presentation-only — the Downloads page filters
|
||||
# client-side per tab. What matters is that EVERY live task is present: an
|
||||
# earlier `items[:limit]` truncation (active-first) starved completed/failed/
|
||||
# unverified rows off the end during a busy batch, so those tabs stayed empty
|
||||
# until the batch drained. `limit` now bounds only the persistent-history
|
||||
# tail (handled above); live in-memory tasks are always returned in full
|
||||
# (they're already bounded by the 5-min cleanup automation).
|
||||
items.sort(key=lambda x: (x['priority'], -x['timestamp']))
|
||||
|
||||
# Build batch summaries for the batch context panel
|
||||
|
|
@ -860,7 +904,7 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
|
||||
return {
|
||||
'success': True,
|
||||
'downloads': items[:limit],
|
||||
'downloads': items,
|
||||
'total': len(items),
|
||||
'batches': batch_summaries,
|
||||
'timestamp': time.time(),
|
||||
|
|
|
|||
|
|
@ -54,6 +54,37 @@ def _cand_user_file(candidate):
|
|||
return getattr(candidate, 'username', None), getattr(candidate, 'filename', None)
|
||||
|
||||
|
||||
def _candidate_ordering():
|
||||
"""Return ``(quality_first, targets)`` for the active search mode + toggle.
|
||||
|
||||
The candidate walk is ordered by the user's profile quality rank
|
||||
(best→worst) instead of confidence-first when EITHER:
|
||||
- best-quality search mode is active (always quality-first), OR
|
||||
- priority mode and the ``rank_candidates_by_quality`` toggle is on
|
||||
(opt-in; default off keeps the byte-for-byte confidence-first walk).
|
||||
|
||||
Quality-first ordering also makes the version-mismatch force-import pick
|
||||
the highest-quality candidate, because that fallback accepts the
|
||||
first-tried (= best-ordered) quarantined entry.
|
||||
|
||||
Fails closed to confidence-first ordering on any error so a profile/DB
|
||||
hiccup never blocks a download. See
|
||||
docs/superpowers/specs/2026-06-14-best-quality-search-mode-design.md.
|
||||
"""
|
||||
try:
|
||||
from core.quality.selection import (
|
||||
load_search_mode,
|
||||
load_profile_targets,
|
||||
load_rank_candidates_by_quality,
|
||||
)
|
||||
if load_search_mode() == 'best_quality' or load_rank_candidates_by_quality():
|
||||
targets, _ = load_profile_targets()
|
||||
return True, targets
|
||||
except Exception as exc:
|
||||
logger.debug("[Modal Worker] quality ordering unavailable: %s", exc)
|
||||
return False, None
|
||||
|
||||
|
||||
def _try_cached_candidates(task_id, batch_id, track, deps):
|
||||
"""Quarantine-retry fast path: attempt the already-found candidates before
|
||||
re-searching anything.
|
||||
|
|
@ -91,7 +122,10 @@ def _try_cached_candidates(task_id, batch_id, track, deps):
|
|||
f"[Modal Worker] Quarantine retry: trying {len(remaining)} cached "
|
||||
f"candidate(s) before re-searching (task {task_id})"
|
||||
)
|
||||
return deps.attempt_download_with_candidates(task_id, remaining, track, batch_id)
|
||||
_qf, _qt = _candidate_ordering()
|
||||
return deps.attempt_download_with_candidates(
|
||||
task_id, remaining, track, batch_id, quality_first=_qf, quality_targets=_qt,
|
||||
)
|
||||
|
||||
|
||||
def _private_album_bundle_staging_miss_reason(batch_id: Optional[str], deps: Any) -> Optional[str]:
|
||||
|
|
@ -369,6 +403,11 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
logger.info(f"[Modal Worker] Generated {len(search_queries)} smart search queries for '{track.name}': {search_queries}")
|
||||
logger.info(f"[Modal Worker] About to start search loop for task {task_id} (track: '{track.name}')")
|
||||
|
||||
# Best-quality search mode: the orchestrator already pooled candidates
|
||||
# across every source for each query, so order the candidate walk by the
|
||||
# user's profile quality rank (best→worst). Computed once per task.
|
||||
_best_quality, _quality_targets = _candidate_ordering()
|
||||
|
||||
# 2. Sequential Query Search (matches GUI's start_search_worker_parallel logic)
|
||||
search_diagnostics = [] # Track what happened per query for detailed error messages
|
||||
all_raw_results = [] # Collect raw results across queries for candidate review modal
|
||||
|
|
@ -495,7 +534,10 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
download_tasks[task_id]['cached_candidates'] = candidates
|
||||
|
||||
# Try to download with these candidates
|
||||
success = deps.attempt_download_with_candidates(task_id, candidates, track, batch_id)
|
||||
success = deps.attempt_download_with_candidates(
|
||||
task_id, candidates, track, batch_id,
|
||||
quality_first=_best_quality, quality_targets=_quality_targets,
|
||||
)
|
||||
if success:
|
||||
# Download initiated successfully - let the download monitoring system handle completion
|
||||
if batch_id:
|
||||
|
|
@ -529,7 +571,10 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
# === HYBRID FALLBACK: If primary source failed, try remaining sources directly ===
|
||||
# The orchestrator's hybrid search stops at the first source with results, even if
|
||||
# those results all fail quality filtering. Try remaining sources individually.
|
||||
if getattr(deps.download_orchestrator, 'mode', '') == 'hybrid':
|
||||
#
|
||||
# Best-quality mode already searched EVERY source per query (the pool), so this
|
||||
# block would only re-search the same sources — skip it there.
|
||||
if not _best_quality and getattr(deps.download_orchestrator, 'mode', '') == 'hybrid':
|
||||
try:
|
||||
orch = deps.download_orchestrator
|
||||
hybrid_order = getattr(orch, 'hybrid_order', None) or []
|
||||
|
|
|
|||
|
|
@ -13,9 +13,48 @@ Pure + import-safe: parsing only, no network.
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Optional, Tuple
|
||||
from typing import Any, List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
def linked_track_id(track: Any) -> str:
|
||||
"""The source track id stamped on a search result, read from
|
||||
``_source_metadata['track_id']`` — the field every ID-downloadable source
|
||||
(Tidal, Qobuz) records. Empty string when absent. ``TrackResult`` has no
|
||||
top-level ``id``, so callers must NOT use ``getattr(t, 'id')`` (that always
|
||||
missed and left the pasted-link bubble a silent no-op — #932)."""
|
||||
meta = getattr(track, '_source_metadata', None)
|
||||
if not isinstance(meta, dict):
|
||||
return ''
|
||||
return str(meta.get('track_id') or '')
|
||||
|
||||
|
||||
def bubble_linked_track_first(tracks: List[Any], link_track_id: str) -> List[Any]:
|
||||
"""Float the result whose source id matches a pasted link to the top so the
|
||||
user sees the EXACT track they linked, not a fuzzy text-search lookalike
|
||||
(#813/#932). Stable + a graceful no-op when no result carries the id."""
|
||||
if not link_track_id or not tracks:
|
||||
return tracks
|
||||
target = str(link_track_id)
|
||||
return sorted(tracks, key=lambda t: linked_track_id(t) != target)
|
||||
|
||||
|
||||
def inject_linked_track_first(
|
||||
tracks: List[Any], linked_result: Any, link_track_id: str
|
||||
) -> List[Any]:
|
||||
"""Put the EXACT linked track first.
|
||||
|
||||
When ``linked_result`` is the track fetched directly by id, prepend it and
|
||||
drop any search duplicate of it — so an obscure track a text search never
|
||||
surfaced is still present and downloadable (#932). When it's None (the source
|
||||
can't fetch one), fall back to bubbling a matching search result. Pure."""
|
||||
if not link_track_id:
|
||||
return tracks
|
||||
target = str(link_track_id)
|
||||
if linked_result is not None:
|
||||
return [linked_result] + [t for t in tracks if linked_track_id(t) != target]
|
||||
return bubble_linked_track_first(tracks, target)
|
||||
|
||||
# host substring → download source id. Only ID-downloadable streaming sources.
|
||||
_HOSTS = (
|
||||
('tidal.com', 'tidal'),
|
||||
|
|
|
|||
|
|
@ -93,6 +93,56 @@ def _backfill_album_context(
|
|||
album_context['image_url'] = first['url']
|
||||
|
||||
|
||||
# Placeholder album ids used when no real source album id is known — never queryable.
|
||||
_SENTINEL_ALBUM_IDS = {'explicit_album', 'from_sync_modal', ''}
|
||||
|
||||
|
||||
def backfill_album_context_from_source(
|
||||
album_context: Dict[str, Any],
|
||||
primary_source: Optional[str],
|
||||
get_album_for_source_fn: Any,
|
||||
) -> bool:
|
||||
"""Hydrate a lean album context from the user's PRIMARY metadata source (#915).
|
||||
|
||||
Post-processing's only album backfill (:func:`hydrate_download_metadata`) goes through
|
||||
``spotify_client.get_track_details`` — Spotify-only. An iTunes/Deezer-primary user's
|
||||
download therefore kept a lean context (no ``release_date``), so the path dropped the
|
||||
``$year`` and the date defaulted to ``YYYY-01-01`` — until they ran a Reorganize, which
|
||||
reads the full album from the PRIMARY source. This closes that gap by doing the same:
|
||||
fetch the full album from the primary source and backfill, so a download's pathing/tags
|
||||
match what a later reorganize would produce.
|
||||
|
||||
``get_album_for_source_fn(source, album_id)`` is injected (the real one is
|
||||
``core.metadata.album_tracks.get_album_for_source``) so this stays pure + testable.
|
||||
No-op when: the context is already complete; the primary source is spotify (the existing
|
||||
track-details path covers it); or no real source album id is present. Returns True when
|
||||
it filled anything. Never raises — a backfill failure must not break a download.
|
||||
"""
|
||||
if not isinstance(album_context, dict) or not _album_is_lean(album_context):
|
||||
return False
|
||||
if not primary_source or primary_source == 'spotify':
|
||||
return False
|
||||
album_id = album_context.get('id')
|
||||
if not album_id or str(album_id) in _SENTINEL_ALBUM_IDS:
|
||||
return False
|
||||
try:
|
||||
album = get_album_for_source_fn(primary_source, str(album_id))
|
||||
except Exception as e: # noqa: BLE001 — defensive: never let backfill break a download
|
||||
logger.warning("[Context] primary-source (%s) album backfill failed: %s", primary_source, e)
|
||||
return False
|
||||
if not isinstance(album, dict):
|
||||
return False
|
||||
before = album_context.get('release_date')
|
||||
_backfill_album_context(album_context, {'album': album})
|
||||
if album_context.get('release_date') and album_context.get('release_date') != before:
|
||||
logger.info(
|
||||
"[Context] Hydrated lean album context from primary source %s "
|
||||
"(release_date=%r, total_tracks=%r)",
|
||||
primary_source, album_context.get('release_date'), album_context.get('total_tracks'),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def hydrate_download_metadata(
|
||||
track: Any,
|
||||
track_info: Any,
|
||||
|
|
@ -172,4 +222,5 @@ def hydrate_download_metadata(
|
|||
__all__ = [
|
||||
'ResolvedTrackMetadata',
|
||||
'hydrate_download_metadata',
|
||||
'backfill_album_context_from_source',
|
||||
]
|
||||
|
|
|
|||
364
core/exports/export_sources.py
Normal file
364
core/exports/export_sources.py
Normal file
|
|
@ -0,0 +1,364 @@
|
|||
"""Wire the real cheapest-first sources for the export MBID waterfall (#903).
|
||||
|
||||
``mbid_resolver`` is the pure waterfall; this module supplies the real I/O behind each
|
||||
source and assembles the ``resolve_fn`` the export job uses:
|
||||
|
||||
1. **cache** — ``recording_mbid_cache`` (persistent (artist,title)->mbid).
|
||||
2. **DB** — a text-matched library track's ``tracks.musicbrainz_recording_id``.
|
||||
3. **file** — ``MUSICBRAINZ_RECORDING_ID`` tag of that track's file (when the DB row had
|
||||
no recording id but the file was tagged on import).
|
||||
4. **MusicBrainz** — live ``match_recording(track, artist)`` (rate-limited tail).
|
||||
|
||||
Every source is wrapped so any failure (missing table, unreadable file, MB timeout) returns
|
||||
None — the waterfall just falls through, the export never breaks. ``build_resolve_fn`` also
|
||||
writes a fresh non-cache hit back to the cache so the next export of the same song is free.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
from core.exports.mbid_resolver import (
|
||||
SRC_CACHE,
|
||||
SRC_DB,
|
||||
SRC_FILE,
|
||||
SRC_MUSICBRAINZ,
|
||||
normalize_key,
|
||||
resolve_recording_mbid,
|
||||
)
|
||||
|
||||
logger = get_logger("exports.export_sources")
|
||||
|
||||
|
||||
def _db_match(artist: str, title: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Text-match a library track by (artist, title); return (recording_mbid, file_path).
|
||||
Either may be None. Fail-safe — any DB error returns (None, None)."""
|
||||
if not title:
|
||||
return (None, None)
|
||||
try:
|
||||
from database.music_database import get_database
|
||||
db = get_database()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"SELECT t.musicbrainz_recording_id, t.file_path "
|
||||
"FROM tracks t JOIN artists a ON t.artist_id = a.id "
|
||||
"WHERE LOWER(t.title) = LOWER(?) AND LOWER(a.name) = LOWER(?) "
|
||||
"LIMIT 1",
|
||||
(title, artist),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return (None, None)
|
||||
mbid = row[0] if not hasattr(row, "keys") else row["musicbrainz_recording_id"]
|
||||
fpath = row[1] if not hasattr(row, "keys") else row["file_path"]
|
||||
return ((mbid or None), (fpath or None))
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug(f"export db_match failed for '{artist} - {title}': {exc}")
|
||||
return (None, None)
|
||||
|
||||
|
||||
def db_recording_mbid(artist: str, title: str) -> Optional[str]:
|
||||
"""Recording MBID stored on a matched library track (``musicbrainz_recording_id``)."""
|
||||
return _db_match(artist, title)[0]
|
||||
|
||||
|
||||
# Service → the tracks-table column carrying that service's track ID (set by enrichment).
|
||||
# Trusted constants — never user input — so safe to interpolate into the SELECT below.
|
||||
_SERVICE_ID_COLUMNS = {"spotify": "spotify_track_id", "deezer": "deezer_id"}
|
||||
|
||||
|
||||
def db_service_track_id(artist: str, title: str, service: str) -> Optional[str]:
|
||||
"""The service track ID (``spotify_track_id`` / ``deezer_id``) stored on a matched
|
||||
library track — what lets a mirrored playlist be exported BACK to Spotify/Deezer
|
||||
without re-searching, since enrichment already pinned it (#945). Text-matches by
|
||||
(artist, title), same as the MBID resolver. Fail-safe: any miss/error returns None."""
|
||||
column = _SERVICE_ID_COLUMNS.get((service or "").lower())
|
||||
if not column or not title:
|
||||
return None
|
||||
try:
|
||||
from database.music_database import get_database
|
||||
db = get_database()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
f"SELECT t.{column} FROM tracks t JOIN artists a ON t.artist_id = a.id "
|
||||
"WHERE LOWER(t.title) = LOWER(?) AND LOWER(a.name) = LOWER(?) LIMIT 1",
|
||||
(title, artist),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
val = row[0] if not hasattr(row, "keys") else row[column]
|
||||
return val or None
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug(f"export service-id lookup failed for '{artist} - {title}' ({service}): {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def build_service_resolve_fn(service: str) -> Callable[[str, str], Tuple[Optional[str], Optional[str]]]:
|
||||
"""resolve_fn for service-playlist export: ``(artist, title) -> (service_track_id, 'library')``.
|
||||
Plugs into ``resolve_playlist_tracks(..., id_key='service_track_id')`` exactly like the
|
||||
MBID resolver plugs in for ListenBrainz."""
|
||||
def resolve_fn(artist: str, title: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
tid = db_service_track_id(artist, title, service)
|
||||
return (tid, "library" if tid else None)
|
||||
return resolve_fn
|
||||
|
||||
|
||||
def service_id_from_extra_data(track: Any, service: str) -> Optional[str]:
|
||||
"""The target-service track ID the DISCOVERY step already resolved for this mirrored
|
||||
track, read from its ``extra_data`` blob (#945 — Boulder: "all 50 are discovered to
|
||||
Deezer already, it's not using any of that"). This is free (no API call) and reliable
|
||||
(it's the same id used to mirror the track).
|
||||
|
||||
Only trusted when the track was discovered ON the export's target service — a
|
||||
Deezer-discovered track carries a Deezer id under ``matched_data['id']``, and its
|
||||
``provider`` is the service name. A ``wing_it_fallback`` provider (the low-confidence
|
||||
guess path) deliberately does NOT match here, so those fall through to the library/
|
||||
none path rather than risk a wrong track in the exported playlist."""
|
||||
raw = track.get("extra_data") if isinstance(track, dict) else None
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(raw) if isinstance(raw, str) else raw
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(data, dict) or not data.get("discovered"):
|
||||
return None
|
||||
if str(data.get("provider") or "").lower() != str(service or "").lower():
|
||||
return None
|
||||
matched = data.get("matched_data")
|
||||
tid = matched.get("id") if isinstance(matched, dict) else None
|
||||
return str(tid) if tid else None
|
||||
|
||||
|
||||
def _track_field(track: Dict[str, Any], *names: str) -> str:
|
||||
for n in names:
|
||||
v = track.get(n)
|
||||
if v:
|
||||
return str(v)
|
||||
return ""
|
||||
|
||||
|
||||
def resolve_service_track_ids(
|
||||
tracks: List[Dict[str, Any]],
|
||||
service: str,
|
||||
*,
|
||||
db_fn: Optional[Callable[[str, str, str], Optional[str]]] = None,
|
||||
search_id_fn: Optional[Callable[[str, str], Optional[str]]] = None,
|
||||
on_progress: Optional[Callable[[int, int, Dict[str, Any]], None]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Resolve a mirrored playlist's tracks to target-service track IDs for export.
|
||||
|
||||
Waterfall per track: the discovery cache (``extra_data`` — free + already confidently
|
||||
matched) → the library track's stored service id → (only when ``search_id_fn`` is
|
||||
given, i.e. the opt-in backfill toggle) a confident live-search match. A track that
|
||||
clears none of these is reported unmatched (caller skips it — never a guessed/wrong
|
||||
id). Returns ``{"resolved": [{artist, title, album, service_track_id}], "stats":
|
||||
{...}}`` with ``from_cache`` / ``from_library`` / ``from_search`` / ``unmatched``
|
||||
tallies for the status display.
|
||||
"""
|
||||
db_fn = db_fn or db_service_track_id
|
||||
total = len(tracks or [])
|
||||
resolved: List[Dict[str, Any]] = []
|
||||
stats: Dict[str, Any] = {
|
||||
"total": total, "resolved": 0, "unmatched": 0,
|
||||
"from_cache": 0, "from_library": 0, "from_search": 0,
|
||||
}
|
||||
for i, t in enumerate(tracks or []):
|
||||
if not isinstance(t, dict):
|
||||
t = {}
|
||||
artist = _track_field(t, "artist", "artist_name", "creator")
|
||||
title = _track_field(t, "title", "track_name", "name")
|
||||
album = _track_field(t, "album", "album_name", "release_name")
|
||||
|
||||
tid = service_id_from_extra_data(t, service)
|
||||
if tid:
|
||||
stats["from_cache"] += 1
|
||||
else:
|
||||
tid = db_fn(artist, title, service)
|
||||
if tid:
|
||||
stats["from_library"] += 1
|
||||
elif search_id_fn is not None:
|
||||
tid = search_id_fn(artist, title)
|
||||
if tid:
|
||||
stats["from_search"] += 1
|
||||
|
||||
resolved.append({"artist": artist, "title": title, "album": album,
|
||||
"service_track_id": tid or None})
|
||||
stats["resolved" if tid else "unmatched"] += 1
|
||||
if on_progress is not None:
|
||||
try:
|
||||
on_progress(i + 1, total, stats)
|
||||
except Exception: # noqa: S110 — a progress error must never fail the export
|
||||
pass
|
||||
|
||||
return {"resolved": resolved, "stats": stats}
|
||||
|
||||
|
||||
# Confidence floor for backfill, on the score_track scale (~1.5 = exact title + exact
|
||||
# artist, thanks to the 1.5x artist boost). A cover/karaoke (x0.05) or a wrong-artist hit
|
||||
# (no boost, caps ~1.0) can't clear this — so backfill never adds a guessed/wrong version.
|
||||
BACKFILL_MIN_SCORE = 1.2
|
||||
|
||||
|
||||
def search_service_track_id(
|
||||
artist: str,
|
||||
title: str,
|
||||
*,
|
||||
search_fn: Callable[[str], List[Any]],
|
||||
min_score: float = BACKFILL_MIN_SCORE,
|
||||
) -> Optional[str]:
|
||||
"""Confident live-search match for export backfill (#945): search the target service
|
||||
for (artist, title), rerank by relevance, and return the top match's id ONLY if it
|
||||
clears the confidence floor. Below the floor → None: the track is left out of the
|
||||
export rather than risk a wrong/cover/karaoke version (the whole point of backfill is
|
||||
coverage WITHOUT the wrong-track risk). ``search_fn(query) -> List[Track]`` is injected
|
||||
so this is unit-testable without a live service."""
|
||||
if not title:
|
||||
return None
|
||||
from core.metadata.relevance import build_combined_search_query, filter_and_rerank
|
||||
query = build_combined_search_query(title, artist)
|
||||
try:
|
||||
candidates = list(search_fn(query) or [])
|
||||
except Exception as exc:
|
||||
logger.debug(f"export backfill search failed for '{artist} - {title}': {exc}")
|
||||
return None
|
||||
if not candidates:
|
||||
return None
|
||||
ranked = filter_and_rerank(
|
||||
candidates, expected_title=title, expected_artist=artist, min_score=min_score,
|
||||
)
|
||||
if not ranked:
|
||||
return None
|
||||
tid = getattr(ranked[0], "id", None)
|
||||
return str(tid) if tid else None
|
||||
|
||||
|
||||
def file_recording_mbid(artist: str, title: str) -> Optional[str]:
|
||||
"""Recording MBID read from the matched track's file tag (set on import post-processing)."""
|
||||
_mbid, fpath = _db_match(artist, title)
|
||||
if not fpath:
|
||||
return None
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(fpath)
|
||||
if audio is None or not getattr(audio, "tags", None):
|
||||
return None
|
||||
tags = audio.tags
|
||||
# ID3 UFID (MusicBrainz), Vorbis/MP4 musicbrainz_trackid, etc.
|
||||
for key in ("UFID:http://musicbrainz.org", "musicbrainz_trackid",
|
||||
"MUSICBRAINZ_TRACKID", "----:com.apple.iTunes:MusicBrainz Track Id"):
|
||||
try:
|
||||
val = tags.get(key)
|
||||
except Exception:
|
||||
val = None
|
||||
if not val:
|
||||
continue
|
||||
if hasattr(val, "data"): # ID3 UFID frame
|
||||
val = val.data.decode("utf-8", "ignore")
|
||||
if isinstance(val, (list, tuple)):
|
||||
val = val[0] if val else ""
|
||||
if isinstance(val, bytes):
|
||||
val = val.decode("utf-8", "ignore")
|
||||
val = str(val).strip()
|
||||
if val:
|
||||
return val
|
||||
except Exception as exc:
|
||||
logger.debug(f"export file_recording_mbid failed for {fpath}: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
_mb_service = None
|
||||
_mb_service_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_mb_service():
|
||||
"""Shared MusicBrainzService (client + cache + DB), created lazily so importing this
|
||||
module never triggers a DB/network connection on paths that don't export."""
|
||||
global _mb_service
|
||||
if _mb_service is None:
|
||||
with _mb_service_lock:
|
||||
if _mb_service is None:
|
||||
from core.musicbrainz_service import MusicBrainzService
|
||||
from database.music_database import get_database
|
||||
_mb_service = MusicBrainzService(get_database())
|
||||
return _mb_service
|
||||
|
||||
|
||||
def musicbrainz_recording_mbid(artist: str, title: str) -> Optional[str]:
|
||||
"""Live MusicBrainz ``match_recording`` — the rate-limited tail."""
|
||||
if not title:
|
||||
return None
|
||||
try:
|
||||
svc = _get_mb_service()
|
||||
if not svc:
|
||||
return None
|
||||
result = svc.match_recording(title, artist)
|
||||
if result and result.get("mbid"):
|
||||
return result["mbid"]
|
||||
except Exception as exc:
|
||||
logger.debug(f"export musicbrainz_recording_mbid failed for '{artist} - {title}': {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def build_resolve_fn(
|
||||
*,
|
||||
db_fn: Callable[[str, str], Optional[str]] = db_recording_mbid,
|
||||
file_fn: Callable[[str, str], Optional[str]] = file_recording_mbid,
|
||||
mb_fn: Callable[[str, str], Optional[str]] = musicbrainz_recording_mbid,
|
||||
cache_lookup: Optional[Callable[[str], Optional[str]]] = None,
|
||||
cache_record: Optional[Callable[[str, str], bool]] = None,
|
||||
) -> Callable[[str, str], Tuple[Optional[str], Optional[str]]]:
|
||||
"""Assemble the export ``resolve_fn(artist, title) -> (mbid, source_label)``.
|
||||
|
||||
Runs cache -> DB -> file -> MusicBrainz, and writes a fresh (non-cache) hit back to the
|
||||
persistent cache. All sources are injectable so the wiring is unit-testable; defaults
|
||||
use the real cache module.
|
||||
"""
|
||||
if cache_lookup is None or cache_record is None:
|
||||
from core.exports import recording_mbid_cache as _cache
|
||||
cache_lookup = cache_lookup or _cache.lookup
|
||||
cache_record = cache_record or _cache.record
|
||||
|
||||
def resolve_fn(artist: str, title: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
sources = [
|
||||
(SRC_CACHE, lambda a, t: cache_lookup(normalize_key(a, t))),
|
||||
(SRC_DB, db_fn),
|
||||
(SRC_FILE, file_fn),
|
||||
(SRC_MUSICBRAINZ, mb_fn),
|
||||
]
|
||||
mbid, label = resolve_recording_mbid(artist, title, sources)
|
||||
if mbid and label and label != SRC_CACHE:
|
||||
try:
|
||||
cache_record(normalize_key(artist, title), mbid)
|
||||
except Exception: # noqa: S110 — cache write is best-effort
|
||||
pass
|
||||
return (mbid, label)
|
||||
|
||||
return resolve_fn
|
||||
|
||||
|
||||
__all__ = [
|
||||
"build_resolve_fn",
|
||||
"db_recording_mbid",
|
||||
"file_recording_mbid",
|
||||
"musicbrainz_recording_mbid",
|
||||
]
|
||||
89
core/exports/jspf_export.py
Normal file
89
core/exports/jspf_export.py
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
"""Build a JSPF playlist (ListenBrainz-compatible) from resolved SoulSync tracks.
|
||||
|
||||
ListenBrainz's ``POST /1/playlist/create`` requires JSPF where **every track carries a
|
||||
``identifier`` of ``https://musicbrainz.org/recording/<recording-mbid>``** — text-only
|
||||
entries (title/creator alone) are rejected. So a track can only be exported once we've
|
||||
resolved its MusicBrainz *recording* MBID (see ``mbid_resolver``); tracks without one are
|
||||
dropped here and surfaced to the user as "unmatched".
|
||||
|
||||
Pure + I/O-free: callers pass already-resolved track dicts, this returns the JSPF dict
|
||||
(and a small coverage summary). The same JSPF is used for both the downloadable ``.jspf``
|
||||
file and the direct create-playlist POST, so there's one source of truth for the shape.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
MB_RECORDING_PREFIX = "https://musicbrainz.org/recording/"
|
||||
|
||||
# A MusicBrainz MBID is a canonical UUID. Validate to avoid emitting garbage identifiers
|
||||
# that LB would reject (or, worse, that silently point nowhere).
|
||||
_UUID_RE = re.compile(
|
||||
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def is_valid_recording_mbid(mbid: Any) -> bool:
|
||||
"""True when ``mbid`` is a well-formed MusicBrainz UUID."""
|
||||
return bool(mbid) and isinstance(mbid, str) and bool(_UUID_RE.match(mbid.strip()))
|
||||
|
||||
|
||||
def _track_entry(track: Dict[str, Any]) -> Dict[str, Any] | None:
|
||||
"""Build one JSPF track entry, or None if the track has no valid recording MBID."""
|
||||
mbid = (track.get("recording_mbid") or "").strip() if isinstance(track.get("recording_mbid"), str) else ""
|
||||
if not is_valid_recording_mbid(mbid):
|
||||
return None
|
||||
entry: Dict[str, Any] = {"identifier": f"{MB_RECORDING_PREFIX}{mbid}"}
|
||||
# Optional, human-friendly fields — LB ignores them on create but they make the
|
||||
# downloaded .jspf readable and round-trippable.
|
||||
if track.get("title"):
|
||||
entry["title"] = str(track["title"])
|
||||
if track.get("artist"):
|
||||
entry["creator"] = str(track["artist"])
|
||||
if track.get("album"):
|
||||
entry["album"] = str(track["album"])
|
||||
return entry
|
||||
|
||||
|
||||
def build_jspf(
|
||||
title: str,
|
||||
tracks: List[Dict[str, Any]],
|
||||
*,
|
||||
creator: str = "",
|
||||
) -> Tuple[Dict[str, Any], Dict[str, int]]:
|
||||
"""Build a ListenBrainz-compatible JSPF dict from resolved tracks.
|
||||
|
||||
``tracks`` is an ordered list of dicts with ``recording_mbid`` (required to be
|
||||
included), plus optional ``title`` / ``artist`` / ``album``. Tracks without a valid
|
||||
recording MBID are skipped (LB rejects them).
|
||||
|
||||
Returns ``(jspf, summary)`` where ``jspf`` is ``{"playlist": {...}}`` and ``summary``
|
||||
is ``{"total", "included", "skipped"}`` for the coverage display.
|
||||
"""
|
||||
jspf_tracks: List[Dict[str, Any]] = []
|
||||
for t in tracks or []:
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
entry = _track_entry(t)
|
||||
if entry is not None:
|
||||
jspf_tracks.append(entry)
|
||||
|
||||
playlist: Dict[str, Any] = {
|
||||
"title": (title or "SoulSync Export").strip() or "SoulSync Export",
|
||||
"track": jspf_tracks,
|
||||
}
|
||||
if creator:
|
||||
playlist["creator"] = str(creator)
|
||||
|
||||
total = sum(1 for t in (tracks or []) if isinstance(t, dict))
|
||||
summary = {
|
||||
"total": total,
|
||||
"included": len(jspf_tracks),
|
||||
"skipped": total - len(jspf_tracks),
|
||||
}
|
||||
return {"playlist": playlist}, summary
|
||||
|
||||
|
||||
__all__ = ["build_jspf", "is_valid_recording_mbid", "MB_RECORDING_PREFIX"]
|
||||
88
core/exports/mbid_resolver.py
Normal file
88
core/exports/mbid_resolver.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
"""Resolve a playlist track's MusicBrainz *recording* MBID, cheapest source first.
|
||||
|
||||
A ListenBrainz playlist export needs each track's recording MBID (``jspf_export``). A
|
||||
SoulSync track can supply it from several places, in increasing cost:
|
||||
|
||||
1. **resolution cache** — a prior (artist,title)->mbid result (persistent; reused across
|
||||
playlists and runs, so the same song never costs twice).
|
||||
2. **library DB** — ``tracks.musicbrainz_recording_id`` (set by the MusicBrainz
|
||||
enrichment worker).
|
||||
3. **file tags** — ``MUSICBRAINZ_RECORDING_ID`` written into the audio file on import
|
||||
post-processing (catches tracks enriched at import but not via the worker).
|
||||
4. **MusicBrainz lookup** — a live ``match_recording(artist, title)`` (rate-limited
|
||||
~1 req/s; the slow tail — only hit when 1–3 miss).
|
||||
|
||||
This module is the **pure waterfall**: the caller passes ordered ``(label, fn)`` sources,
|
||||
each ``fn(artist, title) -> mbid | None``, and ``resolve_recording_mbid`` returns the
|
||||
first valid hit plus its label (for the live status / stats). The actual I/O (DB query,
|
||||
mutagen read, MB request, cache read/write) lives in the export job that wires the real
|
||||
sources — so this stays trivially unit-testable and short-circuits correctly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Callable, List, Optional, Tuple
|
||||
|
||||
# Source labels (also used in the live-status breakdown).
|
||||
SRC_CACHE = "cache"
|
||||
SRC_DB = "db"
|
||||
SRC_FILE = "file"
|
||||
SRC_MUSICBRAINZ = "musicbrainz"
|
||||
SRC_NONE = None
|
||||
|
||||
_UUID_RE = re.compile(
|
||||
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE
|
||||
)
|
||||
|
||||
Source = Tuple[str, Callable[[str, str], Optional[str]]]
|
||||
|
||||
|
||||
def _valid(mbid: Any) -> Optional[str]:
|
||||
"""Return the trimmed MBID if it's a well-formed UUID, else None."""
|
||||
if not isinstance(mbid, str):
|
||||
return None
|
||||
m = mbid.strip()
|
||||
return m if _UUID_RE.match(m) else None
|
||||
|
||||
|
||||
def normalize_key(artist: Any, title: Any) -> str:
|
||||
"""Stable cache key for an (artist, title) pair — lower, punctuation-stripped,
|
||||
whitespace-collapsed — so trivial variations share a cache entry."""
|
||||
def _n(v: Any) -> str:
|
||||
s = re.sub(r"[^\w\s]", "", str(v or "").lower())
|
||||
return re.sub(r"\s+", " ", s).strip()
|
||||
return f"{_n(artist)}␟{_n(title)}"
|
||||
|
||||
|
||||
def resolve_recording_mbid(
|
||||
artist: str,
|
||||
title: str,
|
||||
sources: List[Source],
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Walk ``sources`` in order; return ``(mbid, label)`` of the first that yields a
|
||||
valid recording MBID, or ``(None, None)`` when every source misses.
|
||||
|
||||
Each source is ``(label, fn)`` and ``fn(artist, title)`` returns an MBID or None. A
|
||||
source that raises is treated as a miss (never aborts the waterfall) — so one flaky
|
||||
lookup (e.g. a MusicBrainz timeout) can't fail the whole export. Short-circuits: a
|
||||
later/expensive source isn't called once an earlier one hits.
|
||||
"""
|
||||
for label, fn in sources or []:
|
||||
try:
|
||||
mbid = _valid(fn(artist, title))
|
||||
except Exception:
|
||||
mbid = None
|
||||
if mbid:
|
||||
return (mbid, label)
|
||||
return (None, None)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"resolve_recording_mbid",
|
||||
"normalize_key",
|
||||
"SRC_CACHE",
|
||||
"SRC_DB",
|
||||
"SRC_FILE",
|
||||
"SRC_MUSICBRAINZ",
|
||||
]
|
||||
98
core/exports/playlist_export.py
Normal file
98
core/exports/playlist_export.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
"""Orchestrate resolving a playlist's tracks to recording MBIDs for export (#903).
|
||||
|
||||
This is the testable heart of the export job: walk the playlist's tracks, resolve each to a
|
||||
MusicBrainz recording MBID via an injected ``resolve_fn`` (which the job wires to the
|
||||
cache -> DB -> file -> MusicBrainz waterfall), dedup repeated songs within the run so they
|
||||
only cost one resolution, build the ordered "pseudo-playlist" of resolved tracks, and tally
|
||||
live stats (resolved / unmatched / per-source / deduped) for the on-card status display.
|
||||
|
||||
Pure: all I/O (DB, file reads, MusicBrainz, cache) is behind ``resolve_fn`` and the optional
|
||||
``on_progress`` callback, so the dedup + accounting logic is unit-testable without any
|
||||
network or database. The returned ``resolved`` list feeds straight into ``jspf_export``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from core.exports.mbid_resolver import normalize_key
|
||||
|
||||
# resolve_fn(artist, title) -> (recording_mbid|None, source_label|None)
|
||||
ResolveFn = Callable[[str, str], Tuple[Optional[str], Optional[str]]]
|
||||
ProgressFn = Callable[[int, int, Dict[str, Any]], None]
|
||||
|
||||
|
||||
def _field(track: Dict[str, Any], *names: str) -> str:
|
||||
"""First non-empty value among ``names`` (handles both playlist + LB-cache shapes)."""
|
||||
for n in names:
|
||||
v = track.get(n)
|
||||
if v:
|
||||
return str(v)
|
||||
return ""
|
||||
|
||||
|
||||
def resolve_playlist_tracks(
|
||||
tracks: List[Dict[str, Any]],
|
||||
resolve_fn: ResolveFn,
|
||||
*,
|
||||
on_progress: Optional[ProgressFn] = None,
|
||||
id_key: str = "recording_mbid",
|
||||
) -> Dict[str, Any]:
|
||||
"""Resolve every track to an ID and build the export pseudo-playlist.
|
||||
|
||||
``resolve_fn(artist, title) -> (id, source)`` returns whatever ID the target needs —
|
||||
a MusicBrainz recording MBID for ListenBrainz/JSPF (the default), or a Spotify/Deezer
|
||||
track ID for service export. ``id_key`` names the field that ID lands under in each
|
||||
resolved entry (defaults to ``recording_mbid`` so existing LB/JSPF callers are
|
||||
untouched). The dedup + stats + ordering logic is identical regardless of ID type.
|
||||
|
||||
``tracks`` items may use ``artist``/``artist_name`` and ``title``/``track_name`` and
|
||||
``album``/``album_name`` (both the mirrored-playlist and LB-cache shapes are accepted).
|
||||
|
||||
Returns ``{"resolved": [...], "stats": {...}}`` where each resolved entry is
|
||||
``{artist, title, album, <id_key>}`` (the ID is None when unmatched), in original
|
||||
order, and stats carries ``total, resolved, unmatched, deduped, by_source``.
|
||||
"""
|
||||
total = len(tracks or [])
|
||||
memo: Dict[str, Tuple[Optional[str], Optional[str]]] = {}
|
||||
resolved: List[Dict[str, Any]] = []
|
||||
stats: Dict[str, Any] = {
|
||||
"total": total, "resolved": 0, "unmatched": 0, "deduped": 0, "by_source": {},
|
||||
}
|
||||
|
||||
for i, t in enumerate(tracks or []):
|
||||
if not isinstance(t, dict):
|
||||
t = {}
|
||||
artist = _field(t, "artist", "artist_name", "creator")
|
||||
title = _field(t, "title", "track_name", "name")
|
||||
album = _field(t, "album", "album_name", "release_name")
|
||||
key = normalize_key(artist, title)
|
||||
|
||||
if key in memo:
|
||||
mbid, source = memo[key]
|
||||
stats["deduped"] += 1
|
||||
fresh = False
|
||||
else:
|
||||
mbid, source = resolve_fn(artist, title)
|
||||
memo[key] = (mbid, source)
|
||||
fresh = True
|
||||
|
||||
resolved.append({"artist": artist, "title": title, "album": album, id_key: mbid})
|
||||
|
||||
if mbid:
|
||||
stats["resolved"] += 1
|
||||
if fresh and source:
|
||||
stats["by_source"][source] = stats["by_source"].get(source, 0) + 1
|
||||
else:
|
||||
stats["unmatched"] += 1
|
||||
|
||||
if on_progress is not None:
|
||||
try:
|
||||
on_progress(i + 1, total, stats)
|
||||
except Exception: # noqa: S110 — a progress-display error must never fail the export
|
||||
pass
|
||||
|
||||
return {"resolved": resolved, "stats": stats}
|
||||
|
||||
|
||||
__all__ = ["resolve_playlist_tracks"]
|
||||
126
core/exports/recording_mbid_cache.py
Normal file
126
core/exports/recording_mbid_cache.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
"""Persistent (artist,title) -> MusicBrainz recording-MBID cache for playlist export.
|
||||
|
||||
The export waterfall (``core.exports.mbid_resolver``) ends in a live MusicBrainz lookup
|
||||
that's rate-limited to ~1 req/s — the slow tail of exporting a big playlist. Remembering a
|
||||
resolved recording MBID ONCE means the same song never costs a second lookup, across every
|
||||
future export and every playlist it appears in.
|
||||
|
||||
Mirrors ``core.metadata.album_mbid_cache`` exactly: a tiny SQLite table, lazy DB accessor,
|
||||
every function wrapped so any DB error degrades to a cache miss / no-op. If this module
|
||||
breaks, exports still work — they just re-resolve via the live waterfall like a cold cache.
|
||||
Key is the normalized ``track_key`` from ``mbid_resolver.normalize_key(artist, title)``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("exports.recording_mbid_cache")
|
||||
|
||||
_db_factory_lock = threading.Lock()
|
||||
_db_factory = None
|
||||
|
||||
|
||||
def _get_database():
|
||||
"""Resolve the MusicDatabase singleton lazily; None on any failure (treated as miss)."""
|
||||
global _db_factory
|
||||
with _db_factory_lock:
|
||||
if _db_factory is None:
|
||||
try:
|
||||
from database.music_database import get_database
|
||||
_db_factory = get_database
|
||||
except Exception as exc:
|
||||
logger.warning(f"Recording-MBID cache: could not load database module: {exc}")
|
||||
return None
|
||||
try:
|
||||
return _db_factory()
|
||||
except Exception as exc:
|
||||
logger.warning(f"Recording-MBID cache: database accessor failed: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def lookup(track_key: str) -> Optional[str]:
|
||||
"""Read a cached recording MBID for ``track_key``; None on miss or any DB error."""
|
||||
if not track_key:
|
||||
return None
|
||||
db = _get_database()
|
||||
if db is None:
|
||||
return None
|
||||
conn = None
|
||||
try:
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT recording_mbid FROM mb_recording_cache WHERE track_key = ? LIMIT 1",
|
||||
(track_key,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
return (row[0] if not hasattr(row, "keys") else row["recording_mbid"]) or None
|
||||
except Exception as exc:
|
||||
logger.debug(f"Recording-MBID cache lookup failed: {exc}")
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110 — finally cleanup
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def record(track_key: str, recording_mbid: str) -> bool:
|
||||
"""Persist ``track_key`` -> ``recording_mbid`` (idempotent). False on any failure."""
|
||||
if not track_key or not recording_mbid:
|
||||
return False
|
||||
db = _get_database()
|
||||
if db is None:
|
||||
return False
|
||||
conn = None
|
||||
try:
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO mb_recording_cache "
|
||||
"(track_key, recording_mbid, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)",
|
||||
(track_key, recording_mbid),
|
||||
)
|
||||
conn.commit()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug(f"Recording-MBID cache record failed: {exc}")
|
||||
return False
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110 — finally cleanup
|
||||
pass
|
||||
|
||||
|
||||
def clear_all() -> bool:
|
||||
"""Wipe the cache (tests / forced re-resolve)."""
|
||||
db = _get_database()
|
||||
if db is None:
|
||||
return False
|
||||
conn = None
|
||||
try:
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM mb_recording_cache")
|
||||
conn.commit()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning(f"Recording-MBID cache clear failed: {exc}")
|
||||
return False
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110 — finally cleanup
|
||||
pass
|
||||
|
||||
|
||||
__all__ = ["lookup", "record", "clear_all"]
|
||||
|
|
@ -36,9 +36,34 @@ import requests as http_requests
|
|||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.quality.source_map import quality_from_tidal_tier, quality_tier_for_source
|
||||
|
||||
logger = get_logger("hifi_client")
|
||||
|
||||
# A media playlist whose total runtime is below this fraction of the track's
|
||||
# real duration is a preview (some Monochrome instances only have 30s Tidal
|
||||
# DOWNLOAD access — a 220s track comes back as ~30s of segments + ENDLIST).
|
||||
_PREVIEW_DURATION_RATIO = 0.85
|
||||
_EXTINF_RE = re.compile(r'#EXTINF:\s*([0-9.]+)')
|
||||
|
||||
|
||||
def hls_total_seconds(playlist_text: str) -> float:
|
||||
"""Sum the ``#EXTINF`` segment durations in an HLS media playlist."""
|
||||
return sum(float(x) for x in _EXTINF_RE.findall(playlist_text or ''))
|
||||
|
||||
|
||||
def is_preview_playlist(playlist_s: float, track_s: float,
|
||||
ratio: float = _PREVIEW_DURATION_RATIO) -> bool:
|
||||
"""True when the playlist runtime is far shorter than the track's real
|
||||
duration (a preview). Returns False when either duration is unknown, so a
|
||||
missing reference never false-positives — the post-download audio guard is
|
||||
the safety net.
|
||||
"""
|
||||
if not playlist_s or not track_s or track_s <= 0:
|
||||
return False
|
||||
return playlist_s < track_s * ratio
|
||||
|
||||
|
||||
# HLS quality presets mapping to /trackManifests/ format parameters
|
||||
HLS_QUALITY_MAP = {
|
||||
'hires': {
|
||||
|
|
@ -140,6 +165,81 @@ def compute_new_default_pushes(all_defaults, offered, legacy_baseline, existing)
|
|||
return to_add, new_offered
|
||||
|
||||
|
||||
_EXTINF_RE = re.compile(r'#EXTINF:\s*([0-9]+(?:\.[0-9]+)?)')
|
||||
|
||||
|
||||
def sum_hls_segment_seconds(playlist_text: str) -> float:
|
||||
"""Total audio seconds an HLS media playlist actually provides — the sum of its
|
||||
``#EXTINF`` segment durations. This is the authoritative "how much audio is really
|
||||
here" signal: a PREVIEW manifest serves only ~30s of segments even though the track
|
||||
is full-length, so summing EXTINF catches it before we waste the download. Returns
|
||||
0.0 when the playlist has no EXTINF lines (master playlists, legacy manifests) — the
|
||||
caller treats 0 as 'unknown', never as 'preview'."""
|
||||
total = 0.0
|
||||
for m in _EXTINF_RE.finditer(playlist_text or ''):
|
||||
try:
|
||||
total += float(m.group(1))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return total
|
||||
|
||||
|
||||
def is_short_audio(actual_seconds: float, expected_seconds: float, threshold: float = 0.8) -> bool:
|
||||
"""True when ``actual`` is meaningfully shorter than ``expected`` — i.e. a preview
|
||||
clip or a truncated/corrupt download. Conservative: returns False whenever either
|
||||
value is missing/zero (unknown ⇒ never reject), and only trips below ``threshold``
|
||||
of the expected length (previews are ~15% of full, so the margin is huge)."""
|
||||
try:
|
||||
a, e = float(actual_seconds or 0), float(expected_seconds or 0)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if a <= 0 or e <= 0:
|
||||
return False
|
||||
return a < e * threshold
|
||||
|
||||
|
||||
def is_fake_lossless_bitrate(size_bytes, claimed_seconds, sample_rate, bits_per_sample,
|
||||
channels, min_ratio: float = 0.30) -> bool:
|
||||
"""True when a 'lossless' file's data is FAR too small for its claimed length — the
|
||||
fingerprint of a ~30s preview whose STREAMINFO/container was faked to the full
|
||||
duration (so every length header reads 'full' and only the bitrate gives it away).
|
||||
Real FLAC is ~40-75% of raw PCM; a preview padded to full length implies single-digit
|
||||
%. Conservative: 0 / bad inputs return False (never reject on unknowns)."""
|
||||
try:
|
||||
sz, secs = float(size_bytes or 0), float(claimed_seconds or 0)
|
||||
sr, bits, ch = int(sample_rate or 0), int(bits_per_sample or 0), int(channels or 0)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if sz <= 0 or secs <= 0 or sr <= 0 or bits <= 0 or ch <= 0:
|
||||
return False
|
||||
return (sz * 8 / secs) < (sr * bits * ch) * min_ratio
|
||||
|
||||
|
||||
def parse_ffmpeg_time(stderr_text) -> float:
|
||||
"""The last ``time=HH:MM:SS.xx`` ffmpeg prints while decoding — the REAL decoded
|
||||
length (immune to a faked container/STREAMINFO duration). 0.0 if not found."""
|
||||
last = 0.0
|
||||
for m in re.finditer(r'time=(\d+):(\d+):(\d+(?:\.\d+)?)', stderr_text or ''):
|
||||
last = int(m.group(1)) * 3600 + int(m.group(2)) * 60 + float(m.group(3))
|
||||
return last
|
||||
|
||||
|
||||
def is_preview_download(real_seconds, reference_seconds, *, is_lossless, size_bytes,
|
||||
sample_rate, bits_per_sample, channels):
|
||||
"""Is a finished file a preview/truncated fake? Two independent signals, so it fires
|
||||
even when the fakery declares full length at every layer:
|
||||
1. DECODED length far below the reference (the ground truth, when a decoder ran);
|
||||
2. for lossless, an impossibly-low implied bitrate (no decoder needed).
|
||||
Returns ``(is_fake, reason)``."""
|
||||
if real_seconds and is_short_audio(real_seconds, reference_seconds):
|
||||
return True, "decoded %.0fs of %.0fs" % (real_seconds, reference_seconds)
|
||||
if is_lossless and is_fake_lossless_bitrate(size_bytes, reference_seconds, sample_rate,
|
||||
bits_per_sample, channels):
|
||||
kbps = (float(size_bytes) * 8 / reference_seconds / 1000) if reference_seconds else 0
|
||||
return True, "%.0fkbps lossless over %.0fs (far too low — a ~30s preview)" % (kbps, reference_seconds)
|
||||
return False, ""
|
||||
|
||||
|
||||
# Run the new-default push at most once per process.
|
||||
_pushed_new_defaults = False
|
||||
|
||||
|
|
@ -157,7 +257,10 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
|
||||
self._instances = []
|
||||
self._instance_lock = threading.Lock()
|
||||
|
|
@ -622,7 +725,8 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
|
||||
return init_uri, segment_uris
|
||||
|
||||
def _get_hls_manifest(self, track_id: int, quality: str = 'lossless') -> Optional[Dict]:
|
||||
def _get_hls_manifest(self, track_id: int, quality: str = 'lossless',
|
||||
expected_duration_s: float = 0) -> Optional[Dict]:
|
||||
q_info = HLS_QUALITY_MAP.get(quality, HLS_QUALITY_MAP['lossless'])
|
||||
formats = q_info['formats']
|
||||
|
||||
|
|
@ -662,6 +766,7 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
logger.warning(f"Failed to parse HLS playlist for track {track_id}: {e}")
|
||||
return None
|
||||
|
||||
media_text = playlist_text # the playlist that actually carries the EXTINF segments
|
||||
if '#EXT-X-STREAM-INF' in playlist_text and segment_uris:
|
||||
playlist_uri = segment_uris[0]
|
||||
try:
|
||||
|
|
@ -669,11 +774,26 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
variant_resp = self.session.get(playlist_uri, allow_redirects=True, timeout=30)
|
||||
variant_resp.raise_for_status()
|
||||
variant_text = variant_resp.text
|
||||
media_text = variant_text
|
||||
init_uri, segment_uris = self._parse_hls_playlist(variant_text, playlist_uri)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch variant playlist for track {track_id}: {e}")
|
||||
return None
|
||||
|
||||
# Preview detection — some instances only have 30s Tidal DOWNLOAD
|
||||
# access, returning a playlist far shorter than the real track. Decline
|
||||
# it (and rotate the instance) so the orchestrator falls through to a
|
||||
# real source instead of fetching a 30s file that gets quarantined.
|
||||
playlist_s = hls_total_seconds(media_text)
|
||||
if is_preview_playlist(playlist_s, expected_duration_s):
|
||||
logger.warning(
|
||||
f"HiFi manifest for track {track_id} ({quality}) is a "
|
||||
f"{playlist_s:.0f}s preview of a {expected_duration_s:.0f}s track — "
|
||||
f"declining this instance"
|
||||
)
|
||||
self._rotate_instance(self._current_instance)
|
||||
return None
|
||||
|
||||
if init_uri:
|
||||
logger.info(f"HiFi HLS manifest for track {track_id}: "
|
||||
f"init segment + {len(segment_uris)} segments ({quality})")
|
||||
|
|
@ -687,6 +807,9 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
'extension': q_info['extension'],
|
||||
'codec': q_info['codec'],
|
||||
'quality': quality,
|
||||
# Real audio length the manifest provides (sum of EXTINF) — used to reject
|
||||
# preview manifests before downloading. 0.0 = unknown (don't reject).
|
||||
'manifest_duration': sum_hls_segment_seconds(media_text),
|
||||
}
|
||||
|
||||
def _get_legacy_track_manifest(self, track_id: int, quality: str = 'lossless') -> Optional[Dict]:
|
||||
|
|
@ -711,15 +834,57 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
'quality': quality,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _probe_audio_seconds(path) -> float:
|
||||
"""Real decoded audio length of a finished file, via mutagen (already a dep).
|
||||
0.0 on any failure — the caller treats 0 as 'unknown' and never rejects on it."""
|
||||
try:
|
||||
from mutagen import File as _MutagenFile
|
||||
mf = _MutagenFile(str(path))
|
||||
info = getattr(mf, 'info', None) if mf is not None else None
|
||||
if info is not None:
|
||||
return float(getattr(info, 'length', 0) or 0)
|
||||
except Exception as _probe_err: # noqa: BLE001
|
||||
logger.debug("mutagen audio-length probe failed for %s: %s", path, _probe_err)
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _find_ffmpeg():
|
||||
ff = shutil.which('ffmpeg')
|
||||
if ff:
|
||||
return ff
|
||||
cand = Path(__file__).parent.parent / 'tools' / ('ffmpeg.exe' if os.name == 'nt' else 'ffmpeg')
|
||||
return str(cand) if cand.exists() else None
|
||||
|
||||
def _probe_real_seconds(self, path) -> float:
|
||||
"""REAL decoded audio length via ffmpeg — decodes the actual frames, so it sees
|
||||
through a faked STREAMINFO/container duration (a 30s preview claiming full
|
||||
length decodes to 30s). 0.0 if ffmpeg is unavailable or on error."""
|
||||
ff = self._find_ffmpeg()
|
||||
if not ff:
|
||||
return 0.0
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[ff, '-hide_banner', '-nostdin', '-i', str(path), '-map', '0:a:0', '-f', 'null', '-'],
|
||||
capture_output=True, text=True, timeout=180)
|
||||
return parse_ffmpeg_time(proc.stderr)
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _flac_props(path):
|
||||
"""(sample_rate, bits_per_sample, channels) for the bitrate sanity check, or None."""
|
||||
try:
|
||||
from mutagen.flac import FLAC
|
||||
si = FLAC(str(path)).info
|
||||
return (si.sample_rate, si.bits_per_sample, si.channels)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _demux_flac(self, input_path: Path, output_path: Path) -> None:
|
||||
ffmpeg = shutil.which('ffmpeg')
|
||||
ffmpeg = self._find_ffmpeg()
|
||||
if not ffmpeg:
|
||||
tools_dir = Path(__file__).parent.parent / 'tools'
|
||||
ffmpeg_candidate = tools_dir / ('ffmpeg.exe' if os.name == 'nt' else 'ffmpeg')
|
||||
if ffmpeg_candidate.exists():
|
||||
ffmpeg = str(ffmpeg_candidate)
|
||||
else:
|
||||
raise RuntimeError('ffmpeg is required to demux FLAC from MP4. Install ffmpeg and retry.')
|
||||
raise RuntimeError('ffmpeg is required to demux FLAC from MP4. Install ffmpeg and retry.')
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
|
|
@ -751,13 +916,18 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
loop = asyncio.get_event_loop()
|
||||
tracks = await loop.run_in_executor(None, lambda: self.search_raw(query))
|
||||
|
||||
quality_key = config_manager.get('hifi_download.quality', 'lossless')
|
||||
quality_key = quality_tier_for_source('hifi', default='lossless')
|
||||
q_info = HLS_QUALITY_MAP.get(quality_key, HLS_QUALITY_MAP['lossless'])
|
||||
|
||||
# HiFi is Tidal-backed; stamp the configured tier so the global
|
||||
# ranker sees real sample_rate/bit_depth, not just 'flac'.
|
||||
tier_quality = quality_from_tidal_tier(quality_key)
|
||||
|
||||
results = []
|
||||
for t in tracks:
|
||||
try:
|
||||
tr = self._to_track_result(t, q_info)
|
||||
tr.set_quality(tier_quality)
|
||||
results.append(tr)
|
||||
except Exception as e:
|
||||
logger.debug(f"Skipping track result conversion: {e}")
|
||||
|
|
@ -828,7 +998,7 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
)
|
||||
|
||||
def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]:
|
||||
quality_key = config_manager.get('hifi_download.quality', 'lossless')
|
||||
quality_key = quality_tier_for_source('hifi', default='lossless')
|
||||
chain = ['hires', 'lossless', 'high', 'low']
|
||||
start = chain.index(quality_key) if quality_key in chain else 1
|
||||
allow_fallback = config_manager.get('hifi_download.allow_fallback', True)
|
||||
|
|
@ -836,12 +1006,26 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
|
||||
MIN_AUDIO_SIZE = 100 * 1024
|
||||
|
||||
# Expected track length, drives every preview/truncation guard here:
|
||||
# * _get_hls_manifest's pre-download is_preview_playlist check
|
||||
# * the pre-download is_short_audio manifest check
|
||||
# * the post-download is_preview_download faked-header decode check
|
||||
# Best-effort: a 0 here just disables the duration checks, never rejects.
|
||||
expected_s = 0.0
|
||||
try:
|
||||
info = self.get_track_info(track_id) or {}
|
||||
expected_s = float(info.get('duration_s') or 0)
|
||||
except Exception:
|
||||
expected_s = 0.0
|
||||
expected_duration_s = expected_s # alias for _get_hls_manifest's param name
|
||||
|
||||
for q_key in chain:
|
||||
if self.shutdown_check and self.shutdown_check():
|
||||
logger.info("Shutdown detected, aborting HiFi download")
|
||||
return None
|
||||
|
||||
manifest_info = self._get_hls_manifest(track_id, quality=q_key)
|
||||
manifest_info = self._get_hls_manifest(track_id, quality=q_key,
|
||||
expected_duration_s=expected_duration_s)
|
||||
if (
|
||||
not manifest_info
|
||||
or (
|
||||
|
|
@ -852,6 +1036,18 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
logger.warning(f"No HLS manifest at quality {q_key}, trying next")
|
||||
continue
|
||||
|
||||
# Preview guard #1 (pre-download): a preview manifest serves only ~30s of
|
||||
# segments for a full-length track. A preview means THIS SOURCE only has a
|
||||
# preview of the track — lower quality tiers are the SAME preview — so abort
|
||||
# HiFi entirely and let the orchestrator fall through to the next SOURCE
|
||||
# (soulseek/youtube/…), rather than landing a lower-tier preview.
|
||||
manifest_s = float(manifest_info.get('manifest_duration') or 0)
|
||||
if is_short_audio(manifest_s, expected_s):
|
||||
logger.warning(
|
||||
"HiFi has only a PREVIEW of '%s' (manifest %.0fs of %.0fs at %s) — "
|
||||
"failing HiFi so the next source is tried", display_name, manifest_s, expected_s, q_key)
|
||||
return None
|
||||
|
||||
extension = manifest_info['extension']
|
||||
safe_name = re.sub(r'[<>:"/\\|?*]', '_', display_name)
|
||||
out_filename = f"{safe_name}.{extension}"
|
||||
|
|
@ -931,6 +1127,31 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
out_path.unlink(missing_ok=True)
|
||||
continue
|
||||
|
||||
# Preview guard #2 (post-download): the real catch. HiFi previews fake the
|
||||
# FULL length in every header — manifest EXTINF, m4a moov, FLAC
|
||||
# total_samples — so only the DECODED audio (or, for lossless, the
|
||||
# bitrate) reveals the ~30s truth. Reference = the largest length any
|
||||
# header claims (so the file's own faked claim becomes the bar its real
|
||||
# audio must clear); is_preview_download decodes + bitrate-checks.
|
||||
ref_s = max(expected_s, self._probe_audio_seconds(out_path))
|
||||
real_s = self._probe_real_seconds(out_path)
|
||||
props = self._flac_props(out_path) if is_flac else None
|
||||
fake, why = is_preview_download(
|
||||
real_s, ref_s, is_lossless=is_flac, size_bytes=final_size,
|
||||
sample_rate=(props[0] if props else 0),
|
||||
bits_per_sample=(props[1] if props else 0),
|
||||
channels=(props[2] if props else 0))
|
||||
if fake:
|
||||
# A preview at this tier means the SOURCE only has a preview — every
|
||||
# lower tier is the same 30s clip (and the lossy ones dodge the
|
||||
# bitrate check). Abort HiFi so the orchestrator tries the next
|
||||
# SOURCE, instead of cascading down into an accepted lower-tier preview.
|
||||
logger.warning(
|
||||
"HiFi has only a PREVIEW of '%s' (%s at %s) — failing HiFi so the "
|
||||
"next source is tried", display_name, why, q_key)
|
||||
out_path.unlink(missing_ok=True)
|
||||
return None
|
||||
|
||||
logger.info(f"HiFi download complete ({q_key}): {out_path} "
|
||||
f"({final_size / (1024*1024):.1f} MB)")
|
||||
return str(out_path)
|
||||
|
|
|
|||
98
core/imports/album_grouping.py
Normal file
98
core/imports/album_grouping.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
"""Canonical album grouping for the SoulSync standalone import.
|
||||
|
||||
SoulSync grouped imported tracks into albums by the album NAME string
|
||||
(``_stable_soulsync_id("artist::album_name")``). That splits one release into
|
||||
several album rows whenever the name string drifts between imports (case,
|
||||
punctuation, ``(Deluxe Edition)`` suffixes, source-A-vs-B spelling), and every
|
||||
downstream tool (Library Re-tag, Cover-Art Filler) then dresses each split row
|
||||
in its own cover — so songs that belong to one album end up with different art
|
||||
(Sokhi).
|
||||
|
||||
This module is the pure, seam-testable heart of "group by canonical id, not
|
||||
name": when an imported track carries a metadata-source RELEASE id, prefer
|
||||
matching an existing album row by that id over the fragile name string, so the
|
||||
SAME release always lands in ONE album row regardless of how its name was typed.
|
||||
|
||||
Scope (deliberate): this unifies differently-named imports of the SAME release.
|
||||
It does NOT merge a track that genuinely matched a SINGLE release (a different
|
||||
release id) into its parent album — that needs single->album resolution upstream
|
||||
and is a separate change. New imports only; existing rows are left untouched.
|
||||
|
||||
Pure SQL-over-a-cursor; no app singletons, so it tests against an in-memory DB.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("imports.album_grouping")
|
||||
|
||||
# Album source-id columns this grouping may key on. An allowlist (not arbitrary
|
||||
# interpolation) — the column name IS spliced into SQL, so it must be a known,
|
||||
# trusted identifier. Mirrors get_library_source_id_columns()' 'album' values.
|
||||
ALLOWED_ALBUM_SOURCE_COLS = frozenset({
|
||||
"spotify_album_id",
|
||||
"itunes_album_id",
|
||||
"deezer_id",
|
||||
"soul_id",
|
||||
"discogs_id",
|
||||
"musicbrainz_release_id",
|
||||
})
|
||||
|
||||
|
||||
def find_existing_soulsync_album_id(
|
||||
cursor: Any,
|
||||
*,
|
||||
name_key_id: str,
|
||||
artist_id: str,
|
||||
album_name: str,
|
||||
album_source_col: Optional[str] = None,
|
||||
album_source_id: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve the existing ``soulsync`` album row a track should join, or None
|
||||
(caller inserts a new row keyed by ``name_key_id``).
|
||||
|
||||
Match precedence:
|
||||
1. ``name_key_id`` — the exact prior stable-name-hash id (unchanged
|
||||
behaviour: a re-import with the identical name hits its own row).
|
||||
2. ``album_source_col == album_source_id`` — CANONICAL grouping: an
|
||||
existing row already carrying THIS release's source id, so a
|
||||
differently-named import of the same release unifies instead of
|
||||
splitting. Only when the column is allow-listed and the id is non-empty.
|
||||
3. ``(title, artist_id)`` — the legacy name match (kept so nothing that
|
||||
grouped before stops grouping now).
|
||||
"""
|
||||
cursor.execute(
|
||||
"SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'",
|
||||
(name_key_id,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
|
||||
if album_source_col in ALLOWED_ALBUM_SOURCE_COLS and album_source_id:
|
||||
try:
|
||||
cursor.execute(
|
||||
f"SELECT id FROM albums WHERE {album_source_col} = ? "
|
||||
"AND server_source = 'soulsync' LIMIT 1",
|
||||
(album_source_id,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
except Exception as exc:
|
||||
# That source has no dedicated album column on this DB (e.g. Deezer
|
||||
# doesn't split per-entity id columns) — fall through to the name
|
||||
# match rather than break the import. Mirrors the guarded source-id
|
||||
# UPDATE the caller already does on insert.
|
||||
logger.debug("album source-id lookup skipped (%s): %s", album_source_col, exc)
|
||||
|
||||
cursor.execute(
|
||||
"SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? "
|
||||
"AND server_source = 'soulsync' LIMIT 1",
|
||||
(album_name, artist_id),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
return row[0] if row else None
|
||||
|
|
@ -156,8 +156,11 @@ def score_file_against_track(
|
|||
score = 0.0
|
||||
|
||||
# Title similarity (TITLE_WEIGHT). Falls back to filename stem when
|
||||
# the file has no title tag.
|
||||
# the file has no title tag — strip a leading track-number prefix off that
|
||||
# stem (#890) so "01 - Sun It Rises" scores against "Sun It Rises".
|
||||
title = file_tags.get('title') or os.path.splitext(os.path.basename(file_path))[0]
|
||||
from core.imports.paths import strip_leading_track_number
|
||||
title = strip_leading_track_number(title)
|
||||
track_name = track.get('name', '')
|
||||
score += similarity(title, track_name) * TITLE_WEIGHT
|
||||
|
||||
|
|
|
|||
94
core/imports/album_position.py
Normal file
94
core/imports/album_position.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
"""Resolve a track's position WITHIN its album's track list.
|
||||
|
||||
The bug this fixes: a track auto-downloaded from the playlist pipeline / wishlist /
|
||||
watchlist is identified as belonging to an album, but the per-track position is
|
||||
unknown — Deezer's search/track and MusicBrainz's recording lookups don't carry a
|
||||
track position (only their album endpoint does). ``detect_album_info_web`` then
|
||||
leaves ``track_number = None``, the import pipeline falls through to the default-1
|
||||
floor, and the file lands as ``01/1`` even though the album is known
|
||||
(``core/imports/context.py``). Verified live: e.g. Deezer says "Obelisk" is track
|
||||
9 of *The Grand Mirage*, but it was tagged 1/1.
|
||||
|
||||
This is the pure matcher: given the album's track list (fetched by the caller via
|
||||
``core.metadata.album_tracks.get_album_tracks_for_source`` — so this stays
|
||||
source-agnostic and I/O-free) plus the track's own identifiers, return its real
|
||||
``(track_number, disc_number)``. Match priority is by reliability:
|
||||
|
||||
1. **ISRC** — an exact recording identity; trusted immediately.
|
||||
2. **source track id** — exact within this album.
|
||||
3. **normalized title** — last resort.
|
||||
|
||||
Returns ``(None, None)`` on no confident match, so the caller keeps its existing
|
||||
behaviour (never worse than today).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, List, Optional, Tuple
|
||||
|
||||
|
||||
def _norm_title(value: Any) -> str:
|
||||
"""Lower, strip punctuation, collapse whitespace — for tolerant title match."""
|
||||
s = re.sub(r"[^\w\s]", "", str(value or "").lower())
|
||||
return re.sub(r"\s+", " ", s).strip()
|
||||
|
||||
|
||||
def _pos_int(value: Any) -> Optional[int]:
|
||||
try:
|
||||
n = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return n if n >= 1 else None
|
||||
|
||||
|
||||
def resolve_track_position_in_album(
|
||||
album_tracks: List[dict],
|
||||
*,
|
||||
title: str = "",
|
||||
track_id: str = "",
|
||||
isrc: str = "",
|
||||
) -> Tuple[Optional[int], Optional[int]]:
|
||||
"""Return ``(track_number, disc_number)`` for this track within ``album_tracks``,
|
||||
or ``(None, None)`` when no confident match is found.
|
||||
|
||||
``album_tracks`` is the list under ``get_album_tracks_for_source(...)['tracks']``
|
||||
— each entry has ``track_number`` / ``disc_number`` / ``id`` / ``name`` / ``isrc``.
|
||||
Entries without a valid positive ``track_number`` are skipped. Pure: no I/O.
|
||||
"""
|
||||
if not album_tracks:
|
||||
return (None, None)
|
||||
|
||||
want_isrc = str(isrc or "").strip().upper()
|
||||
want_id = str(track_id or "").strip()
|
||||
want_title = _norm_title(title)
|
||||
|
||||
by_id: Optional[Tuple[int, int]] = None
|
||||
by_title: Optional[Tuple[int, int]] = None
|
||||
|
||||
for t in album_tracks:
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
tn = _pos_int(t.get("track_number"))
|
||||
if tn is None:
|
||||
continue
|
||||
dn = _pos_int(t.get("disc_number")) or 1
|
||||
|
||||
# 1) ISRC — exact recording. Win immediately.
|
||||
if want_isrc and str(t.get("isrc") or "").strip().upper() == want_isrc:
|
||||
return (tn, dn)
|
||||
# 2) source track id — exact within the album.
|
||||
if by_id is None and want_id and str(t.get("id") or "").strip() == want_id:
|
||||
by_id = (tn, dn)
|
||||
# 3) normalized title — last resort.
|
||||
if by_title is None and want_title and _norm_title(t.get("name")) == want_title:
|
||||
by_title = (tn, dn)
|
||||
|
||||
if by_id is not None:
|
||||
return by_id
|
||||
if by_title is not None:
|
||||
return by_title
|
||||
return (None, None)
|
||||
|
||||
|
||||
__all__ = ["resolve_track_position_in_album"]
|
||||
|
|
@ -8,6 +8,10 @@ from __future__ import annotations
|
|||
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("imports.context")
|
||||
|
||||
|
||||
def _as_dict(value: Any) -> Dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
|
@ -179,7 +183,12 @@ def get_import_clean_title(
|
|||
if not title:
|
||||
track_info = get_import_track_info(context)
|
||||
title = _first_value(track_info, "name", "title", default="")
|
||||
return str(title or default)
|
||||
title = str(title or default)
|
||||
# #890: strip a leading track-number prefix that leaked from a filename stem
|
||||
# (e.g. "01 - Sun It Rises" → "Sun It Rises") so it matches the canonical title.
|
||||
# Conservative — clean source titles ("7 Rings" etc.) pass through untouched.
|
||||
from core.imports.paths import strip_leading_track_number
|
||||
return strip_leading_track_number(title)
|
||||
|
||||
|
||||
def get_import_clean_album(
|
||||
|
|
@ -319,7 +328,11 @@ def build_import_album_info(
|
|||
(album_info or {}).get("track_number")
|
||||
or track_info.get("track_number")
|
||||
or original_search.get("track_number")
|
||||
or 1
|
||||
# "Track 01" bug: default to 0 (the codebase's "unknown" sentinel,
|
||||
# same as total_tracks below), NOT 1. A fabricated 1 looks
|
||||
# authoritative and blocks the pipeline's downstream recovery
|
||||
# (embedded file tag / resolve chain); 0 lets it fall through.
|
||||
or 0
|
||||
)
|
||||
disc_number = (
|
||||
(album_info or {}).get("disc_number")
|
||||
|
|
@ -420,20 +433,152 @@ def detect_album_info_web(context, artist_context=None):
|
|||
track_name.strip().lower(),
|
||||
artist_name.strip().lower(),
|
||||
}:
|
||||
_tn = track_info.get("track_number")
|
||||
_dn = track_info.get("disc_number")
|
||||
# The album is identified but discovery often doesn't carry the per-track
|
||||
# POSITION — Deezer's search/track and MusicBrainz's recording lookups omit
|
||||
# it (only their album endpoint has it). Without a position the pipeline
|
||||
# falls through to the default-1 floor and files an album track as 01/1
|
||||
# (e.g. Deezer says "Obelisk" is track 9 of The Grand Mirage). Resolve the
|
||||
# REAL position from the album's own track list when we have its id.
|
||||
# Fail-safe: leaves the numbers untouched on any miss, so behaviour is
|
||||
# never worse than the old preserve-None-and-fall-through.
|
||||
if _tn is None:
|
||||
_tn, _dn = _resolve_album_position_from_source(context, artist_context, _dn)
|
||||
return build_import_album_info(
|
||||
context,
|
||||
album_info={
|
||||
"album_name": album_name,
|
||||
# Preserve missing numbers as None so the import pipeline
|
||||
# can fall through to ``extract_track_number_from_filename``
|
||||
# at ``core/imports/pipeline.py:652`` instead of locking
|
||||
# to track/disc 01 for every wishlist re-attempt.
|
||||
"track_number": track_info.get("track_number"),
|
||||
"disc_number": track_info.get("disc_number"),
|
||||
"track_number": _tn,
|
||||
"disc_number": _dn,
|
||||
"album_image_url": album_ctx.get("image_url", ""),
|
||||
"confidence": 0.5,
|
||||
},
|
||||
force_album=True,
|
||||
)
|
||||
|
||||
return None
|
||||
# Last resort: the track matched a SINGLE with no usable album context —
|
||||
# look up the parent ALBUM that actually contains it (gated, fail-safe).
|
||||
return _resolve_single_to_parent_album(context, artist_context)
|
||||
|
||||
|
||||
def _resolve_album_position_from_source(context, artist_context, current_disc):
|
||||
"""Look up a track's real ``(track_number, disc_number)`` from its album's track
|
||||
list, for the case where the album is known but discovery didn't carry a
|
||||
position (Deezer/MusicBrainz search omit it).
|
||||
|
||||
Uses the SAME album id discovery already resolved (``get_import_source_ids`` →
|
||||
``album_id``), so it re-homes the track onto its own album with no re-search and
|
||||
no edition guessing. Matches by ISRC → source track id → title via the pure
|
||||
``core.imports.album_position`` seam. Returns ``(None, current_disc)`` on any
|
||||
miss/error so the caller falls back exactly as before — never worse than today.
|
||||
"""
|
||||
try:
|
||||
source = get_import_source(context)
|
||||
ids = get_import_source_ids(context)
|
||||
album_id = str(ids.get("album_id") or "")
|
||||
if not source or not album_id:
|
||||
return None, current_disc
|
||||
|
||||
from core.metadata.album_tracks import get_album_tracks_for_source
|
||||
payload = get_album_tracks_for_source(source, album_id) or {}
|
||||
tracks = payload.get("tracks") or []
|
||||
if not tracks:
|
||||
return None, current_disc
|
||||
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
title = (track_info.get("name") or original_search.get("title") or "").strip()
|
||||
isrc = str(track_info.get("isrc") or original_search.get("isrc") or "")
|
||||
|
||||
from core.imports.album_position import resolve_track_position_in_album
|
||||
tn, dn = resolve_track_position_in_album(
|
||||
tracks, title=title, track_id=str(ids.get("track_id") or ""), isrc=isrc)
|
||||
if tn is not None:
|
||||
logger.info("album-position: resolved '%s' to track %s/disc %s from album %s (%s)",
|
||||
title, tn, dn, album_id, source)
|
||||
return tn, (dn if dn is not None else current_disc)
|
||||
return None, current_disc
|
||||
except Exception as e:
|
||||
logger.debug("album-position resolution failed: %s", e)
|
||||
return None, current_disc
|
||||
|
||||
|
||||
def _resolve_single_to_parent_album(context, artist_context):
|
||||
"""A single-matched track -> a promoted album_info for its parent album, or
|
||||
None. GATED by ``metadata_enhancement.single_to_album`` (default OFF — it's a
|
||||
per-import metadata lookup, so it's opt-in). Fail-safe: any miss/error returns
|
||||
None so the track stays exactly as it was matched (never worse than today)."""
|
||||
try:
|
||||
from core.metadata.common import get_config_manager
|
||||
if not get_config_manager().get("metadata_enhancement.single_to_album", False):
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
try:
|
||||
source = get_import_source(context)
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
track_title = (track_info.get("name") or original_search.get("title") or "").strip()
|
||||
artist_name = (extract_artist_name(artist_context)
|
||||
or get_import_clean_artist(context, default="")).strip()
|
||||
if not source or not track_title or not artist_name:
|
||||
return None
|
||||
artist_id = str(get_import_source_ids(context).get("artist_id") or "")
|
||||
|
||||
from core.metadata.album_tracks import (
|
||||
get_artist_albums_for_source,
|
||||
get_artist_album_tracks,
|
||||
)
|
||||
from core.imports.single_to_album import resolve_single_to_album
|
||||
|
||||
def _acc(o, *ks):
|
||||
for k in ks:
|
||||
v = o.get(k) if isinstance(o, dict) else getattr(o, k, None)
|
||||
if v:
|
||||
return v
|
||||
return None
|
||||
|
||||
def fetch_candidates():
|
||||
albums = get_artist_albums_for_source(
|
||||
source, artist_id, artist_name=artist_name,
|
||||
album_type="album", limit=20) or []
|
||||
return [{"name": _acc(a, "name", "title"),
|
||||
"album_type": _acc(a, "album_type") or "album",
|
||||
"id": _acc(a, "id", "album_id")} for a in albums]
|
||||
|
||||
def fetch_tracks(alb):
|
||||
payload = get_artist_album_tracks(
|
||||
str(alb.get("id") or ""), artist_name=artist_name,
|
||||
album_name=alb.get("name") or "") or {}
|
||||
return [(_acc(t, "title", "name", "track_name") or "")
|
||||
for t in (payload.get("tracks") or [])]
|
||||
|
||||
album = resolve_single_to_album(
|
||||
track_title,
|
||||
fetch_album_candidates=fetch_candidates,
|
||||
fetch_album_tracks=fetch_tracks)
|
||||
if not album or not album.get("name"):
|
||||
return None
|
||||
logger.info("single->album: re-homed '%s' onto parent album '%s'",
|
||||
track_title, album["name"])
|
||||
promoted = build_import_album_info(
|
||||
context,
|
||||
album_info={
|
||||
"album_name": album["name"],
|
||||
"track_number": track_info.get("track_number"),
|
||||
"disc_number": track_info.get("disc_number"),
|
||||
"album_image_url": "",
|
||||
"confidence": 0.5,
|
||||
},
|
||||
force_album=True,
|
||||
)
|
||||
# build_import_album_info resolves album_name via get_import_clean_album,
|
||||
# which prefers original_search.album (the SINGLE's name); override it
|
||||
# with the resolved parent album so grouping + tags use the album.
|
||||
promoted["album_name"] = album["name"]
|
||||
return promoted
|
||||
except Exception as e:
|
||||
logger.debug("single->album resolution failed: %s", e)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -52,6 +52,14 @@ _DEFAULT_LENGTH_TOLERANCE_S = 3.0
|
|||
_LENGTH_TOLERANCE_LONG_TRACK_S = 5.0
|
||||
_LONG_TRACK_THRESHOLD_S = 600.0 # 10 minutes
|
||||
|
||||
# A file that runs LONGER than the expected metadata is the opposite of a truncated
|
||||
# download — it's almost always a different master/version (a remaster with a longer
|
||||
# outro, an extended fade, an album cut vs the radio edit). The duration check exists to
|
||||
# catch TRUNCATION (short files) and wildly-wrong matches, so on the auto default we allow
|
||||
# more drift in the longer direction and keep the tight bound for short files. A wrong-song
|
||||
# match still trips this — it's usually off by far more than 15s. (#937)
|
||||
_LONGER_VERSION_TOLERANCE_S = 15.0
|
||||
|
||||
# Upper bound for the user-configurable override. Anything past 60s
|
||||
# means the check is effectively off — cap defends against accidental
|
||||
# nonsense like 9999 making logs misleading. Users who genuinely want
|
||||
|
|
@ -242,18 +250,32 @@ def check_audio_integrity(
|
|||
if expected_length_s > _LONG_TRACK_THRESHOLD_S
|
||||
else _DEFAULT_LENGTH_TOLERANCE_S
|
||||
)
|
||||
user_pinned_tolerance = False
|
||||
else:
|
||||
user_pinned_tolerance = True
|
||||
checks["length_tolerance_s"] = length_tolerance_s
|
||||
|
||||
drift_s = abs(actual_length_s - expected_length_s)
|
||||
# Positive drift = the file runs LONGER than expected (not truncation). On the auto
|
||||
# default, give the longer direction more room so legit longer masters/versions aren't
|
||||
# quarantined (#937); a user-pinned tolerance is honoured symmetrically.
|
||||
signed_drift_s = actual_length_s - expected_length_s
|
||||
drift_s = abs(signed_drift_s)
|
||||
checks["length_drift_s"] = drift_s
|
||||
effective_tolerance_s = length_tolerance_s
|
||||
if signed_drift_s > 0 and not user_pinned_tolerance:
|
||||
effective_tolerance_s = max(length_tolerance_s, _LONGER_VERSION_TOLERANCE_S)
|
||||
checks["effective_tolerance_s"] = effective_tolerance_s
|
||||
|
||||
if drift_s > length_tolerance_s:
|
||||
if drift_s > effective_tolerance_s:
|
||||
runs_long = signed_drift_s > 0
|
||||
return IntegrityResult(
|
||||
ok=False,
|
||||
reason=f"Duration mismatch: file is {actual_length_s:.1f}s, "
|
||||
f"expected {expected_length_s:.1f}s "
|
||||
f"(drift {drift_s:.1f}s > tolerance {length_tolerance_s:.1f}s) — "
|
||||
"likely truncated download or wrong file matched",
|
||||
f"(drift {drift_s:.1f}s > tolerance {effective_tolerance_s:.1f}s) — "
|
||||
+ ("runs longer than expected — likely a different version/master or wrong file"
|
||||
if runs_long
|
||||
else "likely truncated download or wrong file matched"),
|
||||
checks=checks,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
|
@ -102,6 +103,41 @@ def cleanup_slskd_dedup_siblings(source_path) -> List[str]:
|
|||
return deleted
|
||||
|
||||
|
||||
def _atomic_cross_device_move(src: Path, dst: Path) -> None:
|
||||
"""Move ``src`` to ``dst`` across filesystems WITHOUT ever exposing a partial file at
|
||||
the final path.
|
||||
|
||||
Copies into a hidden temp sibling of ``dst`` (same filesystem), fsyncs, then does an
|
||||
atomic ``os.replace`` into place, then deletes ``src``. A media-server file watcher
|
||||
(Jellyfin/Plex real-time monitoring) therefore only ever indexes the COMPLETE file —
|
||||
an incremental in-place copy was what Jellyfin could catch mid-write and cache with
|
||||
null/incomplete metadata (tracks landing with no disc). Cleans up the temp on failure.
|
||||
"""
|
||||
src, dst = Path(src), Path(dst)
|
||||
tmp = dst.parent / f".{dst.name}.ssync-tmp"
|
||||
try:
|
||||
with open(src, "rb") as f_src, open(tmp, "wb") as f_dst:
|
||||
shutil.copyfileobj(f_src, f_dst)
|
||||
f_dst.flush()
|
||||
os.fsync(f_dst.fileno())
|
||||
try:
|
||||
shutil.copystat(str(src), str(tmp)) # preserve mtime/permissions (copy2-like)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(str(tmp), str(dst)) # atomic within dst's filesystem
|
||||
except Exception:
|
||||
try:
|
||||
if tmp.exists():
|
||||
tmp.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
try:
|
||||
src.unlink()
|
||||
except OSError:
|
||||
logger.info(f"Could not delete source after cross-device move (may be owned by another process): {src}")
|
||||
|
||||
|
||||
def safe_move_file(src, dst):
|
||||
"""Move a file safely across filesystems."""
|
||||
src = Path(src)
|
||||
|
|
@ -129,7 +165,13 @@ def safe_move_file(src, dst):
|
|||
break
|
||||
|
||||
try:
|
||||
shutil.move(str(src), str(dst))
|
||||
# Same-filesystem move: an atomic rename that also overwrites dst. A media-server
|
||||
# watcher (Jellyfin/Plex real-time monitoring) therefore never sees a partial file
|
||||
# at the final name. Cross-filesystem raises EXDEV (some network mounts raise
|
||||
# EPERM/EACCES) and we copy atomically below instead of letting the move write the
|
||||
# destination incrementally — the partial-file-at-final-name is what caused tracks
|
||||
# to land in Jellyfin with null/incomplete metadata (no disc).
|
||||
os.replace(str(src), str(dst))
|
||||
return
|
||||
except FileNotFoundError:
|
||||
if dst.exists():
|
||||
|
|
@ -137,8 +179,6 @@ def safe_move_file(src, dst):
|
|||
return
|
||||
raise
|
||||
except (OSError, PermissionError) as e:
|
||||
error_msg = str(e).lower()
|
||||
|
||||
if dst.exists() and dst.stat().st_size > 0:
|
||||
logger.warning(f"Move raised {type(e).__name__} but destination exists, treating as success: {e}")
|
||||
try:
|
||||
|
|
@ -147,23 +187,21 @@ def safe_move_file(src, dst):
|
|||
logger.info(f"Could not delete source file (may be owned by another process): {src}")
|
||||
return
|
||||
|
||||
if "cross-device" in error_msg or "operation not permitted" in error_msg or "permission denied" in error_msg:
|
||||
logger.warning(f"Cross-device move detected, using fallback copy method: {e}")
|
||||
error_msg = str(e).lower()
|
||||
cross_device = (
|
||||
getattr(e, "errno", None) in (errno.EXDEV, errno.EPERM, errno.EACCES)
|
||||
or "cross-device" in error_msg
|
||||
or "operation not permitted" in error_msg
|
||||
or "permission denied" in error_msg
|
||||
)
|
||||
if cross_device:
|
||||
logger.warning(f"Cross-device move, using atomic copy+rename: {e}")
|
||||
try:
|
||||
with open(src, "rb") as f_src:
|
||||
with open(dst, "wb") as f_dst:
|
||||
shutil.copyfileobj(f_src, f_dst)
|
||||
f_dst.flush()
|
||||
os.fsync(f_dst.fileno())
|
||||
|
||||
try:
|
||||
src.unlink()
|
||||
except PermissionError:
|
||||
logger.info(f"Could not delete source file (may be owned by another process): {src}")
|
||||
logger.info(f"Successfully moved file using fallback method: {src} -> {dst}")
|
||||
_atomic_cross_device_move(src, dst)
|
||||
logger.info(f"Successfully moved file atomically across filesystems: {src} -> {dst}")
|
||||
return
|
||||
except Exception as fallback_error:
|
||||
logger.error(f"Fallback copy also failed: {fallback_error}")
|
||||
logger.error(f"Atomic cross-device move failed: {fallback_error}")
|
||||
raise
|
||||
raise
|
||||
|
||||
|
|
@ -192,6 +230,9 @@ def get_audio_quality_string(file_path):
|
|||
if ext == ".flac":
|
||||
from mutagen.flac import FLAC
|
||||
audio = FLAC(file_path)
|
||||
sr = getattr(audio.info, "sample_rate", 0) or 0
|
||||
if sr:
|
||||
return f"FLAC {audio.info.bits_per_sample}bit/{sr / 1000:g}kHz"
|
||||
return f"FLAC {audio.info.bits_per_sample}bit"
|
||||
|
||||
if ext == ".mp3":
|
||||
|
|
@ -225,6 +266,122 @@ def get_audio_quality_string(file_path):
|
|||
return ""
|
||||
|
||||
|
||||
def probe_audio_quality(file_path: str):
|
||||
"""Read the actual file and return an AudioQuality with real measured values.
|
||||
|
||||
Uses mutagen to extract sample_rate, bit_depth, and bitrate from the
|
||||
downloaded file — these are ground-truth values, not estimates.
|
||||
Returns None when the file cannot be read.
|
||||
"""
|
||||
from core.quality.model import AudioQuality
|
||||
try:
|
||||
ext = os.path.splitext(file_path)[1].lower().lstrip('.')
|
||||
|
||||
if ext == 'flac':
|
||||
from mutagen.flac import FLAC
|
||||
audio = FLAC(file_path)
|
||||
return AudioQuality(
|
||||
format='flac',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
bit_depth=audio.info.bits_per_sample,
|
||||
)
|
||||
|
||||
if ext == 'mp3':
|
||||
from mutagen.mp3 import MP3
|
||||
audio = MP3(file_path)
|
||||
return AudioQuality(
|
||||
format='mp3',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext in ('m4a', 'aac', 'mp4'):
|
||||
from mutagen.mp4 import MP4
|
||||
audio = MP4(file_path)
|
||||
# .m4a can carry AAC (lossy) OR ALAC (lossless) — only the real
|
||||
# codec tells them apart, which is why extension-based classification
|
||||
# defaults to 'aac' and we correct it here from the probed file.
|
||||
codec = (getattr(audio.info, 'codec', '') or '').lower()
|
||||
if 'alac' in codec:
|
||||
return AudioQuality(
|
||||
format='alac',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
bit_depth=getattr(audio.info, 'bits_per_sample', None) or None,
|
||||
)
|
||||
return AudioQuality(
|
||||
format='aac',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext == 'ogg':
|
||||
from mutagen.oggvorbis import OggVorbis
|
||||
audio = OggVorbis(file_path)
|
||||
return AudioQuality(
|
||||
format='ogg',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext == 'opus':
|
||||
from mutagen.oggopus import OggOpus
|
||||
audio = OggOpus(file_path)
|
||||
return AudioQuality(
|
||||
format='opus',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext in ('wav', 'aiff', 'aif'):
|
||||
# AIFF must use mutagen.aiff.AIFF — WAVE() can't parse it and would
|
||||
# raise, making the file fail open and silently bypass the quality
|
||||
# filter. Both are uncompressed PCM, so they share the 'wav' tier.
|
||||
if ext == 'wav':
|
||||
from mutagen.wave import WAVE
|
||||
audio = WAVE(file_path)
|
||||
else:
|
||||
from mutagen.aiff import AIFF
|
||||
audio = AIFF(file_path)
|
||||
return AudioQuality(
|
||||
format='wav',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
bit_depth=getattr(audio.info, 'bits_per_sample', None),
|
||||
)
|
||||
|
||||
if ext == 'wma':
|
||||
from mutagen.asf import ASF
|
||||
audio = ASF(file_path)
|
||||
return AudioQuality(
|
||||
format='wma',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=getattr(audio.info, 'sample_rate', None),
|
||||
)
|
||||
|
||||
if ext in ('dsf', 'dff'):
|
||||
# DSD (DSD Stream File / DSDIFF) — 1-bit hi-res lossless (#939). mutagen
|
||||
# reads .dsf (rate/bitrate/bit_depth); .dff has no mutagen reader, so it
|
||||
# still classifies as the lossless 'dsf' tier just without measured detail.
|
||||
sr = bd = br = None
|
||||
if ext == 'dsf':
|
||||
try:
|
||||
from mutagen.dsf import DSF
|
||||
info = DSF(file_path).info
|
||||
sr = getattr(info, 'sample_rate', None)
|
||||
bd = getattr(info, 'bits_per_sample', None)
|
||||
br = info.bitrate // 1000 if getattr(info, 'bitrate', None) else None
|
||||
except Exception: # noqa: S110 — unreadable DSF still classifies lossless, just without measured detail
|
||||
pass
|
||||
return AudioQuality(format='dsf', bitrate=br, sample_rate=sr, bit_depth=bd)
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug("probe_audio_quality failed for %s: %s", file_path, e)
|
||||
return None
|
||||
|
||||
|
||||
def get_quality_tier_from_extension(file_path):
|
||||
"""Classify a file extension into a quality tier."""
|
||||
if not file_path:
|
||||
|
|
@ -367,14 +524,29 @@ def downsample_hires_flac(final_path, context):
|
|||
return None
|
||||
|
||||
|
||||
def m4a_codec(path):
|
||||
"""Codec of an .m4a/.mp4 container ('alac', 'aac', …) or None — lets the
|
||||
lossless check tell ALAC (lossless) from AAC (lossy), since both are .m4a."""
|
||||
try:
|
||||
from mutagen.mp4 import MP4
|
||||
return (getattr(MP4(path).info, 'codec', '') or '').lower() or None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def create_lossy_copy(final_path):
|
||||
"""Convert a FLAC file to a lossy copy using the configured codec."""
|
||||
from mutagen.flac import FLAC
|
||||
"""Convert a lossless file (FLAC / ALAC / WAV / AIFF / DSD) to a lossy copy
|
||||
using the configured codec. Non-lossless inputs are skipped (#941)."""
|
||||
from core.quality.lossless import (
|
||||
is_lossless_audio_path,
|
||||
lossy_output_would_overwrite_source,
|
||||
)
|
||||
|
||||
if not config_manager.get("lossy_copy.enabled", False):
|
||||
return None
|
||||
|
||||
if os.path.splitext(final_path)[1].lower() != ".flac":
|
||||
# Was FLAC-only; now any lossless source. .m4a is probed (ALAC vs AAC).
|
||||
if not is_lossless_audio_path(final_path, probe_codec=m4a_codec):
|
||||
return None
|
||||
|
||||
codec = config_manager.get("lossy_copy.codec", "mp3").lower()
|
||||
|
|
@ -403,6 +575,16 @@ def create_lossy_copy(final_path):
|
|||
out_basename = out_basename.replace(original_quality, quality_label)
|
||||
out_path = os.path.join(os.path.dirname(out_path), out_basename)
|
||||
|
||||
# Safety invariant: never write the lossy copy over its own source (an .m4a
|
||||
# ALAC source + AAC target lands on the same .m4a path). ffmpeg runs with -y,
|
||||
# so this guard MUST precede it — the later delete-original guard is too late.
|
||||
if lossy_output_would_overwrite_source(final_path, out_path):
|
||||
logger.info(
|
||||
f"[Lossy Copy] Skipping — {codec.upper()} output would overwrite the "
|
||||
f"source: {os.path.basename(final_path)}"
|
||||
)
|
||||
return None
|
||||
|
||||
ffmpeg_bin = shutil.which("ffmpeg")
|
||||
if not ffmpeg_bin:
|
||||
local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg")
|
||||
|
|
|
|||
|
|
@ -105,30 +105,72 @@ def move_to_quarantine(file_path: str, context: dict, reason: str, automation_en
|
|||
|
||||
|
||||
def check_flac_bit_depth(file_path: str, context: dict) -> Optional[str]:
|
||||
"""Return a rejection message if a FLAC file violates the configured bit depth."""
|
||||
if not context.get("_audio_quality", "").startswith("FLAC"):
|
||||
"""Legacy wrapper — delegates to check_quality_target.
|
||||
|
||||
Kept for callers that still pass trigger='bit_depth'; the new guard
|
||||
covers bit_depth as part of the full quality target check.
|
||||
"""
|
||||
return check_quality_target(file_path, context)
|
||||
|
||||
|
||||
def check_quality_target(file_path: str, context: dict) -> Optional[str]:
|
||||
"""Return a rejection message when the downloaded file does not satisfy
|
||||
the user's quality priority list.
|
||||
|
||||
Probes the actual file with mutagen (ground-truth sample_rate,
|
||||
bit_depth, bitrate) and checks it against the profile's
|
||||
``ranked_targets``. Falls back gracefully when fallback_enabled=True.
|
||||
|
||||
Works for all formats and all download sources — no Soulseek-specific
|
||||
logic here.
|
||||
"""
|
||||
from core.imports.file_ops import probe_audio_quality
|
||||
from core.quality.selection import targets_from_profile, quality_meets_profile
|
||||
|
||||
# Master toggle (Settings → Import). When OFF, the quality check is skipped
|
||||
# entirely and files import regardless of quality — the user opted out of
|
||||
# quality-filtering on import. Default ON preserves existing behaviour. The
|
||||
# library Quality Upgrade scanner still flags below-profile files either way.
|
||||
if _get_config_manager().get("import.quality_filter_enabled", True) is False:
|
||||
logger.debug(
|
||||
"[QualityGuard] import.quality_filter_enabled=False — skipping quality "
|
||||
"filter for %s", os.path.basename(file_path),
|
||||
)
|
||||
return None
|
||||
|
||||
quality_profile = MusicDatabase().get_quality_profile()
|
||||
flac_config = quality_profile.get("qualities", {}).get("flac", {})
|
||||
flac_pref = flac_config.get("bit_depth", "any")
|
||||
if flac_pref == "any":
|
||||
aq = probe_audio_quality(file_path)
|
||||
if aq is None:
|
||||
logger.debug("[QualityGuard] Could not probe %s — skipping check", os.path.basename(file_path))
|
||||
return None
|
||||
|
||||
actual_bits = context["_audio_quality"].replace("FLAC ", "").replace("bit", "")
|
||||
if actual_bits == flac_pref:
|
||||
profile = MusicDatabase().get_quality_profile()
|
||||
targets, fallback_enabled = targets_from_profile(profile)
|
||||
|
||||
if not targets:
|
||||
return None
|
||||
|
||||
flac_fallback = flac_config.get("bit_depth_fallback", True)
|
||||
downsample_enabled = _get_config_manager().get("lossy_copy.downsample_hires", False)
|
||||
|
||||
matched = quality_meets_profile(aq, targets)
|
||||
|
||||
track_info = context.get("track_info", {})
|
||||
track_name = track_info.get("name", os.path.basename(file_path))
|
||||
actual_label = aq.label()
|
||||
|
||||
if flac_fallback or downsample_enabled:
|
||||
if downsample_enabled:
|
||||
logger.info("[FLAC Downsample] Accepted %s-bit FLAC (will be downsampled to %s-bit): %s", actual_bits, flac_pref, track_name)
|
||||
else:
|
||||
logger.warning("[FLAC Fallback] Accepted %s-bit FLAC (preferred %s-bit): %s", actual_bits, flac_pref, track_name)
|
||||
if matched:
|
||||
logger.info("[QualityGuard] %s meets profile: %s", track_name, actual_label)
|
||||
return None
|
||||
|
||||
return f"FLAC bit depth mismatch: file is {actual_bits}-bit, preference is {flac_pref}-bit"
|
||||
# No target matched
|
||||
best_label = targets[0].label if targets else "?"
|
||||
if fallback_enabled or downsample_enabled:
|
||||
logger.warning(
|
||||
"[QualityGuard] %s did not match any target (got %s, wanted %s) — accepting via fallback",
|
||||
track_name, actual_label, best_label,
|
||||
)
|
||||
return None
|
||||
|
||||
return (
|
||||
f"Quality mismatch: file is {actual_label}, "
|
||||
f"does not satisfy any configured target (best wanted: {best_label})"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -158,6 +158,33 @@ def clean_track_title(track_title: str, artist_name: str) -> str:
|
|||
return cleaned if cleaned else original
|
||||
|
||||
|
||||
# A leading track-number prefix is EITHER a zero-padded number (01, 04, 099 — no
|
||||
# real song title starts with one), OR a plain number followed by a real separator
|
||||
# AND a space ("3 - ", "12. "). Deliberately NOT a bare "number space word", so it
|
||||
# leaves "7 Rings", "99 Luftballons", "50 Ways to Leave Your Lover" and
|
||||
# "1-800-273-8255" untouched.
|
||||
_TRACK_NUM_PREFIX_RE = re.compile(r"^\s*(?:0\d{1,2}[\s._)\-]*|\d{1,3}\s*[._)\-]\s+)(?=\S)")
|
||||
|
||||
|
||||
def strip_leading_track_number(title: str) -> str:
|
||||
"""Conservatively remove a leading track-number prefix from a track title.
|
||||
|
||||
Fixes #890 — files named ``01 - Sun It Rises.flac`` whose stem leaks into the
|
||||
title as ``01 - Sun It Rises``, which then never matches the canonical
|
||||
``Sun It Rises`` (false "missing"). Only strips an unambiguous track-number
|
||||
prefix; a coincidental leading number that's part of the title is preserved, and
|
||||
it never reduces a title to empty or a bare number."""
|
||||
s = (title or "").strip()
|
||||
if not s:
|
||||
return title or ""
|
||||
stripped = _TRACK_NUM_PREFIX_RE.sub("", s, count=1).strip()
|
||||
# Keep the original if stripping left nothing real — empty, a bare number, or
|
||||
# only punctuation (e.g. "01 - " → "-"). A real title has a letter/digit.
|
||||
if stripped.isdigit() or not re.search(r"[^\W_]", stripped):
|
||||
return s
|
||||
return stripped
|
||||
|
||||
|
||||
|
||||
|
||||
def get_album_type_display(raw_type, track_count) -> str:
|
||||
|
|
|
|||
|
|
@ -34,9 +34,11 @@ from core.imports.context import (
|
|||
)
|
||||
from core.imports.file_integrity import check_audio_integrity, expected_duration_for_check, resolve_duration_tolerance
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
from core.imports.guards import check_flac_bit_depth, move_to_quarantine
|
||||
from core.imports.guards import check_flac_bit_depth, check_quality_target, move_to_quarantine
|
||||
from core.imports.silence import detect_broken_audio
|
||||
from core.imports.quarantine import (
|
||||
approve_quarantine_entry,
|
||||
delete_quarantine_entry,
|
||||
entry_id_from_quarantined_filename,
|
||||
list_quarantine_entries,
|
||||
)
|
||||
|
|
@ -160,7 +162,9 @@ def import_rejection_reason(context: dict) -> str | None:
|
|||
f"{context.get('_acoustid_failure_msg', 'fingerprint mismatch')}"
|
||||
)
|
||||
if context.get('_bitdepth_rejected'):
|
||||
return "rejected by bit-depth filter"
|
||||
return "rejected by quality filter"
|
||||
if context.get('_silence_rejected'):
|
||||
return "rejected by audio guard (incomplete or silent audio)"
|
||||
if context.get('_race_guard_failed'):
|
||||
return "source file disappeared before import completed"
|
||||
return None
|
||||
|
|
@ -321,11 +325,14 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.error(f"File quarantined due to integrity failure: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting broken file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as del_error:
|
||||
logger.error(f"Could not delete broken file either: {del_error}")
|
||||
# Quarantine MOVE failed (e.g. cross-device / permission on a NAS).
|
||||
# Do NOT delete — destroying a download we couldn't even quarantine is
|
||||
# data loss and forces a re-download. Leave it in place so it can be
|
||||
# retried; the task is still marked failed below either way (#kettui).
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file in place "
|
||||
f"for retry (not deleting): {file_path}"
|
||||
)
|
||||
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
|
|
@ -353,6 +360,118 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
f"drift={integrity.checks.get('length_drift_s', 'n/a')})"
|
||||
)
|
||||
|
||||
# Audio-completeness guard — runs right where the length is verified,
|
||||
# BEFORE the AcoustID/quality gates, so a truncated file (container
|
||||
# claims the full length but only ~30s actually decodes, e.g. HiFi/
|
||||
# Monochrome HLS assembly) or a mostly-silent file is caught regardless
|
||||
# of its quality verdict and gets the right reason. Same quarantine +
|
||||
# next-candidate retry pattern as the integrity check.
|
||||
#
|
||||
# Opt-in (default OFF): this is the one check that fully DECODES the file
|
||||
# with ffmpeg, so it is the most CPU-expensive step in post-processing.
|
||||
# Most preview/truncation cases are already caught at the source
|
||||
# (HiFi/Qobuz have their own guards), so it stays off unless the user
|
||||
# turns it on under Settings → Post-processing.
|
||||
_audio_guard_enabled = config_manager.get('post_processing.audio_completeness_check', False)
|
||||
_skip_audio = (not _audio_guard_enabled) or _should_skip_quarantine_check(context, 'silence')
|
||||
audio_reason = None if _skip_audio else detect_broken_audio(file_path)
|
||||
if audio_reason:
|
||||
logger.error(f"[AudioGuard] Rejected {_basename}: {audio_reason}")
|
||||
context['_silence_rejected'] = True
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path, context, audio_reason, automation_engine,
|
||||
trigger='silence',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.warning("File quarantined — incomplete/silent audio: %s", quarantine_path)
|
||||
except Exception as quarantine_error:
|
||||
# Don't delete a file we couldn't quarantine — leave it for retry
|
||||
# instead of forcing a re-download (data loss). See integrity block.
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file in place "
|
||||
f"for retry (not deleting): {file_path}"
|
||||
)
|
||||
|
||||
with matched_context_lock:
|
||||
matched_downloads_context.pop(context_key, None)
|
||||
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
# Try the next-best candidate before giving up — same pattern as
|
||||
# the integrity / AcoustID / quality failures.
|
||||
if _requeue_quarantined_task_for_retry(task_id, batch_id, 'silence'):
|
||||
logger.info(
|
||||
"Incomplete/silent audio for task %s — retrying next-best candidate: %s",
|
||||
task_id, audio_reason,
|
||||
)
|
||||
return
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Audio guard: {audio_reason}"
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
# QUALITY GATE — runs BEFORE AcoustID so (1) a wrong-quality file is
|
||||
# rejected without paying for an AcoustID fingerprint, and (2) the real
|
||||
# audio quality is recorded on the context (→ quarantine sidecar) for
|
||||
# EVERY trigger, so it's known when reviewing/approving any quarantined
|
||||
# file. force_import still never fires on a quality mismatch.
|
||||
context['_audio_quality'] = get_audio_quality_string(file_path)
|
||||
if context['_audio_quality']:
|
||||
logger.info(f"Audio quality detected: {context['_audio_quality']}")
|
||||
|
||||
_skip_quality = _should_skip_quarantine_check(context, 'bit_depth') or \
|
||||
_should_skip_quarantine_check(context, 'quality')
|
||||
rejection_reason = None if _skip_quality else check_quality_target(file_path, context)
|
||||
if _skip_quality:
|
||||
logger.info(f"[QualityGuard] Skipped (user approval) for {_basename}")
|
||||
if rejection_reason:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
rejection_reason,
|
||||
automation_engine,
|
||||
trigger='quality',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.info(f"File quarantined due to quality mismatch: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
# Don't delete a file we couldn't quarantine — leave it for retry
|
||||
# instead of forcing a re-download (data loss). See integrity block.
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file in place "
|
||||
f"for retry (not deleting): {file_path}"
|
||||
)
|
||||
|
||||
context['_bitdepth_rejected'] = True
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
with matched_context_lock:
|
||||
matched_downloads_context.pop(context_key, None)
|
||||
|
||||
# Try the next-best candidate before giving up — same pattern
|
||||
# as AcoustID and integrity failures.
|
||||
if _requeue_quarantined_task_for_retry(task_id, batch_id, 'quality'):
|
||||
logger.info(
|
||||
"Quality mismatch for task %s — retrying next-best candidate: %s",
|
||||
task_id, rejection_reason,
|
||||
)
|
||||
return
|
||||
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Quality filter: {rejection_reason}"
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
_skip_acoustid = _should_skip_quarantine_check(context, 'acoustid')
|
||||
if _skip_acoustid:
|
||||
logger.info(f"[AcoustID] Skipped (user approval) for {_basename}")
|
||||
|
|
@ -390,24 +509,44 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}")
|
||||
context['_acoustid_result'] = verification_result.value
|
||||
|
||||
if verification_result == VerificationResult.FAIL:
|
||||
# Fail-closed mode: when the user requires a hard AcoustID
|
||||
# PASS, a SKIP (ran but couldn't confirm — no fingerprint
|
||||
# match / cross-script metadata) is treated like a FAIL:
|
||||
# quarantine + try the next candidate, instead of importing
|
||||
# an unverified file. ERROR (rate-limit / infra) is NOT
|
||||
# blocked — that would stall the whole pipeline during an
|
||||
# outage; those still import with their existing flag.
|
||||
require_verified = config_manager.get('acoustid.require_verified', False)
|
||||
_skip_as_fail = (
|
||||
require_verified
|
||||
and verification_result == VerificationResult.SKIP
|
||||
)
|
||||
if _skip_as_fail:
|
||||
verification_msg = (
|
||||
f"AcoustID could not confirm the track and 'require verified' "
|
||||
f"is on — rejecting unverified file ({verification_msg})"
|
||||
)
|
||||
logger.warning("[AcoustID] Require-verified: SKIP treated as FAIL — %s", verification_msg)
|
||||
|
||||
if verification_result == VerificationResult.FAIL or _skip_as_fail:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
verification_msg,
|
||||
automation_engine,
|
||||
trigger='acoustid',
|
||||
trigger='acoustid_unverified' if _skip_as_fail else 'acoustid',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.error(f"File quarantined due to verification failure: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}")
|
||||
logger.error(f"Quarantine failed, deleting wrong file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as del_error:
|
||||
logger.error(f"Could not delete wrong file either: {del_error}")
|
||||
# Don't delete a file we couldn't quarantine — leave it for
|
||||
# retry instead of forcing a re-download (data loss). The
|
||||
# task is still marked failed / requeued below. See integrity.
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file "
|
||||
f"in place for retry (not deleting): {file_path}"
|
||||
)
|
||||
|
||||
context['_acoustid_quarantined'] = True
|
||||
context['_acoustid_failure_msg'] = verification_msg
|
||||
|
|
@ -577,48 +716,6 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
album_info.get('album_name', 'None'),
|
||||
)
|
||||
|
||||
context['_audio_quality'] = get_audio_quality_string(file_path)
|
||||
if context['_audio_quality']:
|
||||
logger.info(f"Audio quality detected: {context['_audio_quality']}")
|
||||
|
||||
_skip_bit_depth = _should_skip_quarantine_check(context, 'bit_depth')
|
||||
rejection_reason = None if _skip_bit_depth else check_flac_bit_depth(file_path, context)
|
||||
if _skip_bit_depth:
|
||||
logger.info(f"[BitDepth] Skipped (user approval) for {_basename}")
|
||||
if rejection_reason:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
rejection_reason,
|
||||
automation_engine,
|
||||
trigger='bit_depth',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.info(f"File quarantined due to bit depth filter: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as e:
|
||||
logger.debug("delete quarantine fallback: %s", e)
|
||||
|
||||
context['_bitdepth_rejected'] = True
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}"
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
file_ext = os.path.splitext(file_path)[1]
|
||||
clean_track_name = get_import_clean_title(
|
||||
context,
|
||||
|
|
@ -629,9 +726,17 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
# See ``core/imports/track_number.py`` for the resolution
|
||||
# chain — pure function, unit-tested in isolation, single
|
||||
# place to fix the rule.
|
||||
from core.imports.track_number import resolve_track_number
|
||||
from core.imports.track_number import resolve_track_number, read_embedded_track_number
|
||||
track_info_for_resolve = context.get('track_info') if isinstance(context, dict) else None
|
||||
track_number = resolve_track_number(album_info, track_info_for_resolve, file_path)
|
||||
# "Track 01" bug: a single Deezer track is matched via an endpoint
|
||||
# that omits track_position, so the context never carried the real
|
||||
# number. The downloaded file itself does (deemix/source wrote it),
|
||||
# so read it as a source between metadata and the filename guess.
|
||||
embedded_track_number = read_embedded_track_number(file_path)
|
||||
track_number = resolve_track_number(
|
||||
album_info, track_info_for_resolve, file_path,
|
||||
embedded_track_number=embedded_track_number,
|
||||
)
|
||||
logger.debug(
|
||||
"Final track_number processing: source=%s album_info=%s resolved=%s",
|
||||
album_info.get('source', 'unknown'),
|
||||
|
|
@ -647,6 +752,16 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
album_info['clean_track_name'] = clean_track_name
|
||||
logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata")
|
||||
|
||||
# Sync the disc number the SAME way (and via the SAME resolver) the embedded
|
||||
# tag will use — otherwise the "Disc N" folder is built from album_info's
|
||||
# original disc while the tag takes the per-track disc, so a disc-2/3 track
|
||||
# lands in the Disc 1 folder and every disc's tracks collapse together (Sokhi).
|
||||
from core.imports.track_number import resolve_disc_for_track
|
||||
_resolved_disc = resolve_disc_for_track(get_import_original_search(context), album_info)
|
||||
if album_info.get('disc_number') != _resolved_disc:
|
||||
logger.info(f"[FIX] Updated album_info disc_number to {_resolved_disc} for consistent metadata")
|
||||
album_info['disc_number'] = _resolved_disc
|
||||
|
||||
final_path, _ = build_final_path_for_track(context, artist_context, album_info, file_ext)
|
||||
logger.info(f"Resolved path: '{final_path}'")
|
||||
context['_final_processed_path'] = final_path
|
||||
|
|
@ -969,6 +1084,22 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
if original_batch_id:
|
||||
context['batch_id'] = original_batch_id
|
||||
|
||||
# Quality / audio-guard quarantine. Unlike acoustid/integrity (whose
|
||||
# retry+fail is driven by THIS wrapper below), the inner pipeline fully
|
||||
# owns the quality and audio-guard outcome: it quarantines the file and
|
||||
# then either re-queues the next-best candidate or marks the task failed
|
||||
# and notifies. The wrapper must NOT continue to the "assume success"
|
||||
# fall-through — that marked the quarantined file Completed, so the same
|
||||
# track showed in BOTH Completed and Quarantine (e.g. an MP3-VBR rejected
|
||||
# by a FLAC-only profile). It must also NOT mark it failed here, which
|
||||
# would clobber a successful next-candidate retry. Just return.
|
||||
if context.get('_bitdepth_rejected') or context.get('_silence_rejected'):
|
||||
logger.info(
|
||||
f"Task {task_id} quarantined by the quality/audio guard — inner "
|
||||
f"pipeline already handled retry/fail; wrapper not marking completed"
|
||||
)
|
||||
return
|
||||
|
||||
if context.get('_race_guard_failed'):
|
||||
logger.info(f"Race guard: source file gone for task {task_id} — marking as failed")
|
||||
with tasks_lock:
|
||||
|
|
@ -982,6 +1113,33 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
return
|
||||
|
||||
if context.get('_acoustid_quarantined'):
|
||||
# Race-condition guard: if the user approved an alternative quarantine
|
||||
# entry for this track while this download was already in flight, the
|
||||
# pipeline will have created a new quarantine entry for the just-finished
|
||||
# file. Delete that entry and exit — the user's choice already won.
|
||||
_approved_alt = False
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
_ct = download_tasks.get(task_id)
|
||||
if isinstance(_ct, dict) and _ct.get('_quarantine_approved_alternative'):
|
||||
_approved_alt = True
|
||||
if _approved_alt:
|
||||
_eid = context.get('_quarantine_entry_id')
|
||||
if _eid:
|
||||
try:
|
||||
from core.imports.guards import _get_config_manager
|
||||
_dl_dir = _get_config_manager().get('soulseek.download_path', './downloads')
|
||||
_qdir = os.path.join(docker_resolve_path(_dl_dir), 'ss_quarantine')
|
||||
delete_quarantine_entry(_qdir, _eid)
|
||||
logger.info(
|
||||
f"[Quarantine] Discarded late entry {_eid} — task {task_id} "
|
||||
f"was cancelled by prior quarantine approval"
|
||||
)
|
||||
except Exception as _dqe:
|
||||
logger.debug(f"[Quarantine] Could not clean up late entry {_eid}: {_dqe}")
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed')
|
||||
# Before failing outright, try the next-best candidate. The wrong
|
||||
# file was just quarantined; re-running the worker (with the bad
|
||||
|
|
@ -1098,6 +1256,10 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
_mark_task_completed(task_id, context.get('track_info'))
|
||||
if context.get('_verification_status'):
|
||||
download_tasks[task_id]['verification_status'] = context['_verification_status']
|
||||
if context.get('_history_id'):
|
||||
download_tasks[task_id]['history_id'] = context['_history_id']
|
||||
if context.get('_audio_quality'):
|
||||
download_tasks[task_id]['quality'] = context['_audio_quality']
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
|
@ -1112,6 +1274,10 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
download_tasks[task_id]['metadata_enhanced'] = True
|
||||
if context.get('_verification_status'):
|
||||
download_tasks[task_id]['verification_status'] = context['_verification_status']
|
||||
if context.get('_history_id'):
|
||||
download_tasks[task_id]['history_id'] = context['_history_id']
|
||||
if context.get('_audio_quality'):
|
||||
download_tasks[task_id]['quality'] = context['_audio_quality']
|
||||
redownload_ctx = download_tasks[task_id].get('_redownload_context')
|
||||
|
||||
with matched_context_lock:
|
||||
|
|
|
|||
|
|
@ -153,6 +153,76 @@ def get_quarantined_source_keys(quarantine_dir: str) -> set:
|
|||
return keys
|
||||
|
||||
|
||||
def quarantine_group_key(
|
||||
expected_artist: Any, expected_track: Any, context: Any = None
|
||||
) -> Optional[str]:
|
||||
"""Grouping key for "the same intended download target".
|
||||
|
||||
#876: when several sources are downloaded for one wishlist/queue
|
||||
track they each fail verification and land in quarantine as separate
|
||||
entries. They are *alternatives for the same song*, so they should
|
||||
group together — and once the user accepts one, the rest are
|
||||
redundant failed attempts at a song they now own.
|
||||
|
||||
The key identifies the *intended* target — what SoulSync was trying to
|
||||
fetch — NOT the downloaded file's own tags. That matters: the file's
|
||||
metadata is frequently *wrong* (that's why it failed acoustid /
|
||||
integrity), whereas the target is fixed and identical across every
|
||||
alternative for one song.
|
||||
|
||||
Uses ISRC when available (truly universal across sources and batches).
|
||||
Falls back to normalized artist|track name, which is stable across
|
||||
different batches and sources.
|
||||
|
||||
Source-specific IDs (Spotify track id, Qobuz id, uri) are intentionally
|
||||
NOT used: the same song imported from different playlists or sources gets
|
||||
different source IDs, so id-based keys break cross-batch sibling matching.
|
||||
|
||||
Returns ``None`` when nothing identifies the target (no usable id and
|
||||
both name fields empty). Callers treat a ``None`` key as "its own
|
||||
singleton group" — ungroupable entries must never collapse together.
|
||||
"""
|
||||
ti = {}
|
||||
if isinstance(context, dict):
|
||||
maybe_ti = context.get("track_info")
|
||||
if isinstance(maybe_ti, dict):
|
||||
ti = maybe_ti
|
||||
isrc = str(ti.get("isrc") or "").strip().lower()
|
||||
if isrc:
|
||||
return f"isrc:{isrc}"
|
||||
artist = " ".join(str(expected_artist or "").split()).lower()
|
||||
track = " ".join(str(expected_track or "").split()).lower()
|
||||
if not artist and not track:
|
||||
return None
|
||||
return f"nm:{artist}|{track}"
|
||||
|
||||
|
||||
def find_quarantine_siblings(quarantine_dir: str, entry_id: str) -> List[str]:
|
||||
"""Other entry ids that share ``entry_id``'s intended-target group key.
|
||||
|
||||
Returns the ids of every *other* quarantine entry whose
|
||||
`expected_artist`/`expected_track` normalize to the same key as
|
||||
``entry_id`` (see :func:`quarantine_group_key`). Excludes ``entry_id``
|
||||
itself. Returns ``[]`` when the entry is missing, has an ungroupable
|
||||
(``None``) key, or has no siblings. Never raises.
|
||||
"""
|
||||
if not entry_id:
|
||||
return []
|
||||
entries = list_quarantine_entries(quarantine_dir)
|
||||
target_key = None
|
||||
for e in entries:
|
||||
if e.get("id") == entry_id:
|
||||
target_key = e.get("group_key")
|
||||
break
|
||||
if target_key is None:
|
||||
return []
|
||||
return [
|
||||
e["id"]
|
||||
for e in entries
|
||||
if e.get("id") != entry_id and e.get("group_key") == target_key
|
||||
]
|
||||
|
||||
|
||||
def list_quarantine_entries(quarantine_dir: str) -> List[Dict[str, Any]]:
|
||||
"""Enumerate quarantined files paired with their sidecars.
|
||||
|
||||
|
|
@ -213,6 +283,11 @@ def list_quarantine_entries(quarantine_dir: str) -> List[Dict[str, Any]]:
|
|||
"reason": sidecar.get("quarantine_reason", "Unknown reason"),
|
||||
"expected_track": sidecar.get("expected_track", ""),
|
||||
"expected_artist": sidecar.get("expected_artist", ""),
|
||||
"group_key": quarantine_group_key(
|
||||
sidecar.get("expected_artist", ""),
|
||||
sidecar.get("expected_track", ""),
|
||||
ctx,
|
||||
),
|
||||
"timestamp": sidecar.get("timestamp", ""),
|
||||
"size_bytes": size_bytes,
|
||||
"has_full_context": isinstance(sidecar.get("context"), dict),
|
||||
|
|
@ -220,6 +295,10 @@ def list_quarantine_entries(quarantine_dir: str) -> List[Dict[str, Any]]:
|
|||
"source_username": source_username,
|
||||
"source_filename": source_filename,
|
||||
"thumb_url": _extract_context_thumb(ctx),
|
||||
# Real probed audio quality (recorded on the context before the
|
||||
# quality/AcoustID gates) so the review UI shows what the file
|
||||
# actually is when deciding to approve/delete.
|
||||
"quality": ctx.get("_audio_quality", "") if isinstance(ctx, dict) else "",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
92
core/imports/rematch_apply.py
Normal file
92
core/imports/rematch_apply.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
"""#889 Phase 4/5: apply a re-identify — stage the library file + write the hint.
|
||||
|
||||
When the user confirms a release in the Re-identify modal, we:
|
||||
1. COPY (never move) the track's library file into the auto-import staging folder,
|
||||
so the original is untouched until the re-import succeeds,
|
||||
2. fingerprint the staged copy (rename-proof binding), and
|
||||
3. write a single-use hint carrying the chosen release's IDs (+ ``replace_track_id``
|
||||
when 'replace original' is ticked).
|
||||
|
||||
The auto-import worker then picks the staged file up, finds the hint, and re-imports
|
||||
it against the user-chosen release (Phase 2). The pieces here are split so the
|
||||
naming + hint construction are pure/unit-tested and the actual copy is injectable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from core.imports.paths import sanitize_filename
|
||||
from core.imports.rematch_hints import RematchHint, quick_file_signature
|
||||
|
||||
|
||||
def staged_destination(staging_dir: str, real_path: str, library_track_id: Any) -> str:
|
||||
"""Where the staged copy lands: a single loose file in the staging ROOT (so the
|
||||
worker treats it as a single-track candidate), named to keep the extension and
|
||||
be unique + traceable to the track it re-identifies. The filename is cosmetic —
|
||||
matching is driven by the hint, not the name."""
|
||||
base = os.path.basename(real_path)
|
||||
stem, ext = os.path.splitext(base)
|
||||
safe_stem = sanitize_filename(stem).strip() or "track"
|
||||
name = f"{safe_stem} [reid-{library_track_id}]{ext}"
|
||||
return os.path.join(staging_dir, name)
|
||||
|
||||
|
||||
def stage_file_for_reidentify(
|
||||
real_path: str,
|
||||
staging_dir: str,
|
||||
library_track_id: Any,
|
||||
*,
|
||||
copy_fn: Callable[[str, str], object] = shutil.copy2,
|
||||
signature_fn: Callable[[str], Optional[str]] = quick_file_signature,
|
||||
) -> Dict[str, Any]:
|
||||
"""Copy the library file into staging and fingerprint the copy. Returns
|
||||
``{staged_path, content_hash}``. Raises ``FileNotFoundError`` if the source is
|
||||
gone (caller surfaces a clear error rather than writing a dangling hint)."""
|
||||
if not real_path or not os.path.isfile(real_path):
|
||||
raise FileNotFoundError(real_path or "(empty path)")
|
||||
os.makedirs(staging_dir, exist_ok=True)
|
||||
dest = staged_destination(staging_dir, real_path, library_track_id)
|
||||
copy_fn(real_path, dest)
|
||||
return {"staged_path": dest, "content_hash": signature_fn(dest)}
|
||||
|
||||
|
||||
def build_reidentify_hint(
|
||||
library_track_id: Any,
|
||||
hint_fields: Dict[str, Any],
|
||||
staged_path: str,
|
||||
content_hash: Optional[str],
|
||||
*,
|
||||
replace: bool,
|
||||
) -> RematchHint:
|
||||
"""Pure: assemble the RematchHint from the resolved release fields + staging
|
||||
info. ``replace_track_id`` is the library row to delete on success, but only
|
||||
when 'replace original' was ticked. ``exempt_dedup`` is always True — a
|
||||
re-identify is explicit and must bypass dedup-skip."""
|
||||
return RematchHint(
|
||||
staged_path=staged_path,
|
||||
content_hash=content_hash,
|
||||
source=hint_fields.get("source") or "",
|
||||
isrc=hint_fields.get("isrc"),
|
||||
track_id=hint_fields.get("track_id"),
|
||||
album_id=hint_fields.get("album_id"),
|
||||
artist_id=hint_fields.get("artist_id"),
|
||||
track_title=hint_fields.get("track_title"),
|
||||
album_name=hint_fields.get("album_name"),
|
||||
artist_name=hint_fields.get("artist_name"),
|
||||
album_type=hint_fields.get("album_type"),
|
||||
track_number=hint_fields.get("track_number"),
|
||||
disc_number=hint_fields.get("disc_number"),
|
||||
replace_track_id=(int(library_track_id) if replace and str(library_track_id).isdigit() else
|
||||
(library_track_id if replace else None)),
|
||||
exempt_dedup=True,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"staged_destination",
|
||||
"stage_file_for_reidentify",
|
||||
"build_reidentify_hint",
|
||||
]
|
||||
334
core/imports/rematch_hints.py
Normal file
334
core/imports/rematch_hints.py
Normal file
|
|
@ -0,0 +1,334 @@
|
|||
"""Re-identify hints (#889) — a single-use, user-designated answer to "which
|
||||
release does this already-imported track belong to".
|
||||
|
||||
Flow: the user clicks *Re-identify* on a library track, searches a source, and
|
||||
picks the exact release (single / EP / album) it should live under. We write a
|
||||
**hint** here and stage the file for auto-import. The import flow then reads the
|
||||
hint at the very TOP of matching — before any fuzzy tier — builds the match from
|
||||
these exact IDs, and consumes the row. So the original ambiguity that mis-filed
|
||||
the track (which release?) is gone: the user already answered it.
|
||||
|
||||
Two safety properties live in the hint, not the import code:
|
||||
|
||||
- ``replace_track_id`` — the library row to delete AFTER the re-import lands (so a
|
||||
re-identify *replaces* rather than *duplicates*). Cleanup is deferred to success
|
||||
so a failed import can never lose the file.
|
||||
- ``exempt_dedup`` — always set: a re-identify is an explicit user action and must
|
||||
not be silently dropped by the quality dedup-skip (which would otherwise see the
|
||||
incoming file as a duplicate of the very row we're replacing).
|
||||
|
||||
This module is pure DB mechanics over an injected ``cursor`` (sqlite3-style,
|
||||
``?`` params) — no connection management, no app state — so the create / find /
|
||||
consume seam is unit-tested against an in-memory DB with no live metadata client.
|
||||
The binding is keyed on the staged path, with ``content_hash`` as a rename-proof
|
||||
fallback in case the staging watcher normalizes the filename on ingest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
# Columns in INSERT/SELECT order — single source of truth so the dataclass, the
|
||||
# write, and the read can't drift apart.
|
||||
_FIELDS = (
|
||||
"staged_path",
|
||||
"content_hash",
|
||||
"source",
|
||||
"isrc",
|
||||
"track_id",
|
||||
"album_id",
|
||||
"artist_id",
|
||||
"track_title",
|
||||
"album_name",
|
||||
"artist_name",
|
||||
"album_type",
|
||||
"track_number",
|
||||
"disc_number",
|
||||
"replace_track_id",
|
||||
"exempt_dedup",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RematchHint:
|
||||
"""One user-designated re-identify answer. ``id``/``status`` are set by the DB."""
|
||||
staged_path: str
|
||||
source: str
|
||||
content_hash: Optional[str] = None
|
||||
isrc: Optional[str] = None
|
||||
track_id: Optional[str] = None
|
||||
album_id: Optional[str] = None
|
||||
artist_id: Optional[str] = None
|
||||
track_title: Optional[str] = None
|
||||
album_name: Optional[str] = None
|
||||
artist_name: Optional[str] = None
|
||||
album_type: Optional[str] = None
|
||||
track_number: Optional[int] = None
|
||||
disc_number: Optional[int] = None
|
||||
replace_track_id: Optional[int] = None
|
||||
exempt_dedup: bool = True
|
||||
id: Optional[int] = None
|
||||
status: str = "pending"
|
||||
|
||||
def _values(self) -> tuple:
|
||||
return (
|
||||
self.staged_path,
|
||||
self.content_hash,
|
||||
self.source,
|
||||
self.isrc,
|
||||
self.track_id,
|
||||
self.album_id,
|
||||
self.artist_id,
|
||||
self.track_title,
|
||||
self.album_name,
|
||||
self.artist_name,
|
||||
self.album_type,
|
||||
self.track_number,
|
||||
self.disc_number,
|
||||
self.replace_track_id,
|
||||
1 if self.exempt_dedup else 0,
|
||||
)
|
||||
|
||||
|
||||
def _row_to_hint(row: Any) -> RematchHint:
|
||||
"""Map a sqlite3.Row (or any mapping/sequence-by-name) to a RematchHint."""
|
||||
def g(key, default=None):
|
||||
try:
|
||||
return row[key]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return default
|
||||
return RematchHint(
|
||||
id=g("id"),
|
||||
staged_path=g("staged_path") or "",
|
||||
content_hash=g("content_hash"),
|
||||
source=g("source") or "",
|
||||
isrc=g("isrc"),
|
||||
track_id=g("track_id"),
|
||||
album_id=g("album_id"),
|
||||
artist_id=g("artist_id"),
|
||||
track_title=g("track_title"),
|
||||
album_name=g("album_name"),
|
||||
artist_name=g("artist_name"),
|
||||
album_type=g("album_type"),
|
||||
track_number=g("track_number"),
|
||||
disc_number=g("disc_number"),
|
||||
replace_track_id=g("replace_track_id"),
|
||||
exempt_dedup=bool(g("exempt_dedup", 1)),
|
||||
status=g("status") or "pending",
|
||||
)
|
||||
|
||||
|
||||
def create_hint(cursor: Any, hint: RematchHint) -> int:
|
||||
"""Insert a pending hint; return its new id. Caller owns commit."""
|
||||
placeholders = ", ".join("?" for _ in _FIELDS)
|
||||
cursor.execute(
|
||||
f"INSERT INTO rematch_hints ({', '.join(_FIELDS)}) VALUES ({placeholders})",
|
||||
hint._values(),
|
||||
)
|
||||
new_id = cursor.lastrowid
|
||||
hint.id = new_id
|
||||
return new_id
|
||||
|
||||
|
||||
def find_hint_for_file(
|
||||
cursor: Any,
|
||||
staged_path: str,
|
||||
content_hash: Optional[str] = None,
|
||||
) -> Optional[RematchHint]:
|
||||
"""Return the newest PENDING hint for a staged file, or ``None``.
|
||||
|
||||
Matched by exact ``staged_path`` first; if that misses and a ``content_hash``
|
||||
is given, fall back to it (covers a staging watcher that renamed the file on
|
||||
ingest). Only ``status='pending'`` rows are returned, so a consumed hint is
|
||||
never reused."""
|
||||
if staged_path:
|
||||
cursor.execute(
|
||||
"SELECT * FROM rematch_hints WHERE staged_path = ? AND status = 'pending' "
|
||||
"ORDER BY id DESC LIMIT 1",
|
||||
(staged_path,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row is not None:
|
||||
return _row_to_hint(row)
|
||||
# Try by basename too — the watcher may move the file into a different dir.
|
||||
base = os.path.basename(staged_path)
|
||||
if base and base != staged_path:
|
||||
cursor.execute(
|
||||
"SELECT * FROM rematch_hints WHERE staged_path LIKE ? AND status = 'pending' "
|
||||
"ORDER BY id DESC LIMIT 1",
|
||||
("%/" + base,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row is not None:
|
||||
return _row_to_hint(row)
|
||||
|
||||
if content_hash:
|
||||
cursor.execute(
|
||||
"SELECT * FROM rematch_hints WHERE content_hash = ? AND status = 'pending' "
|
||||
"ORDER BY id DESC LIMIT 1",
|
||||
(content_hash,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row is not None:
|
||||
return _row_to_hint(row)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def consume_hint(cursor: Any, hint_id: int) -> None:
|
||||
"""Mark a hint consumed (single-use). Caller owns commit."""
|
||||
cursor.execute(
|
||||
"UPDATE rematch_hints SET status = 'consumed', consumed_at = CURRENT_TIMESTAMP "
|
||||
"WHERE id = ?",
|
||||
(hint_id,),
|
||||
)
|
||||
|
||||
|
||||
def list_pending_hints(cursor: Any) -> list:
|
||||
"""All pending hints (newest first) — for a 'pending re-identify' view and
|
||||
orphan recovery when a staged file never imports."""
|
||||
cursor.execute("SELECT * FROM rematch_hints WHERE status = 'pending' ORDER BY id DESC")
|
||||
return [_row_to_hint(r) for r in cursor.fetchall()]
|
||||
|
||||
|
||||
def build_identification_from_hint(hint: RematchHint) -> dict:
|
||||
"""Turn a hint into the ``identification`` dict the auto-import matcher expects,
|
||||
so a re-identify SKIPS the guessing tiers entirely and matches straight against
|
||||
the user-chosen release. Mirrors the shape `_identify_folder` returns (album_id
|
||||
/ source / track_number drive the album fetch + file→track match)."""
|
||||
return {
|
||||
"album_id": hint.album_id or None,
|
||||
"album_name": hint.album_name or hint.track_title or "",
|
||||
"artist_name": hint.artist_name or "",
|
||||
"artist_id": hint.artist_id or "",
|
||||
"track_name": hint.track_title or "",
|
||||
"track_id": hint.track_id or "",
|
||||
"image_url": "",
|
||||
"release_date": "",
|
||||
"track_number": hint.track_number or 1,
|
||||
"total_tracks": 1,
|
||||
"source": hint.source,
|
||||
"method": "rematch_hint",
|
||||
"identification_confidence": 1.0,
|
||||
# is_single reflects the CHOSEN release, but force_album_match makes the
|
||||
# matcher FETCH that release (even for a lone staged file) instead of taking
|
||||
# the singles fast-path — so the re-imported track gets the real album
|
||||
# metadata: year, the correct in-album track number, and the album art.
|
||||
"is_single": (str(hint.album_type or "").lower() == "single"),
|
||||
"force_album_match": True,
|
||||
"album_type": hint.album_type,
|
||||
}
|
||||
|
||||
|
||||
def _canonical(path: Optional[str]) -> str:
|
||||
"""Canonical form of a path for same-file comparison (symlinks + case + sep)."""
|
||||
if not path:
|
||||
return ""
|
||||
try:
|
||||
return os.path.normcase(os.path.realpath(path))
|
||||
except OSError:
|
||||
return os.path.normcase(os.path.normpath(path))
|
||||
|
||||
|
||||
def delete_replaced_track(
|
||||
cursor: Any,
|
||||
replace_track_id: Any,
|
||||
*,
|
||||
unlink=os.remove,
|
||||
resolve_fn: Optional[Callable[[str], Optional[str]]] = None,
|
||||
new_paths: Optional[list] = None,
|
||||
) -> Optional[str]:
|
||||
"""Remove the OLD library row a re-identify replaces, and its file.
|
||||
|
||||
Called only AFTER the re-import has landed the track at its new home, so the
|
||||
original is never lost on failure. Safe by construction:
|
||||
|
||||
* **Same-home guard (CRITICAL):** if the re-import landed at the SAME file as the
|
||||
old one (``new_paths`` — the paths the import actually wrote), this is a no-op:
|
||||
we DON'T delete the row or the file, because that file IS the re-imported track.
|
||||
This is what stops "re-identify to the release it's already in" from deleting
|
||||
the file (the import reuses the same row, so deleting it would orphan the file).
|
||||
* the file is unlinked only if it still exists and **no other track row references
|
||||
it** (guards against yanking a file a different row legitimately points to).
|
||||
|
||||
Returns the path it removed, or ``None`` if there was nothing to do. ``unlink`` is
|
||||
injectable for tests. ``resolve_fn`` maps the STORED DB path to the file's actual
|
||||
on-disk location (the stored path may be a Docker/media-server view this process
|
||||
can't read literally — without it we'd delete the row but orphan the file)."""
|
||||
if not replace_track_id:
|
||||
return None
|
||||
cursor.execute("SELECT file_path FROM tracks WHERE id = ?", (replace_track_id,))
|
||||
row = cursor.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
old_path = (row["file_path"] if not isinstance(row, (tuple, list)) else row[0]) or ""
|
||||
if not old_path:
|
||||
cursor.execute("DELETE FROM tracks WHERE id = ?", (replace_track_id,))
|
||||
return None
|
||||
|
||||
# Resolve the old stored path to its real on-disk location up front.
|
||||
real_path = old_path
|
||||
if resolve_fn is not None:
|
||||
try:
|
||||
real_path = resolve_fn(old_path) or old_path
|
||||
except Exception:
|
||||
real_path = old_path
|
||||
|
||||
# Same-home guard: if the re-import wrote to this very file, do NOTHING — the row
|
||||
# is the re-imported track's row and the file is its file. Deleting either would
|
||||
# be data loss (the "picked the same release" bug).
|
||||
if new_paths:
|
||||
landed = {_canonical(p) for p in new_paths if p}
|
||||
if _canonical(real_path) in landed or _canonical(old_path) in landed:
|
||||
return None
|
||||
|
||||
cursor.execute("DELETE FROM tracks WHERE id = ?", (replace_track_id,))
|
||||
# Only unlink if no surviving row still points at this file (rows store the
|
||||
# stored path, so compare against the stored path, not the resolved one).
|
||||
cursor.execute("SELECT 1 FROM tracks WHERE file_path = ? LIMIT 1", (old_path,))
|
||||
if cursor.fetchone() is not None:
|
||||
return None
|
||||
try:
|
||||
if os.path.exists(real_path): # real_path resolved above
|
||||
unlink(real_path)
|
||||
return real_path
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def quick_file_signature(path: str, *, chunk: int = 65536) -> Optional[str]:
|
||||
"""A cheap, rename-proof content fingerprint: size + first/last chunk, hashed.
|
||||
|
||||
Audio files are large, so a full hash is wasteful when we only need to re-bind
|
||||
a hint to *this* file after a possible rename. Size + head + tail is plenty to
|
||||
distinguish staged files in practice. Returns ``None`` if the file can't be
|
||||
read (caller falls back to path-only binding)."""
|
||||
import hashlib
|
||||
|
||||
try:
|
||||
size = os.path.getsize(path)
|
||||
h = hashlib.sha256()
|
||||
h.update(str(size).encode())
|
||||
with open(path, "rb") as f:
|
||||
h.update(f.read(chunk))
|
||||
if size > chunk:
|
||||
f.seek(max(0, size - chunk))
|
||||
h.update(f.read(chunk))
|
||||
return h.hexdigest()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RematchHint",
|
||||
"create_hint",
|
||||
"find_hint_for_file",
|
||||
"consume_hint",
|
||||
"list_pending_hints",
|
||||
"build_identification_from_hint",
|
||||
"delete_replaced_track",
|
||||
"quick_file_signature",
|
||||
]
|
||||
247
core/imports/rematch_search.py
Normal file
247
core/imports/rematch_search.py
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
"""#889 Phase 3: search a metadata source for the releases a track appears on.
|
||||
|
||||
The Re-identify modal lets the user search ANY configured source (tabs, defaulting
|
||||
to the active one) and shows the SAME song across its different collections —
|
||||
single / EP / album — so they can pick which release the track should be filed
|
||||
under. Two steps, deliberately split:
|
||||
|
||||
* ``search_release_candidates(source, query)`` — lightweight DISPLAY rows from the
|
||||
normal typed ``search_tracks`` (title, artist, release name, type badge, year,
|
||||
track count, art, ISRC, track_id). No album_id needed to draw the list.
|
||||
* ``resolve_hint_fields(source, track_id)`` — runs ONCE, on the row the user
|
||||
picks: ``get_track_details`` yields the album_id / isrc / track#/disc the hint
|
||||
needs. We don't pay that lookup for every search result, only the chosen one.
|
||||
|
||||
Pure normalization + injected client factory, so the search/normalize/resolve seam
|
||||
is unit-tested with a fake client and no network.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
|
||||
def _get(obj: Any, key: str, default=None):
|
||||
"""Read ``key`` from either an object (attr) or a mapping (item)."""
|
||||
if obj is None:
|
||||
return default
|
||||
if isinstance(obj, dict):
|
||||
return obj.get(key, default)
|
||||
return getattr(obj, key, default)
|
||||
|
||||
|
||||
def _year(release_date: Any) -> Optional[str]:
|
||||
s = str(release_date or "").strip()
|
||||
return s[:4] if len(s) >= 4 and s[:4].isdigit() else None
|
||||
|
||||
|
||||
def infer_release_type(album_type: Any, total_tracks: Any) -> str:
|
||||
"""Normalize a source's release type to one of album / ep / single / compilation.
|
||||
|
||||
Sources disagree: Spotify has no 'EP' — EPs come back as ``album_type='single'``
|
||||
with several tracks; MusicBrainz/Deezer label EPs properly. So when a 'single'
|
||||
carries more than a handful of tracks, call it an EP for the badge. The actual
|
||||
filing is unaffected — that's driven by the real album_id, not this label."""
|
||||
t = str(album_type or "").strip().lower()
|
||||
try:
|
||||
n = int(total_tracks) if total_tracks is not None else 0
|
||||
except (TypeError, ValueError):
|
||||
n = 0
|
||||
if t in ("compilation", "comp"):
|
||||
return "compilation"
|
||||
if t == "ep":
|
||||
return "ep"
|
||||
if t == "album":
|
||||
return "album" # an explicit album stays an album; only 'single' gets promoted to EP
|
||||
if t == "single":
|
||||
# 1–3 tracks → single; 4+ → almost always an EP in practice.
|
||||
return "ep" if n >= 4 else "single"
|
||||
# Unknown type: infer purely from track count.
|
||||
if n >= 7:
|
||||
return "album"
|
||||
if n >= 4:
|
||||
return "ep"
|
||||
if n >= 1:
|
||||
return "single"
|
||||
return t or "album"
|
||||
|
||||
|
||||
def normalize_search_result(result: Any, source: str) -> Optional[Dict[str, Any]]:
|
||||
"""One typed search Track (or raw dict) → a display row, or ``None`` if it has
|
||||
no usable id/title. ``album`` is just a name at search time; album_id is
|
||||
resolved later for the picked row only."""
|
||||
track_id = _get(result, "id") or _get(result, "track_id")
|
||||
title = _get(result, "name") or _get(result, "title")
|
||||
if not track_id or not title:
|
||||
return None
|
||||
|
||||
artists = _get(result, "artists")
|
||||
if isinstance(artists, list):
|
||||
artist_name = ", ".join(str(_get(a, "name", a) if not isinstance(a, str) else a) for a in artists)
|
||||
else:
|
||||
artist_name = str(artists or _get(result, "artist") or "")
|
||||
|
||||
album = _get(result, "album")
|
||||
album_name = album if isinstance(album, str) else (_get(album, "name") or "")
|
||||
raw_type = _get(result, "album_type")
|
||||
total = _get(result, "total_tracks")
|
||||
ext = _get(result, "external_ids") or {}
|
||||
isrc = _get(result, "isrc") or (ext.get("isrc") if isinstance(ext, dict) else None)
|
||||
|
||||
return {
|
||||
"source": source,
|
||||
"track_id": str(track_id),
|
||||
"track_title": str(title),
|
||||
"artist_name": artist_name,
|
||||
"album_name": str(album_name or ""),
|
||||
"album_type": infer_release_type(raw_type, total),
|
||||
"raw_album_type": str(raw_type or ""),
|
||||
"total_tracks": int(total) if isinstance(total, int) else None,
|
||||
"year": _year(_get(result, "release_date")),
|
||||
"image_url": _get(result, "image_url") or "",
|
||||
"isrc": isrc or None,
|
||||
}
|
||||
|
||||
|
||||
def search_release_candidates(
|
||||
source: str,
|
||||
query: str,
|
||||
*,
|
||||
limit: int = 25,
|
||||
client_factory: Optional[Callable[[str], Any]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search ``source`` for tracks matching ``query`` → normalized display rows.
|
||||
|
||||
Returns ``[]`` (never raises) when the source has no client or errors — the UI
|
||||
just shows an empty tab. Rows keep duplicate releases; the UI groups them."""
|
||||
query = (query or "").strip()
|
||||
if not query:
|
||||
return []
|
||||
factory = client_factory or _default_client_factory
|
||||
try:
|
||||
client = factory(source)
|
||||
except Exception:
|
||||
client = None
|
||||
if client is None or not hasattr(client, "search_tracks"):
|
||||
return []
|
||||
try:
|
||||
results = client.search_tracks(query, limit=limit)
|
||||
except TypeError:
|
||||
results = client.search_tracks(query) # clients with no limit kwarg
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
rows: List[Dict[str, Any]] = []
|
||||
for r in results or []:
|
||||
row = normalize_search_result(r, source)
|
||||
if row is not None:
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def resolve_hint_fields(
|
||||
source: str,
|
||||
track_id: str,
|
||||
*,
|
||||
client_factory: Optional[Callable[[str], Any]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Resolve the picked track to the fields a hint needs (album_id critically,
|
||||
plus isrc / track# / disc# / album name+type). One lookup for one chosen row.
|
||||
Returns ``None`` if it can't be resolved (caller surfaces an error)."""
|
||||
factory = client_factory or _default_client_factory
|
||||
try:
|
||||
client = factory(source)
|
||||
except Exception:
|
||||
client = None
|
||||
if client is None or not hasattr(client, "get_track_details"):
|
||||
return None
|
||||
try:
|
||||
details = client.get_track_details(track_id)
|
||||
except Exception:
|
||||
return None
|
||||
if not details:
|
||||
return None
|
||||
|
||||
album = _get(details, "album") or {}
|
||||
album_id = _get(album, "id") if not isinstance(album, str) else None
|
||||
album_name = _get(album, "name") if not isinstance(album, str) else album
|
||||
album_type = _get(album, "album_type") or _get(details, "album_type")
|
||||
total = _get(album, "total_tracks") or _get(details, "total_tracks")
|
||||
|
||||
artists = _get(details, "artists") or []
|
||||
artist_id = None
|
||||
artist_name = ""
|
||||
if isinstance(artists, list) and artists:
|
||||
artist_id = _get(artists[0], "id")
|
||||
artist_name = ", ".join(str(_get(a, "name", a) if not isinstance(a, str) else a) for a in artists)
|
||||
|
||||
ext = _get(details, "external_ids") or {}
|
||||
isrc = _get(details, "isrc") or (ext.get("isrc") if isinstance(ext, dict) else None)
|
||||
|
||||
if not album_id:
|
||||
return None # without an album_id the import can't fetch the tracklist
|
||||
|
||||
return {
|
||||
"source": source,
|
||||
"track_id": str(track_id),
|
||||
"album_id": str(album_id),
|
||||
"artist_id": str(artist_id) if artist_id else None,
|
||||
"track_title": _get(details, "name") or _get(details, "title") or "",
|
||||
"album_name": str(album_name or ""),
|
||||
"artist_name": artist_name,
|
||||
"album_type": infer_release_type(album_type, total),
|
||||
"track_number": _get(details, "track_number"),
|
||||
"disc_number": _get(details, "disc_number") or 1,
|
||||
"isrc": isrc or None,
|
||||
}
|
||||
|
||||
|
||||
def _default_client_factory(source: str):
|
||||
from core.metadata.registry import get_client_for_source
|
||||
return get_client_for_source(source)
|
||||
|
||||
|
||||
def available_sources() -> List[Dict[str, Any]]:
|
||||
"""The source tabs for the modal: every metadata source with a live client,
|
||||
the primary one flagged ``active`` so the UI selects it by default."""
|
||||
from core.metadata.registry import (
|
||||
METADATA_SOURCE_PRIORITY,
|
||||
get_client_for_source,
|
||||
get_primary_source,
|
||||
)
|
||||
|
||||
try:
|
||||
primary = get_primary_source()
|
||||
except Exception:
|
||||
primary = None
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
seen = set()
|
||||
for src in METADATA_SOURCE_PRIORITY:
|
||||
if src in seen:
|
||||
continue
|
||||
seen.add(src)
|
||||
try:
|
||||
client = get_client_for_source(src)
|
||||
except Exception:
|
||||
client = None
|
||||
if client is None or not hasattr(client, "search_tracks"):
|
||||
continue
|
||||
out.append({
|
||||
"source": src,
|
||||
"label": src.replace("_", " ").title(),
|
||||
"active": src == primary,
|
||||
})
|
||||
# Guarantee the primary is selectable + first even if priority ordering missed it.
|
||||
if primary and not any(s["active"] for s in out):
|
||||
out.insert(0, {"source": primary, "label": primary.replace("_", " ").title(), "active": True})
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"infer_release_type",
|
||||
"normalize_search_result",
|
||||
"search_release_candidates",
|
||||
"resolve_hint_fields",
|
||||
"available_sources",
|
||||
]
|
||||
|
|
@ -3,10 +3,12 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import as_completed
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Dict
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from core.imports.album import build_album_import_context, build_album_import_match_payload, resolve_album_artist_context
|
||||
from core.imports.context import get_import_context_artist, get_import_track_info, normalize_import_context
|
||||
|
|
@ -16,6 +18,7 @@ from core.imports.staging import (
|
|||
AUDIO_EXTENSIONS,
|
||||
get_import_suggestions_cache,
|
||||
get_primary_source as _get_primary_source,
|
||||
get_primary_source_label as _get_primary_source_label,
|
||||
get_staging_path as _get_staging_path,
|
||||
read_staging_file_metadata as _read_staging_file_metadata,
|
||||
refresh_import_suggestions_cache as _refresh_import_suggestions_cache,
|
||||
|
|
@ -48,6 +51,7 @@ class ImportRouteRuntime:
|
|||
read_staging_file_metadata: Callable[[str, str], Dict[str, Any]] = _read_staging_file_metadata
|
||||
read_tags: Callable[[str], Any] = _default_read_tags
|
||||
get_primary_source: Callable[[], str] = _get_primary_source
|
||||
get_primary_source_label: Callable[[], str] = _get_primary_source_label
|
||||
search_import_albums: Callable[..., list] = _search_import_albums
|
||||
search_import_tracks: Callable[..., list] = _search_import_tracks
|
||||
build_album_import_match_payload: Callable[..., Dict[str, Any]] = build_album_import_match_payload
|
||||
|
|
@ -69,36 +73,219 @@ class ImportRouteRuntime:
|
|||
logger: Any = module_logger
|
||||
|
||||
|
||||
# ── Shared staging scan ──────────────────────────────────────────────────────
|
||||
# Opening the Import page fires staging files/groups/hints together; each used to
|
||||
# os.walk the whole staging folder AND mutagen-read every file independently — 3×
|
||||
# the directory walk + 3× the tag I/O on every page open (the import-page scan
|
||||
# storm + memory spike, issue #935). They all need the same per-file tag data, so
|
||||
# scan ONCE and let all three derive their views in-memory. A short TTL + a lock
|
||||
# means the three near-simultaneous page-open requests (and any concurrent caller)
|
||||
# share a single scan instead of each kicking off a full re-read.
|
||||
_STAGING_SCAN_LOCK = threading.Lock()
|
||||
_STAGING_SCAN_TTL = 6.0 # seconds — covers the page-open burst; re-scans after
|
||||
_staging_scan_cache: Dict[str, Any] = {"path": None, "ts": 0.0, "records": None}
|
||||
# Bumped by invalidate_staging_scan_cache() so a background scan that finishes after an
|
||||
# import doesn't re-commit stale (pre-import) records (see the generation guard above).
|
||||
_staging_scan_generation: Dict[str, int] = {"value": 0}
|
||||
|
||||
# Background-scan plumbing: a large staging folder (whole-library migration, #947) makes
|
||||
# the synchronous scan exceed gunicorn's 120s request timeout. The runner moves the SAME
|
||||
# scan off the request thread; the endpoints report progress instead of blocking.
|
||||
_staging_scan_status: Dict[str, Any] = {
|
||||
"status": "idle", "scanned": 0, "total": 0, "path": None, "error": None,
|
||||
}
|
||||
_staging_scan_status_lock = threading.Lock()
|
||||
|
||||
|
||||
def _staging_cache_hit(staging_path: str) -> Optional[list]:
|
||||
"""The cached records for ``staging_path`` if still fresh, else None (no scan triggered)."""
|
||||
c = _staging_scan_cache
|
||||
if (c["records"] is not None and c["path"] == staging_path
|
||||
and (time.time() - c["ts"]) < _STAGING_SCAN_TTL):
|
||||
return c["records"]
|
||||
return None
|
||||
|
||||
|
||||
def ensure_background_staging_scan(runtime: ImportRouteRuntime, staging_path: str) -> None:
|
||||
"""Start a background scan for ``staging_path`` unless the cache is warm or a scan for
|
||||
this path is already running. Idempotent — safe to call on every request."""
|
||||
if _staging_cache_hit(staging_path) is not None:
|
||||
return
|
||||
with _staging_scan_status_lock:
|
||||
if (_staging_scan_status["status"] == "scanning"
|
||||
and _staging_scan_status["path"] == staging_path):
|
||||
return
|
||||
_staging_scan_status.update({"status": "scanning", "scanned": 0, "total": 0,
|
||||
"path": staging_path, "error": None})
|
||||
|
||||
def _run() -> None:
|
||||
try:
|
||||
_scan_staging_records(runtime, staging_path, progress=_staging_scan_status)
|
||||
with _staging_scan_status_lock:
|
||||
if _staging_scan_status["path"] == staging_path:
|
||||
_staging_scan_status["status"] = "done"
|
||||
except Exception as exc: # noqa: BLE001 — surface any scan error to the poller
|
||||
with _staging_scan_status_lock:
|
||||
_staging_scan_status.update({"status": "error", "error": str(exc)})
|
||||
|
||||
threading.Thread(target=_run, name="staging-scan", daemon=True).start()
|
||||
|
||||
|
||||
def get_staging_records_or_status(runtime: ImportRouteRuntime, staging_path: str,
|
||||
*, grace_seconds: float = 3.0) -> tuple[str, Any]:
|
||||
"""Non-blocking staging access for the page endpoints. Returns ``("ready", records)``
|
||||
when the cache is warm or the scan completes within ``grace_seconds`` (so small/normal
|
||||
folders still answer in a single request), otherwise ``("scanning", status_dict)`` after
|
||||
making sure a background scan is running."""
|
||||
records = _staging_cache_hit(staging_path)
|
||||
if records is not None:
|
||||
return ("ready", records)
|
||||
ensure_background_staging_scan(runtime, staging_path)
|
||||
deadline = time.time() + max(0.0, grace_seconds)
|
||||
while True:
|
||||
records = _staging_cache_hit(staging_path)
|
||||
if records is not None:
|
||||
return ("ready", records)
|
||||
with _staging_scan_status_lock:
|
||||
status = dict(_staging_scan_status)
|
||||
if status.get("status") == "error":
|
||||
return ("error", status)
|
||||
if time.time() >= deadline:
|
||||
return ("scanning", status)
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def _records_or_scanning_payload(runtime: ImportRouteRuntime, staging_path: str):
|
||||
"""Shared helper for the page endpoints: returns ``(records, None)`` when the scan is
|
||||
ready, or ``(None, payload)`` when a background scan is still running — the caller
|
||||
returns that payload so the page polls + shows progress instead of blocking/timing out.
|
||||
|
||||
A scan error is re-raised so the endpoint's own try/except logs + returns it exactly as
|
||||
when the scan ran inline (preserves the existing error contract)."""
|
||||
state, val = get_staging_records_or_status(runtime, staging_path)
|
||||
if state == "error":
|
||||
raise RuntimeError(val.get("error") or "staging scan failed")
|
||||
if state == "scanning":
|
||||
return None, {"success": True, "scanning": True,
|
||||
"progress": {"scanned": val.get("scanned", 0),
|
||||
"total": val.get("total", 0)}}
|
||||
return val, None
|
||||
|
||||
|
||||
def staging_scan_status(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
||||
"""Lightweight, instant scan-progress poll for the page (no grace-wait, no file I/O) —
|
||||
``ready`` true once the cache is warm and the files/groups/hints calls will answer fast."""
|
||||
try:
|
||||
staging_path = runtime.get_staging_path()
|
||||
except Exception as exc:
|
||||
return {"success": False, "error": str(exc)}, 500
|
||||
with _staging_scan_status_lock:
|
||||
st = dict(_staging_scan_status)
|
||||
return {
|
||||
"success": True,
|
||||
"ready": _staging_cache_hit(staging_path) is not None,
|
||||
"status": st.get("status", "idle"),
|
||||
"scanned": st.get("scanned", 0),
|
||||
"total": st.get("total", 0),
|
||||
"error": st.get("error"),
|
||||
}, 200
|
||||
|
||||
|
||||
def _scan_staging_records(runtime: ImportRouteRuntime, staging_path: str,
|
||||
*, progress: Optional[Dict[str, Any]] = None) -> list[Dict[str, Any]]:
|
||||
"""Walk staging + read each audio file's tags ONCE, returning per-file records
|
||||
that staging files/groups/hints all derive from. Briefly cached + locked so the
|
||||
page-open trio shares a single scan rather than each re-walking and re-reading.
|
||||
|
||||
``progress`` (optional, default None = unchanged behaviour) is a dict the scan
|
||||
updates live with ``total`` (audio-file count, from a fast first pass) and ``scanned``
|
||||
(tag-reads done so far) so a background runner can report progress. A generation guard
|
||||
keeps a scan that finishes AFTER an import (which bumped ``_staging_scan_generation``)
|
||||
from committing stale records to the cache."""
|
||||
now = time.time()
|
||||
cached = _staging_scan_cache
|
||||
if (cached["records"] is not None and cached["path"] == staging_path
|
||||
and (now - cached["ts"]) < _STAGING_SCAN_TTL):
|
||||
return cached["records"]
|
||||
|
||||
with _STAGING_SCAN_LOCK:
|
||||
# Double-check: another request may have filled the cache while we waited.
|
||||
now = time.time()
|
||||
if (cached["records"] is not None and cached["path"] == staging_path
|
||||
and (now - cached["ts"]) < _STAGING_SCAN_TTL):
|
||||
return cached["records"]
|
||||
|
||||
start_generation = _staging_scan_generation["value"]
|
||||
|
||||
# Pass 1 (fast): collect the audio-file list — no tag I/O — so we know the total.
|
||||
audio_files: list[tuple[str, str, Optional[str]]] = []
|
||||
if os.path.isdir(staging_path):
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
rel_dir = os.path.relpath(root, staging_path)
|
||||
top_folder = rel_dir.split(os.sep)[0] if rel_dir != "." else None
|
||||
for fname in filenames:
|
||||
if os.path.splitext(fname)[1].lower() in AUDIO_EXTENSIONS:
|
||||
audio_files.append((root, fname, top_folder))
|
||||
if progress is not None:
|
||||
progress["total"] = len(audio_files)
|
||||
progress["scanned"] = 0
|
||||
|
||||
# Pass 2 (slow): read each file's tags, updating progress as we go.
|
||||
records: list[Dict[str, Any]] = []
|
||||
for root, fname, top_folder in audio_files:
|
||||
full_path = os.path.join(root, fname)
|
||||
rel_path = os.path.relpath(full_path, staging_path)
|
||||
meta = runtime.read_staging_file_metadata(full_path, rel_path)
|
||||
records.append({
|
||||
"filename": fname, "rel_path": rel_path, "full_path": full_path,
|
||||
"extension": os.path.splitext(fname)[1].lower(),
|
||||
"title": meta["title"], "album": meta["album"],
|
||||
"artist": meta["artist"], "albumartist": meta["albumartist"],
|
||||
"track_number": meta["track_number"], "disc_number": meta["disc_number"],
|
||||
"top_folder": top_folder,
|
||||
})
|
||||
if progress is not None:
|
||||
progress["scanned"] += 1
|
||||
|
||||
# Generation guard: if an import invalidated the cache mid-scan, these records are
|
||||
# stale — return them to this caller but do NOT commit them as the shared cache.
|
||||
if _staging_scan_generation["value"] == start_generation:
|
||||
_staging_scan_cache.update({"path": staging_path, "ts": time.time(), "records": records})
|
||||
return records
|
||||
|
||||
|
||||
def invalidate_staging_scan_cache() -> None:
|
||||
"""Drop the cached staging scan (call after an import moves/removes files so the
|
||||
next files/groups/hints request reflects the new state immediately). Also bumps the
|
||||
scan generation so an in-flight background scan won't re-commit pre-import records."""
|
||||
_staging_scan_generation["value"] += 1
|
||||
_staging_scan_cache.update({"path": None, "ts": 0.0, "records": None})
|
||||
|
||||
|
||||
def staging_files(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
||||
"""Scan the staging folder and return audio files with tag metadata."""
|
||||
try:
|
||||
staging_path = runtime.get_staging_path()
|
||||
os.makedirs(staging_path, exist_ok=True)
|
||||
|
||||
files = []
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
for fname in filenames:
|
||||
ext = os.path.splitext(fname)[1].lower()
|
||||
if ext not in AUDIO_EXTENSIONS:
|
||||
continue
|
||||
full_path = os.path.join(root, fname)
|
||||
rel_path = os.path.relpath(full_path, staging_path)
|
||||
records, scanning = _records_or_scanning_payload(runtime, staging_path)
|
||||
if scanning is not None:
|
||||
return scanning, 200
|
||||
|
||||
meta = runtime.read_staging_file_metadata(full_path, rel_path)
|
||||
|
||||
files.append(
|
||||
{
|
||||
"filename": fname,
|
||||
"rel_path": rel_path,
|
||||
"full_path": full_path,
|
||||
"title": meta["title"],
|
||||
"artist": meta["albumartist"] or meta["artist"] or "Unknown Artist",
|
||||
"album": meta["album"],
|
||||
"track_number": meta["track_number"],
|
||||
"disc_number": meta["disc_number"],
|
||||
"extension": ext,
|
||||
}
|
||||
)
|
||||
files = [
|
||||
{
|
||||
"filename": r["filename"],
|
||||
"rel_path": r["rel_path"],
|
||||
"full_path": r["full_path"],
|
||||
"title": r["title"],
|
||||
"artist": r["albumartist"] or r["artist"] or "Unknown Artist",
|
||||
"album": r["album"],
|
||||
"track_number": r["track_number"],
|
||||
"disc_number": r["disc_number"],
|
||||
"extension": r["extension"],
|
||||
}
|
||||
for r in records
|
||||
]
|
||||
|
||||
files.sort(key=lambda f: f["filename"].lower())
|
||||
return {"success": True, "files": files, "staging_path": staging_path}, 200
|
||||
|
|
@ -114,32 +301,28 @@ def staging_groups(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
|||
if not os.path.isdir(staging_path):
|
||||
return {"success": True, "groups": []}, 200
|
||||
|
||||
records, scanning = _records_or_scanning_payload(runtime, staging_path)
|
||||
if scanning is not None:
|
||||
return scanning, 200
|
||||
|
||||
album_groups = {}
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
for fname in filenames:
|
||||
ext = os.path.splitext(fname)[1].lower()
|
||||
if ext not in AUDIO_EXTENSIONS:
|
||||
continue
|
||||
full_path = os.path.join(root, fname)
|
||||
rel_path = os.path.relpath(full_path, staging_path)
|
||||
for r in records:
|
||||
album = r["album"]
|
||||
artist = r["albumartist"] or r["artist"]
|
||||
if not album or not artist:
|
||||
continue
|
||||
|
||||
meta = runtime.read_staging_file_metadata(full_path, rel_path)
|
||||
album = meta["album"]
|
||||
artist = meta["albumartist"] or meta["artist"]
|
||||
if not album or not artist:
|
||||
continue
|
||||
|
||||
key = (album.lower().strip(), artist.lower().strip())
|
||||
if key not in album_groups:
|
||||
album_groups[key] = {"album": album.strip(), "artist": artist.strip(), "files": []}
|
||||
album_groups[key]["files"].append(
|
||||
{
|
||||
"filename": fname,
|
||||
"full_path": full_path,
|
||||
"title": meta["title"],
|
||||
"track_number": meta["track_number"],
|
||||
}
|
||||
)
|
||||
key = (album.lower().strip(), artist.lower().strip())
|
||||
if key not in album_groups:
|
||||
album_groups[key] = {"album": album.strip(), "artist": artist.strip(), "files": []}
|
||||
album_groups[key]["files"].append(
|
||||
{
|
||||
"filename": r["filename"],
|
||||
"full_path": r["full_path"],
|
||||
"title": r["title"],
|
||||
"track_number": r["track_number"],
|
||||
}
|
||||
)
|
||||
|
||||
groups = []
|
||||
for group in album_groups.values():
|
||||
|
|
@ -169,30 +352,21 @@ def staging_hints(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
|||
if not os.path.isdir(staging_path):
|
||||
return {"success": True, "hints": []}, 200
|
||||
|
||||
records, scanning = _records_or_scanning_payload(runtime, staging_path)
|
||||
if scanning is not None:
|
||||
return scanning, 200
|
||||
|
||||
tag_albums = {}
|
||||
folder_hints = {}
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS]
|
||||
if not audio_files:
|
||||
continue
|
||||
for r in records:
|
||||
if r["top_folder"]:
|
||||
folder_hints[r["top_folder"]] = folder_hints.get(r["top_folder"], 0) + 1
|
||||
|
||||
rel_dir = os.path.relpath(root, staging_path)
|
||||
if rel_dir != ".":
|
||||
top_folder = rel_dir.split(os.sep)[0]
|
||||
folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files)
|
||||
|
||||
for fname in audio_files:
|
||||
full_path = os.path.join(root, fname)
|
||||
try:
|
||||
tags = runtime.read_tags(full_path)
|
||||
if tags:
|
||||
album = (tags.get("album") or [None])[0]
|
||||
artist = (tags.get("artist") or (tags.get("albumartist") or [None]))[0]
|
||||
if album:
|
||||
key = (album.strip(), (artist or "").strip())
|
||||
tag_albums[key] = tag_albums.get(key, 0) + 1
|
||||
except Exception as exc:
|
||||
runtime.logger.debug("tag read failed: %s", exc)
|
||||
album = r["album"]
|
||||
artist = r["artist"] or r["albumartist"]
|
||||
if album:
|
||||
key = (album.strip(), (artist or "").strip())
|
||||
tag_albums[key] = tag_albums.get(key, 0) + 1
|
||||
|
||||
queries = []
|
||||
seen_queries_lower = set()
|
||||
|
|
@ -222,7 +396,7 @@ def staging_suggestions() -> tuple[Dict[str, Any], int]:
|
|||
"success": True,
|
||||
"suggestions": cache["suggestions"],
|
||||
"ready": cache["built"],
|
||||
"primary_source": _get_primary_source(),
|
||||
"primary_source": _get_primary_source_label(),
|
||||
}, 200
|
||||
|
||||
|
||||
|
|
@ -239,7 +413,10 @@ def search_albums(runtime: ImportRouteRuntime, query: str, limit: int = 12) -> t
|
|||
runtime.hydrabase_worker.enqueue(query, "albums")
|
||||
|
||||
albums = runtime.search_import_albums(query, limit=limit)
|
||||
return {"success": True, "albums": albums, "primary_source": primary_source}, 200
|
||||
# The label names the user's CONFIGURED source (Spotify Free reads as
|
||||
# 'spotify', not the deezer fallback the functional source downgrades to).
|
||||
return {"success": True, "albums": albums,
|
||||
"primary_source": runtime.get_primary_source_label()}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error searching albums for import: %s", exc)
|
||||
return {"success": False, "error": str(exc)}, 500
|
||||
|
|
@ -366,6 +543,11 @@ def album_process(runtime: ImportRouteRuntime, data: Dict[str, Any]) -> tuple[Di
|
|||
)
|
||||
runtime.refresh_import_suggestions_cache()
|
||||
|
||||
# Files just left staging — drop the shared scan so the next files/groups/hints
|
||||
# reflects reality immediately instead of waiting out the cache TTL.
|
||||
if processed > 0:
|
||||
invalidate_staging_scan_cache()
|
||||
|
||||
return {"success": True, "processed": processed, "total": len(matches), "errors": errors}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error processing album import: %s", exc)
|
||||
|
|
@ -385,7 +567,8 @@ def search_tracks(runtime: ImportRouteRuntime, query: str, limit: int = 10) -> t
|
|||
runtime.hydrabase_worker.enqueue(query, "tracks")
|
||||
|
||||
tracks = runtime.search_import_tracks(query, limit=limit)
|
||||
return {"success": True, "tracks": tracks, "primary_source": primary_source}, 200
|
||||
return {"success": True, "tracks": tracks,
|
||||
"primary_source": runtime.get_primary_source_label()}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error searching tracks for import: %s", exc)
|
||||
return {"success": False, "error": str(exc)}, 500
|
||||
|
|
@ -500,6 +683,10 @@ def singles_process(runtime: ImportRouteRuntime, files: list[Dict[str, Any]]) ->
|
|||
)
|
||||
runtime.refresh_import_suggestions_cache()
|
||||
|
||||
# Files just left staging — drop the shared scan so the list updates immediately.
|
||||
if processed > 0:
|
||||
invalidate_staging_scan_cache()
|
||||
|
||||
return {"success": True, "processed": processed, "total": len(files), "errors": errors}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error processing singles import: %s", exc)
|
||||
|
|
|
|||
|
|
@ -252,7 +252,7 @@ def record_library_history_download(context: Dict[str, Any]) -> None:
|
|||
origin, origin_context = derive_download_origin(context)
|
||||
|
||||
db = get_database()
|
||||
db.add_library_history_entry(
|
||||
_history_id = db.add_library_history_entry(
|
||||
event_type="download",
|
||||
title=title,
|
||||
artist_name=artist_name,
|
||||
|
|
@ -270,6 +270,10 @@ def record_library_history_download(context: Dict[str, Any]) -> None:
|
|||
origin_context=origin_context,
|
||||
verification_status=context.get("_verification_status"),
|
||||
)
|
||||
# Stash the row id so the live download task can link to its
|
||||
# library_history row (the Unverified review queue needs it).
|
||||
if isinstance(_history_id, int) and _history_id > 0:
|
||||
context["_history_id"] = _history_id
|
||||
except Exception as e:
|
||||
logger.debug("library history record failed: %s", e)
|
||||
|
||||
|
|
@ -565,19 +569,22 @@ def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[
|
|||
# ── Album row: same insert-or-fill-empty-fields shape ──
|
||||
album_source_col = source_columns.get("album")
|
||||
|
||||
cursor.execute(
|
||||
"SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'",
|
||||
(album_id,),
|
||||
# Group by CANONICAL release id when we have one (not just the name
|
||||
# string), so differently-named imports of the SAME release land in
|
||||
# one album row instead of splitting — which left the repair jobs
|
||||
# dressing each split row in its own cover art (Sokhi). Precedence:
|
||||
# name-hash id -> source release id -> (title, artist). Falls back to
|
||||
# the legacy name match, so nothing that grouped before stops now.
|
||||
from core.imports.album_grouping import find_existing_soulsync_album_id
|
||||
existing_album_id = find_existing_soulsync_album_id(
|
||||
cursor, name_key_id=album_id, artist_id=artist_id, album_name=album_name,
|
||||
album_source_col=album_source_col, album_source_id=album_source_id,
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
cursor.execute(
|
||||
"SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? AND server_source = 'soulsync' LIMIT 1",
|
||||
(album_name, artist_id),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
album_id = row[0]
|
||||
if existing_album_id is not None:
|
||||
album_id = existing_album_id
|
||||
row = (album_id,)
|
||||
else:
|
||||
row = None
|
||||
|
||||
if row:
|
||||
_fill_empty_columns(
|
||||
|
|
|
|||
289
core/imports/silence.py
Normal file
289
core/imports/silence.py
Normal file
|
|
@ -0,0 +1,289 @@
|
|||
"""Audio-completeness guard — detect files whose container duration looks
|
||||
right but whose REAL audio is far shorter, or mostly silence.
|
||||
|
||||
Motivating bug: HiFi/Monochrome HLS assembly can yield a file whose container
|
||||
claims the full track length (e.g. 3:08) while only ~30s of audio actually
|
||||
decodes — the rest is missing. The duration-agreement and quality guards both
|
||||
pass (mutagen reads the container's 3:08 and the format/bitrate are fine), so
|
||||
nothing catches it until you listen. ffmpeg's ``time=`` even reports 0 with no
|
||||
error on such a file, so the robust signal is to DECODE the audio and compare
|
||||
the real duration (sample count / sample rate, via ``astats``) against the
|
||||
container duration. A separate ``silencedetect`` pass also flags genuine
|
||||
silence-padding.
|
||||
|
||||
The parsers here are pure and unit-tested; the ffmpeg invocations are
|
||||
integration glue that fails open (returns None) when ffmpeg or mutagen can't
|
||||
run, so a tooling problem never blocks a legitimate import.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("imports.silence")
|
||||
|
||||
# Real decoded audio must cover at least this fraction of the container
|
||||
# duration. A legit file decodes to ~100% (encoder padding aside); a truncated
|
||||
# file decodes to a small fraction (the Blossom file: 30s of a 188s container
|
||||
# = 16%). 0.85 leaves generous headroom against false positives.
|
||||
DEFAULT_MIN_DURATION_RATIO = 0.85
|
||||
|
||||
_SAMPLES_RE = re.compile(r"Number of samples:\s*([0-9]+)")
|
||||
|
||||
# Defaults: treat audio below -50 dB lasting >= 2s as silence, and reject when
|
||||
# more than half the track is silent. A normal song — even with quiet intros/
|
||||
# outros — sits far below 0.5; a 30s-real + padded-silence file sits near 0.85.
|
||||
DEFAULT_NOISE_DB = -50
|
||||
DEFAULT_MIN_SILENCE_S = 2.0
|
||||
DEFAULT_THRESHOLD = 0.5
|
||||
|
||||
_SILENCE_DURATION_RE = re.compile(r"silence_duration:\s*([0-9]+(?:\.[0-9]+)?)")
|
||||
|
||||
|
||||
def silence_ratio_from_output(ffmpeg_stderr: str, total_duration_s: float) -> float:
|
||||
"""Fraction of *total_duration_s* covered by detected silence.
|
||||
|
||||
Sums every ``silence_duration: X`` reported by ffmpeg ``silencedetect``
|
||||
and divides by the track length. Capped at 1.0; returns 0.0 when the
|
||||
duration is unknown/zero or no silence was reported.
|
||||
"""
|
||||
if not total_duration_s or total_duration_s <= 0:
|
||||
return 0.0
|
||||
total_silence = sum(float(m) for m in _SILENCE_DURATION_RE.findall(ffmpeg_stderr or ""))
|
||||
if total_silence <= 0:
|
||||
return 0.0
|
||||
return min(total_silence / total_duration_s, 1.0)
|
||||
|
||||
|
||||
def is_mostly_silent_reason(
|
||||
ffmpeg_stderr: str,
|
||||
total_duration_s: float,
|
||||
*,
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
) -> Optional[str]:
|
||||
"""Return a rejection reason when the silent fraction meets *threshold*."""
|
||||
ratio = silence_ratio_from_output(ffmpeg_stderr, total_duration_s)
|
||||
if ratio >= threshold:
|
||||
pct = round(ratio * 100)
|
||||
audible_s = round(total_duration_s * (1 - ratio))
|
||||
return (
|
||||
f"Audio is mostly silent: {pct}% silence (only ~{audible_s}s audible of "
|
||||
f"{round(total_duration_s)}s) — likely a truncated/preview file padded "
|
||||
f"to full length"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _ffmpeg_available() -> bool:
|
||||
try:
|
||||
subprocess.run(
|
||||
["ffmpeg", "-version"],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=10, check=True,
|
||||
)
|
||||
return True
|
||||
except (FileNotFoundError, subprocess.SubprocessError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def _probe_duration_s(file_path: str) -> Optional[float]:
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(file_path)
|
||||
if audio and audio.info and getattr(audio.info, "length", None):
|
||||
return float(audio.info.length)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("silence guard duration probe failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
|
||||
def detect_mostly_silent(
|
||||
file_path: str,
|
||||
*,
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
noise_db: int = DEFAULT_NOISE_DB,
|
||||
min_silence_s: float = DEFAULT_MIN_SILENCE_S,
|
||||
) -> Optional[str]:
|
||||
"""Run ffmpeg ``silencedetect`` over *file_path* and return a rejection
|
||||
reason when the file is mostly silence, else None.
|
||||
|
||||
Fails open: returns None when ffmpeg/mutagen are unavailable or error, so
|
||||
a tooling problem never quarantines a legitimate file.
|
||||
"""
|
||||
if not _ffmpeg_available():
|
||||
logger.debug("silence guard skipped — ffmpeg not available")
|
||||
return None
|
||||
|
||||
total_duration_s = _probe_duration_s(file_path)
|
||||
if not total_duration_s:
|
||||
return None
|
||||
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
"ffmpeg", "-hide_banner", "-nostats", "-i", file_path,
|
||||
"-af", f"silencedetect=noise={noise_db}dB:d={min_silence_s}",
|
||||
"-f", "null", "-",
|
||||
],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
|
||||
timeout=120,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
logger.debug("silence guard ffmpeg run failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace") if proc.stderr else ""
|
||||
return is_mostly_silent_reason(stderr, total_duration_s, threshold=threshold)
|
||||
|
||||
|
||||
# ── Truncation: real decoded duration vs container duration ────────────────
|
||||
|
||||
def measured_duration_from_astats(astats_stderr: str, sample_rate: int) -> Optional[float]:
|
||||
"""Real decoded audio duration in seconds from ffmpeg ``astats`` output.
|
||||
|
||||
``astats`` reports the per-channel ``Number of samples``; dividing by the
|
||||
sample rate gives the true decoded length. Returns None when the sample
|
||||
count or sample rate is unavailable.
|
||||
"""
|
||||
if not sample_rate or sample_rate <= 0:
|
||||
return None
|
||||
m = _SAMPLES_RE.search(astats_stderr or "")
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)) / float(sample_rate)
|
||||
|
||||
|
||||
def is_dsd_path(file_path: str) -> bool:
|
||||
"""True for DSD audio (.dsf / .dff). The decoded-samples truncation check is
|
||||
invalid for DSD: ffmpeg decodes DSD to PCM at a different rate than the DSD
|
||||
container's 2.8 MHz, so samples ÷ container-sample-rate massively under-counts
|
||||
and would falsely report the file as truncated (#939)."""
|
||||
return os.path.splitext(str(file_path or ''))[1].lower() in ('.dsf', '.dff')
|
||||
|
||||
|
||||
def incomplete_audio_reason(
|
||||
measured_s: Optional[float],
|
||||
container_s: Optional[float],
|
||||
*,
|
||||
min_ratio: float = DEFAULT_MIN_DURATION_RATIO,
|
||||
) -> Optional[str]:
|
||||
"""Return a rejection reason when the real decoded duration falls short of
|
||||
the container duration (a truncated file whose metadata over-states length).
|
||||
"""
|
||||
if not measured_s or not container_s or container_s <= 0:
|
||||
return None
|
||||
if measured_s >= container_s * min_ratio:
|
||||
return None
|
||||
pct = round(measured_s / container_s * 100)
|
||||
return (
|
||||
f"Incomplete audio: only ~{round(measured_s)}s actually decodes of a "
|
||||
f"{round(container_s)}s file ({pct}%) — truncated/broken download "
|
||||
f"(container duration over-states the real audio)"
|
||||
)
|
||||
|
||||
|
||||
def _measured_audio_duration_s(file_path: str, sample_rate: int) -> Optional[float]:
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["ffmpeg", "-hide_banner", "-nostats", "-i", file_path,
|
||||
"-af", "astats=metadata=1", "-f", "null", "-"],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=120,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
logger.debug("astats run failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace") if proc.stderr else ""
|
||||
return measured_duration_from_astats(stderr, sample_rate)
|
||||
|
||||
|
||||
def detect_incomplete_audio(
|
||||
file_path: str,
|
||||
*,
|
||||
min_ratio: float = DEFAULT_MIN_DURATION_RATIO,
|
||||
) -> Optional[str]:
|
||||
"""Decode the file and reject when the real audio is far shorter than the
|
||||
container claims. Fails open when ffmpeg/mutagen are unavailable.
|
||||
"""
|
||||
if not _ffmpeg_available():
|
||||
return None
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(file_path)
|
||||
if not (audio and audio.info):
|
||||
return None
|
||||
container_s = float(getattr(audio.info, "length", 0) or 0)
|
||||
sample_rate = int(getattr(audio.info, "sample_rate", 0) or 0)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("container probe failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
measured_s = _measured_audio_duration_s(file_path, sample_rate)
|
||||
return incomplete_audio_reason(measured_s, container_s, min_ratio=min_ratio)
|
||||
|
||||
|
||||
def detect_broken_audio(
|
||||
file_path: str,
|
||||
*,
|
||||
min_ratio: float = DEFAULT_MIN_DURATION_RATIO,
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
noise_db: int = DEFAULT_NOISE_DB,
|
||||
min_silence_s: float = DEFAULT_MIN_SILENCE_S,
|
||||
) -> Optional[str]:
|
||||
"""Combined post-download audio guard: reject a file that is truncated
|
||||
(real audio far shorter than the container) or mostly silence. Returns the
|
||||
first failure reason, or None when the audio looks complete.
|
||||
|
||||
Runs a SINGLE ffmpeg decode pass with both the ``astats`` (truncation) and
|
||||
``silencedetect`` (silence) filters chained — one decode of the file feeds
|
||||
both checks instead of two full decodes. Halves the CPU cost versus running
|
||||
``detect_incomplete_audio`` and ``detect_mostly_silent`` back to back.
|
||||
|
||||
Fails open: returns None when ffmpeg/mutagen are unavailable or error, so a
|
||||
tooling problem never quarantines a legitimate file.
|
||||
"""
|
||||
if not _ffmpeg_available():
|
||||
logger.debug("audio guard skipped — ffmpeg not available")
|
||||
return None
|
||||
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(file_path)
|
||||
if not (audio and audio.info):
|
||||
return None
|
||||
container_s = float(getattr(audio.info, "length", 0) or 0)
|
||||
sample_rate = int(getattr(audio.info, "sample_rate", 0) or 0)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("container probe failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
"ffmpeg", "-hide_banner", "-nostats", "-i", file_path,
|
||||
"-af", f"astats=metadata=1,silencedetect=noise={noise_db}dB:d={min_silence_s}",
|
||||
"-f", "null", "-",
|
||||
],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=120,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
logger.debug("audio guard ffmpeg run failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace") if proc.stderr else ""
|
||||
|
||||
# Truncation check first (real audio far shorter than the container) — but
|
||||
# NOT for DSD: the astats sample-count ÷ DSD-rate math is invalid there and
|
||||
# would always false-positive (#939). Silence detection below still applies.
|
||||
if not is_dsd_path(file_path):
|
||||
measured_s = measured_duration_from_astats(stderr, sample_rate)
|
||||
reason = incomplete_audio_reason(measured_s, container_s, min_ratio=min_ratio)
|
||||
if reason:
|
||||
return reason
|
||||
|
||||
# Then silence-padding (mostly-silent file).
|
||||
return is_mostly_silent_reason(stderr, container_s, threshold=threshold)
|
||||
124
core/imports/single_to_album.py
Normal file
124
core/imports/single_to_album.py
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
"""Single -> parent-album resolution.
|
||||
|
||||
When a track is matched to a SINGLE release (album_type 'single', the single's
|
||||
name usually equal to the track title), it carries the single's name + the
|
||||
single's source album id. The canonical grouping in
|
||||
[core/imports/album_grouping.py] then files it under a different album row than
|
||||
its album-mates, and the album-grouped repair jobs dress that row in the
|
||||
single's art — songs of one album end up with different covers (Sokhi).
|
||||
|
||||
This module re-homes such a track onto the ALBUM it actually belongs to, so it
|
||||
carries the album's name/id and groups with the rest of the album.
|
||||
|
||||
Design: the SELECTION is a pure, conservative function (no I/O), and the lookup
|
||||
loop takes INJECTED fetchers, so both are unit-testable without a live metadata
|
||||
client. CONSERVATIVE by intent — it only re-homes a track when a real
|
||||
``album``-type release's tracklist *contains that exact track*. It never
|
||||
promotes a genuine standalone single and never guesses, because a wrong
|
||||
promotion would mis-home a real single onto an album (the inverse bug).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
_WS = re.compile(r"\s+")
|
||||
# Trailing version qualifiers that differ between a single and its album cut but
|
||||
# don't change track identity (kept conservative — only the obvious ones).
|
||||
_QUALIFIER = re.compile(
|
||||
r"\s*[\(\[]\s*(album version|single version|radio edit|remaster(ed)?( \d{4})?)\s*[\)\]]\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _norm(s: Any) -> str:
|
||||
"""Lowercase, strip a trailing '(Album Version)'-style qualifier, collapse
|
||||
whitespace — so 'Song' matches 'Song (Album Version)'."""
|
||||
t = str(s or "").strip().lower()
|
||||
t = _QUALIFIER.sub("", t)
|
||||
return _WS.sub(" ", t).strip()
|
||||
|
||||
|
||||
def _get(obj: Any, *keys: str, default=None):
|
||||
for k in keys:
|
||||
if isinstance(obj, dict):
|
||||
if obj.get(k) is not None:
|
||||
return obj.get(k)
|
||||
else:
|
||||
v = getattr(obj, k, None)
|
||||
if v is not None:
|
||||
return v
|
||||
return default
|
||||
|
||||
|
||||
def select_parent_album(track_title: str, candidate_albums: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||
"""Pick the parent ALBUM for ``track_title`` from normalized candidates, or
|
||||
None. Each candidate is ``{name, album_type, tracks: [title, ...], ...}``.
|
||||
|
||||
Conservative rules — a candidate qualifies ONLY when:
|
||||
* it is an ``album`` release (never single / ep / compilation), and
|
||||
* its name is not just the track title (that IS the single), and
|
||||
* its tracklist contains the track by exact normalized title.
|
||||
Returns the FIRST qualifying candidate (caller passes them in priority
|
||||
order, so the result is deterministic).
|
||||
"""
|
||||
tgt = _norm(track_title)
|
||||
if not tgt:
|
||||
return None
|
||||
for alb in candidate_albums or []:
|
||||
if str(_get(alb, "album_type", default="album")).lower() != "album":
|
||||
continue
|
||||
if _norm(_get(alb, "name", "title", default="")) == tgt:
|
||||
continue
|
||||
tracks = _get(alb, "tracks", default=[]) or []
|
||||
if any(_norm(t) == tgt for t in tracks):
|
||||
return alb
|
||||
return None
|
||||
|
||||
|
||||
def resolve_single_to_album(
|
||||
track_title: str,
|
||||
*,
|
||||
fetch_album_candidates: Callable[[], List[Dict[str, Any]]],
|
||||
fetch_album_tracks: Callable[[Dict[str, Any]], List[str]],
|
||||
max_albums: int = 8,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Find the parent album for a single-matched track. I/O is INJECTED so this
|
||||
is testable without a live client:
|
||||
* ``fetch_album_candidates()`` -> the artist's ALBUM-type releases (dicts
|
||||
with name/album_type/id/source), in priority order.
|
||||
* ``fetch_album_tracks(album)`` -> that album's track titles.
|
||||
Probes at most ``max_albums`` albums, lazily (stops at the first that
|
||||
contains the track). Fail-safe: any error / no confident match -> None
|
||||
(the track stays as it was matched). Returns the normalized winning album
|
||||
``{name, album_type, album_id, source, tracks}`` or None.
|
||||
"""
|
||||
if not _norm(track_title):
|
||||
return None
|
||||
try:
|
||||
albums = fetch_album_candidates() or []
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
probed = 0
|
||||
for alb in albums:
|
||||
if str(_get(alb, "album_type", default="album")).lower() != "album":
|
||||
continue
|
||||
if probed >= max_albums:
|
||||
break
|
||||
probed += 1
|
||||
try:
|
||||
tracks = fetch_album_tracks(alb) or []
|
||||
except Exception:
|
||||
continue
|
||||
normalized = {
|
||||
"name": _get(alb, "name", "title", default=""),
|
||||
"album_type": "album",
|
||||
"album_id": _get(alb, "id", "album_id"),
|
||||
"source": _get(alb, "source"),
|
||||
"tracks": list(tracks),
|
||||
}
|
||||
if select_parent_album(track_title, [normalized]):
|
||||
return normalized
|
||||
return None
|
||||
|
|
@ -56,6 +56,12 @@ def get_primary_source() -> str:
|
|||
return _get_primary_source()
|
||||
|
||||
|
||||
def get_primary_source_label() -> str:
|
||||
from core.metadata_service import get_primary_source_label as _get_primary_source_label
|
||||
|
||||
return _get_primary_source_label()
|
||||
|
||||
|
||||
def get_source_priority(preferred_source: str):
|
||||
from core.metadata_service import get_source_priority as _get_source_priority
|
||||
|
||||
|
|
|
|||
|
|
@ -65,17 +65,64 @@ def _coerce_spotify_data(track_info: Any) -> dict:
|
|||
return {}
|
||||
|
||||
|
||||
def read_embedded_track_number(file_path: str) -> Optional[int]:
|
||||
"""Read the track position from a downloaded audio file's own tags.
|
||||
|
||||
Streaming sources (Deezer/deemix, Qobuz, Tidal) and most Soulseek
|
||||
uploads write the correct album position into the file itself. That
|
||||
tag is authoritative for the *source's* idea of the track's place on
|
||||
its album — more reliable than a filename guess — so the resolver
|
||||
consults it before falling back to the filename / default-1 floor.
|
||||
|
||||
Issue #874-adjacent / "Track 01" bug: a single Deezer track is matched
|
||||
via Deezer's ``/search/track`` endpoint, which omits ``track_position``
|
||||
(core/deezer_client.py), so the metadata context never carried the
|
||||
real number — but the downloaded file *does* (deemix wrote it). This
|
||||
recovers it with no network call.
|
||||
|
||||
Returns a positive int, or None when the file has no usable
|
||||
tracknumber tag / can't be read. Never raises. Handles the common
|
||||
``"2/15"`` (number/total) form by taking the leading number.
|
||||
"""
|
||||
if not file_path:
|
||||
return None
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(file_path, easy=True)
|
||||
if audio is None:
|
||||
return None
|
||||
raw = audio.get('tracknumber')
|
||||
if isinstance(raw, list):
|
||||
raw = raw[0] if raw else None
|
||||
if raw is None:
|
||||
return None
|
||||
# "2/15" -> "2"; bare "2" -> "2".
|
||||
text = str(raw).split('/', 1)[0].strip()
|
||||
return _coerce_positive(text)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def resolve_track_number(
|
||||
album_info: Any,
|
||||
track_info: Any,
|
||||
file_path: str,
|
||||
embedded_track_number: Any = None,
|
||||
) -> Optional[int]:
|
||||
"""Walk the resolution chain and return the first valid positive
|
||||
int found, or None when every source is missing / unusable.
|
||||
|
||||
Caller is responsible for the final default-1 floor — leaving
|
||||
that out of this function so tests can pin "everything missing
|
||||
Order: album_info -> track_info -> nested spotify_data -> filename ->
|
||||
``embedded_track_number`` (the source-written file tag, when the caller
|
||||
supplies it). Caller is responsible for the final default-1 floor —
|
||||
leaving that out of this function so tests can pin "everything missing
|
||||
returns None" separate from the floor behaviour.
|
||||
|
||||
``embedded_track_number`` is passed in (not read here) so this stays a
|
||||
pure function — the file I/O lives in :func:`read_embedded_track_number`.
|
||||
It is consulted **last**, only when every other source came up empty, so
|
||||
it can never override a value the pre-fix resolver already produced — it
|
||||
only fills the gap that would otherwise hit the default-1 floor.
|
||||
"""
|
||||
album_info = album_info if isinstance(album_info, dict) else {}
|
||||
track_info = track_info if isinstance(track_info, dict) else {}
|
||||
|
|
@ -96,10 +143,69 @@ def resolve_track_number(
|
|||
# default-1 floor is the single source of that fallback —
|
||||
# otherwise this resolver would silently fill 1 and the
|
||||
# downstream floor logic would have no effect.
|
||||
if not file_path:
|
||||
return None
|
||||
if file_path:
|
||||
try:
|
||||
from_filename = extract_explicit_track_number(file_path)
|
||||
except Exception:
|
||||
from_filename = None
|
||||
ff = _coerce_positive(from_filename)
|
||||
if ff is not None:
|
||||
return ff
|
||||
|
||||
# Embedded source-written file tag is consulted LAST — only when every
|
||||
# other source (metadata + the ripped-album "NN - Title" filename) came
|
||||
# up empty. This is deliberate: it can ONLY fill the gap that would
|
||||
# otherwise hit the caller's default-1 floor, so it never overrides a
|
||||
# value the pre-fix resolver would have used. A correctly-named file
|
||||
# with a stale/wrong embedded tag is therefore never regressed.
|
||||
return _coerce_positive(embedded_track_number)
|
||||
|
||||
|
||||
def normalize_disc_number(value) -> int:
|
||||
"""Coerce a disc value to a positive int, defaulting to 1.
|
||||
|
||||
Every track in a multi-disc album MUST carry a disc number, or Jellyfin/Plex
|
||||
leave the disc-less ones floating ungrouped above the disc sections (Sokhi's
|
||||
"tracks 3/9/15 at the top"). Upstream sources can hand back 0, None, '', or a
|
||||
non-numeric string for some tracks — especially when a track resolved to a
|
||||
different edition than its siblings — and the tag-writer only wrote the disc
|
||||
tag when it was truthy, so those tracks lost it entirely on the clear-then-
|
||||
rewrite. Flooring to >=1 here means a track is never written disc-less.
|
||||
"""
|
||||
try:
|
||||
from_filename = extract_explicit_track_number(file_path)
|
||||
except Exception:
|
||||
from_filename = None
|
||||
return _coerce_positive(from_filename)
|
||||
n = int(value) # int, float, or clean int-string
|
||||
except (TypeError, ValueError):
|
||||
try:
|
||||
n = int(float(str(value).strip())) # tolerate "2.0"
|
||||
except (TypeError, ValueError):
|
||||
return 1
|
||||
return n if n >= 1 else 1
|
||||
|
||||
|
||||
def resolve_disc_for_track(original_search, album_info) -> int:
|
||||
"""The disc number for a track — resolved IDENTICALLY for the 'Disc N' folder
|
||||
(import pipeline) and the embedded tag (metadata.source), so the two can never
|
||||
disagree.
|
||||
|
||||
Sokhi: the pipeline synced the resolved track_number into album_info (so the
|
||||
folder matched the tag) but never did the same for disc — the folder used
|
||||
album_info's original disc (often 1) while the tag took the per-track disc
|
||||
(e.g. 2/3 from a MusicBrainz multi-medium release). Result: a disc-2/3 track
|
||||
landed in the Disc 1 folder, collapsing every disc's tracks into one folder.
|
||||
|
||||
Returns the first VALID positive disc — the per-track search's, else the album
|
||||
context's — else 1. A falsy/unknown (0/None/'') per-track disc falls through to
|
||||
the album rather than flooring early. Single source of truth so both call sites
|
||||
stay in lockstep."""
|
||||
for src in ((original_search or {}), (album_info or {})):
|
||||
raw = src.get("disc_number")
|
||||
try:
|
||||
n = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
try:
|
||||
n = int(float(str(raw).strip()))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if n >= 1:
|
||||
return n
|
||||
return 1
|
||||
|
|
|
|||
|
|
@ -103,9 +103,12 @@ def select_version_mismatch_fallback(
|
|||
# don't guess which the user wants.
|
||||
return None
|
||||
|
||||
# First tried = oldest = highest-confidence (the retry walks candidates
|
||||
# best-first). The id is a "<date>_<time>_<name>" timestamp prefix, so the
|
||||
# lexicographically smallest id is the earliest attempt.
|
||||
# First tried = oldest = best (the retry walks candidates best-first; what
|
||||
# "best" means follows the active ordering — confidence-first by default, or
|
||||
# ranked-target quality when best_quality mode / the rank_candidates_by_quality
|
||||
# toggle is on, so this naturally accepts the highest-quality candidate then).
|
||||
# The id is a "<date>_<time>_<name>" timestamp prefix, so the lexicographically
|
||||
# smallest id is the earliest attempt.
|
||||
return min((e for _, e in candidates), key=lambda e: e["id"])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -739,9 +739,27 @@ class iTunesClient:
|
|||
cache = get_metadata_cache()
|
||||
cached = cache.get_entity('itunes', 'album', f"{album_id}_tracks")
|
||||
if cached and cached.get('items'):
|
||||
return cached
|
||||
# #918 follow-up: a tracks entry cached BEFORE the limit=200 fix is truncated
|
||||
# to 50 and survives in the persistent cache (30-day TTL), so every window that
|
||||
# loads this album from cache still shows 50 — not just the one path that was
|
||||
# re-fetched fresh. Self-heal: entries written by the fixed fetch carry
|
||||
# `_complete`; a legacy entry without it is re-validated against the album's
|
||||
# known trackCount and re-fetched if it's short. (trackCount comes from the
|
||||
# collection metadata and is unaffected by the tracks-limit bug.)
|
||||
if cached.get('_complete'):
|
||||
return cached
|
||||
album_meta = cache.get_entity('itunes', 'album', str(album_id))
|
||||
expected = (album_meta or {}).get('trackCount')
|
||||
if not (isinstance(expected, int) and expected > len(cached['items'])):
|
||||
return cached
|
||||
logger.info(
|
||||
"iTunes album %s tracks cache looks truncated (%d cached < %d trackCount) — refetching",
|
||||
album_id, len(cached['items']), expected,
|
||||
)
|
||||
|
||||
results = self._lookup(id=album_id, entity='song')
|
||||
# #918: the iTunes Lookup API returns only 50 related entities unless `limit` is
|
||||
# passed (max 200), so albums >50 tracks were truncated in the download window.
|
||||
results = self._lookup(id=album_id, entity='song', limit=200)
|
||||
|
||||
if not results:
|
||||
return None
|
||||
|
|
@ -763,7 +781,7 @@ class iTunesClient:
|
|||
try:
|
||||
fb_results = self.session.get(
|
||||
self.LOOKUP_URL,
|
||||
params={'id': album_id, 'entity': 'song', 'country': fallback},
|
||||
params={'id': album_id, 'entity': 'song', 'country': fallback, 'limit': 200}, # #918
|
||||
timeout=15
|
||||
)
|
||||
if fb_results.status_code == 200:
|
||||
|
|
@ -849,7 +867,11 @@ class iTunesClient:
|
|||
'items': tracks,
|
||||
'total': len(tracks),
|
||||
'limit': len(tracks),
|
||||
'next': None
|
||||
'next': None,
|
||||
# Marks this entry as fetched with the limit=200 query (#918) so the
|
||||
# stale-cache self-heal above trusts it and never re-fetches in a loop —
|
||||
# important for region-restricted albums where len(tracks) < trackCount.
|
||||
'_complete': True,
|
||||
}
|
||||
|
||||
# Cache the album tracks listing
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from datetime import datetime
|
|||
import json
|
||||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
from core.library.bulk_paginate import paginate_all_items
|
||||
|
||||
# Shared dataclasses live in the neutral media_server package — every
|
||||
# server client used to define a near-identical XTrackInfo /
|
||||
|
|
@ -105,6 +106,7 @@ class JellyfinTrack:
|
|||
self.title = jellyfin_data.get('Name', 'Unknown Track')
|
||||
self.duration = jellyfin_data.get('RunTimeTicks', 0) // 10000 # Convert from ticks to milliseconds
|
||||
self.trackNumber = jellyfin_data.get('IndexNumber')
|
||||
self.discNumber = jellyfin_data.get('ParentIndexNumber') # multi-disc: disc number
|
||||
self.year = jellyfin_data.get('ProductionYear')
|
||||
self.userRating = jellyfin_data.get('UserData', {}).get('Rating')
|
||||
self.addedAt = self._parse_date(jellyfin_data.get('DateCreated'))
|
||||
|
|
@ -511,12 +513,8 @@ class JellyfinClient(MediaServerClient):
|
|||
try:
|
||||
# SIMPLIFIED APPROACH: Fetch all tracks, then all albums separately (robust and fast)
|
||||
logger.info("Fetching all tracks in bulk...")
|
||||
all_tracks = []
|
||||
start_index = 0
|
||||
limit = 10000
|
||||
consecutive_failures = 0
|
||||
|
||||
while True:
|
||||
|
||||
def _fetch_tracks_page(start_index, limit):
|
||||
params = {
|
||||
'ParentId': self.music_library_id,
|
||||
'IncludeItemTypes': 'Audio',
|
||||
|
|
@ -527,41 +525,19 @@ class JellyfinClient(MediaServerClient):
|
|||
'StartIndex': start_index,
|
||||
'Limit': limit
|
||||
}
|
||||
|
||||
response = self._make_request(f'/Users/{self.user_id}/Items', params)
|
||||
|
||||
if not response:
|
||||
consecutive_failures += 1
|
||||
# Wait before retrying — the server may still be processing the timed-out request
|
||||
time.sleep(5)
|
||||
if limit > 1000:
|
||||
limit = limit // 2
|
||||
consecutive_failures = 0 # Reset — give the smaller batch a fair chance
|
||||
logger.warning(f"Track fetch failed - reducing batch size to {limit}")
|
||||
continue
|
||||
elif consecutive_failures >= 2:
|
||||
logger.warning("Multiple track fetch failures at minimum batch size - stopping")
|
||||
break
|
||||
else:
|
||||
logger.warning("Track fetch failed at minimum batch size - retrying once")
|
||||
continue
|
||||
return response.get('Items', []) if response else None # None = failed page
|
||||
|
||||
# Page in modest chunks so progress is reported every page — a single
|
||||
# huge silent request used to trip the 300s no-progress watchdog on
|
||||
# slow servers even though it was alive (see bulk_paginate docstring).
|
||||
all_tracks = paginate_all_items(
|
||||
_fetch_tracks_page,
|
||||
report_progress=self._progress_callback,
|
||||
label="tracks",
|
||||
on_retry_wait=lambda: time.sleep(5),
|
||||
)
|
||||
|
||||
consecutive_failures = 0
|
||||
batch_tracks = response.get('Items', [])
|
||||
if not batch_tracks:
|
||||
break
|
||||
|
||||
all_tracks.extend(batch_tracks)
|
||||
|
||||
if len(batch_tracks) < limit:
|
||||
break
|
||||
|
||||
start_index += limit
|
||||
progress_msg = f"Fetched {len(all_tracks)} tracks so far..."
|
||||
logger.info(f" {progress_msg} (batch size: {limit})")
|
||||
if self._progress_callback:
|
||||
self._progress_callback(progress_msg)
|
||||
|
||||
# Group tracks by album ID for instant lookup
|
||||
self._track_cache = {}
|
||||
for track_data in all_tracks:
|
||||
|
|
@ -577,12 +553,8 @@ class JellyfinClient(MediaServerClient):
|
|||
|
||||
# STEP 2: Fetch all albums in bulk (same proven pattern)
|
||||
logger.info("Fetching all albums in bulk...")
|
||||
all_albums = []
|
||||
start_index = 0
|
||||
limit = 10000
|
||||
consecutive_failures = 0
|
||||
|
||||
while True:
|
||||
|
||||
def _fetch_albums_page(start_index, limit):
|
||||
params = {
|
||||
'ParentId': self.music_library_id,
|
||||
'IncludeItemTypes': 'MusicAlbum',
|
||||
|
|
@ -593,41 +565,16 @@ class JellyfinClient(MediaServerClient):
|
|||
'StartIndex': start_index,
|
||||
'Limit': limit
|
||||
}
|
||||
|
||||
response = self._make_request(f'/Users/{self.user_id}/Items', params)
|
||||
|
||||
if not response:
|
||||
consecutive_failures += 1
|
||||
# Wait before retrying — the server may still be processing the timed-out request
|
||||
time.sleep(5)
|
||||
if limit > 1000:
|
||||
limit = limit // 2
|
||||
consecutive_failures = 0 # Reset — give the smaller batch a fair chance
|
||||
logger.warning(f"Album fetch failed - reducing batch size to {limit}")
|
||||
continue
|
||||
elif consecutive_failures >= 2:
|
||||
logger.warning("Multiple album fetch failures at minimum batch size - stopping")
|
||||
break
|
||||
else:
|
||||
logger.warning("Album fetch failed at minimum batch size - retrying once")
|
||||
continue
|
||||
return response.get('Items', []) if response else None # None = failed page
|
||||
|
||||
all_albums = paginate_all_items(
|
||||
_fetch_albums_page,
|
||||
report_progress=self._progress_callback,
|
||||
label="albums",
|
||||
on_retry_wait=lambda: time.sleep(5),
|
||||
)
|
||||
|
||||
consecutive_failures = 0
|
||||
batch_albums = response.get('Items', [])
|
||||
if not batch_albums:
|
||||
break
|
||||
|
||||
all_albums.extend(batch_albums)
|
||||
|
||||
if len(batch_albums) < limit:
|
||||
break
|
||||
|
||||
start_index += limit
|
||||
progress_msg = f"Fetched {len(all_albums)} albums so far..."
|
||||
logger.info(f" {progress_msg} (batch size: {limit})")
|
||||
if self._progress_callback:
|
||||
self._progress_callback(progress_msg)
|
||||
|
||||
# Group albums by artist ID for instant lookup
|
||||
self._album_cache = {}
|
||||
for album_data in all_albums:
|
||||
|
|
@ -1711,6 +1658,81 @@ class JellyfinClient(MediaServerClient):
|
|||
logger.error(f"Error reconciling Jellyfin playlist '{playlist_name}': {e}")
|
||||
return False
|
||||
|
||||
def get_playlist_track_ids(self, playlist_id: str) -> List[str]:
|
||||
"""The playlist's current track ids (Item Ids), in current order. [] on miss."""
|
||||
if not self.ensure_connection():
|
||||
return []
|
||||
try:
|
||||
resp = self._make_request(f'/Playlists/{playlist_id}/Items', {'UserId': self.user_id})
|
||||
if not resp:
|
||||
return []
|
||||
return [str(i.get('Id')) for i in resp.get('Items', []) if i.get('Id')]
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting Jellyfin playlist track ids '{playlist_id}': {e}")
|
||||
return []
|
||||
|
||||
def reorder_playlist(self, playlist_id, playlist_name: str, ordered_ids) -> bool:
|
||||
"""In-place reorder a playlist to an exact ordered track-id list ('Align
|
||||
playlists'). Removes any current entry whose track NOT in ``ordered_ids``
|
||||
('Mirror source' drops extras; 'Keep extras' keeps them in the list), then
|
||||
moves each desired track to its target index via the Jellyfin Move endpoint.
|
||||
Operates on the existing playlist (DELETE EntryIds + Items/{entryId}/Move/{i})
|
||||
so its poster, name and Id survive — no delete/recreate."""
|
||||
if not self.ensure_connection():
|
||||
return False
|
||||
try:
|
||||
import requests
|
||||
ordered = [str(i) for i in (ordered_ids or []) if str(i)]
|
||||
ordered_set = set(ordered)
|
||||
|
||||
# Entries carry both the track Id and the PlaylistItemId (entry id);
|
||||
# move/remove operate on the entry id.
|
||||
entries = [] # (track_id, entry_id) in current order
|
||||
resp = self._make_request(f'/Playlists/{playlist_id}/Items', {'UserId': self.user_id})
|
||||
if resp:
|
||||
for item in resp.get('Items', []):
|
||||
tid = str(item.get('Id') or '')
|
||||
eid = str(item.get('PlaylistItemId') or '')
|
||||
if tid:
|
||||
entries.append((tid, eid))
|
||||
if not entries:
|
||||
logger.error(f"Jellyfin reorder: no entries for playlist {playlist_id}")
|
||||
return False
|
||||
|
||||
by_tid = {tid: eid for tid, eid in entries}
|
||||
hdr = {'X-Emby-Token': self.api_key}
|
||||
|
||||
# Drop entries not in the desired list (extras, for Mirror source).
|
||||
extra_eids = [eid for tid, eid in entries if tid not in ordered_set and eid]
|
||||
for i in range(0, len(extra_eids), 100):
|
||||
batch = extra_eids[i:i + 100]
|
||||
r = requests.delete(
|
||||
f"{self.base_url}/Playlists/{playlist_id}/Items",
|
||||
params={'EntryIds': ','.join(batch)}, headers=hdr, timeout=30,
|
||||
)
|
||||
if r.status_code not in (200, 204):
|
||||
logger.error(f"Jellyfin reorder remove failed: HTTP {r.status_code}")
|
||||
return False
|
||||
|
||||
# Move each desired track to its target index, ascending — each move
|
||||
# lands the item exactly at index i without disturbing 0..i-1.
|
||||
for idx, tid in enumerate(ordered):
|
||||
eid = by_tid.get(tid)
|
||||
if not eid:
|
||||
continue
|
||||
r = requests.post(
|
||||
f"{self.base_url}/Playlists/{playlist_id}/Items/{eid}/Move/{idx}",
|
||||
headers=hdr, timeout=30,
|
||||
)
|
||||
if r.status_code not in (200, 204):
|
||||
logger.warning(f"Jellyfin reorder move failed for {tid}: HTTP {r.status_code}")
|
||||
return False
|
||||
logger.info(f"Aligned Jellyfin playlist '{playlist_name}' order ({len(ordered)} tracks)")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error reordering Jellyfin playlist '{playlist_name}': {e}")
|
||||
return False
|
||||
|
||||
def update_playlist(self, playlist_name: str, tracks) -> bool:
|
||||
"""Update an existing playlist or create it if it doesn't exist"""
|
||||
if not self.ensure_connection():
|
||||
|
|
|
|||
93
core/library/bulk_paginate.py
Normal file
93
core/library/bulk_paginate.py
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
"""Paginate a bulk media-server fetch while feeding the no-progress watchdog.
|
||||
|
||||
The library scan fetches every track/album by paging a server API. The DB-update
|
||||
watchdog (``core/database_update_health.py``) kills a job that reports no progress
|
||||
for 300s. The old Jellyfin fetch used a single 10 000-item page, so a whole
|
||||
library came back in ONE request that emitted NO progress while it was in flight —
|
||||
on a slow server that single request exceeded 300s and the watchdog declared the
|
||||
job "stuck" even though it was alive, not hung (Discord: DXP4800 NAS, 7148 tracks,
|
||||
"Fetching all tracks in bulk…").
|
||||
|
||||
``paginate_all_items`` pages at a size chosen so progress is emitted on a cadence
|
||||
set by the PAGE SIZE, not the library size — the watchdog is fed every page, so it
|
||||
can never starve mid-fetch regardless of how big the library is. It is pure: all
|
||||
I/O lives in the injected ``fetch_page``, so the pagination + progress + failure-
|
||||
shrink logic is unit-testable without a server.
|
||||
|
||||
This does NOT change WHAT is fetched (same query, same fields, same items) — only
|
||||
how it's paged and that every page reports progress (the old loop skipped progress
|
||||
on the final/only page, which is the entire bug for a sub-page-size library).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, List, Optional
|
||||
|
||||
# Page size for bulk library fetches. Small enough that a single request stays
|
||||
# well under the 300s no-progress watchdog even on a slow NAS, and that progress
|
||||
# is reported every page. NOT a performance knob — a resilience/observability one.
|
||||
DEFAULT_PAGE_SIZE = 1000
|
||||
|
||||
# Floor the failure-shrink can reach before giving up — a server that can't return
|
||||
# even this many items in one request is genuinely struggling.
|
||||
DEFAULT_MIN_PAGE_SIZE = 250
|
||||
|
||||
|
||||
def paginate_all_items(
|
||||
fetch_page: Callable[[int, int], Optional[List[Any]]],
|
||||
*,
|
||||
report_progress: Optional[Callable[[str], None]] = None,
|
||||
label: str = "items",
|
||||
page_size: int = DEFAULT_PAGE_SIZE,
|
||||
min_page_size: int = DEFAULT_MIN_PAGE_SIZE,
|
||||
on_retry_wait: Optional[Callable[[], None]] = None,
|
||||
) -> List[Any]:
|
||||
"""Page through ``fetch_page(start_index, limit)`` until the server is drained.
|
||||
|
||||
``fetch_page`` returns the page's items (a list, possibly empty = end), or
|
||||
``None`` to signal a FAILED request (timeout/error) — on failure the page size
|
||||
is halved down to ``min_page_size`` and retried, then abandoned after two
|
||||
consecutive failures at the floor.
|
||||
|
||||
Progress is reported after EVERY non-empty page (including the final/only one),
|
||||
so a no-progress watchdog is fed on a cadence set by ``page_size`` — never by
|
||||
the total library size. Returns every item gathered.
|
||||
"""
|
||||
items: List[Any] = []
|
||||
start_index = 0
|
||||
limit = page_size
|
||||
consecutive_failures = 0
|
||||
|
||||
while True:
|
||||
batch = fetch_page(start_index, limit)
|
||||
|
||||
if batch is None: # failed request
|
||||
consecutive_failures += 1
|
||||
if on_retry_wait is not None:
|
||||
on_retry_wait()
|
||||
if limit > min_page_size:
|
||||
limit = max(min_page_size, limit // 2)
|
||||
consecutive_failures = 0 # give the smaller batch a fair chance
|
||||
continue
|
||||
if consecutive_failures >= 2:
|
||||
break # struggling at the floor — stop with what we have
|
||||
continue
|
||||
|
||||
consecutive_failures = 0
|
||||
if not batch:
|
||||
break # drained
|
||||
|
||||
items.extend(batch)
|
||||
# Feed the watchdog on EVERY page — this is the line the old loop only ran
|
||||
# when there was a *next* page, so a sub-page-size library reported nothing.
|
||||
if report_progress is not None:
|
||||
report_progress(f"Fetched {len(items)} {label} so far...")
|
||||
|
||||
if len(batch) < limit:
|
||||
break # last (partial) page
|
||||
start_index += limit
|
||||
|
||||
return items
|
||||
|
||||
|
||||
__all__ = ["paginate_all_items", "DEFAULT_PAGE_SIZE", "DEFAULT_MIN_PAGE_SIZE"]
|
||||
|
|
@ -10,6 +10,7 @@ from __future__ import annotations
|
|||
from dataclasses import dataclass
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
|
@ -121,6 +122,20 @@ def import_existing_track_for_album_slot(album_id: str, payload: dict, deps: Mis
|
|||
if album_data.get("server_source") and source_track.get("server_source") and album_data["server_source"] != source_track["server_source"]:
|
||||
raise MissingTrackImportError("Selected track belongs to a different library source", 400)
|
||||
|
||||
# #917: "I have this" rebuilds the destination path from album metadata. When the album row
|
||||
# has no year, the rebuilt path drops the $year and the copied file lands in a NEW, yearless
|
||||
# directory instead of the album's existing folder. Recover the year from a sibling track so
|
||||
# the import reuses the same directory.
|
||||
if not album_data.get("year"):
|
||||
recovered_year = _existing_album_year_from_sibling(
|
||||
database, album_id, deps.resolve_library_file_path_fn,
|
||||
int(expected.get("disc_number") or 1), int(expected.get("track_number") or 1),
|
||||
)
|
||||
if recovered_year:
|
||||
album_data["year"] = recovered_year
|
||||
logger.info("[I Have This] recovered album year %s from existing folder for album %s",
|
||||
recovered_year, album_id)
|
||||
|
||||
source_path = deps.resolve_library_file_path_fn(source_track.get("file_path"))
|
||||
if not source_path:
|
||||
raise MissingTrackImportError(_file_not_found_message(source_track.get("file_path")), 404)
|
||||
|
|
@ -426,6 +441,50 @@ def _sync_imported_track(deps: MissingTrackImportDeps, track_id, expected_title:
|
|||
logger.debug("Existing-track import server sync skipped/failed: %s", sync_err)
|
||||
|
||||
|
||||
def _existing_album_year_from_sibling(
|
||||
database,
|
||||
album_id: str,
|
||||
resolve_library_file_path_fn: Callable[[Optional[str]], Optional[str]],
|
||||
target_disc: int,
|
||||
target_track: int,
|
||||
) -> Optional[str]:
|
||||
"""Find the release year already baked into this album's on-disk folder (#917).
|
||||
|
||||
Read from a sibling track — its own ``year`` column first, else a ``(YYYY)`` /
|
||||
``[YYYY]`` in the album folder name — so an "I have this" import reuses the album's
|
||||
existing directory instead of rebuilding a yearless one. Returns the 4-digit year
|
||||
string, or None when no signal exists.
|
||||
"""
|
||||
try:
|
||||
with database._get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT file_path, year FROM tracks
|
||||
WHERE album_id = ?
|
||||
AND file_path IS NOT NULL AND file_path != ''
|
||||
AND NOT (COALESCE(disc_number, 1) = ? AND track_number = ?)
|
||||
ORDER BY COALESCE(disc_number, 1), track_number
|
||||
LIMIT 12
|
||||
""",
|
||||
(album_id, target_disc, target_track),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
for row in rows:
|
||||
year = row["year"]
|
||||
if year is not None and str(year).strip()[:4].isdigit():
|
||||
return str(year).strip()[:4]
|
||||
resolved = resolve_library_file_path_fn(row["file_path"])
|
||||
if resolved:
|
||||
folder = os.path.basename(os.path.dirname(resolved))
|
||||
match = re.search(r"[(\[](\d{4})[)\]]", folder) # "Album (2019)" / "Album [2019]"
|
||||
if match:
|
||||
return match.group(1)
|
||||
except Exception as exc:
|
||||
logger.debug("Could not recover album year from sibling for %s: %s", album_id, exc)
|
||||
return None
|
||||
|
||||
|
||||
def copy_album_identity_from_target_sibling(
|
||||
database,
|
||||
album_id: str,
|
||||
|
|
|
|||
|
|
@ -138,10 +138,25 @@ def _collect_base_dirs(
|
|||
except Exception as e:
|
||||
logger.debug("music paths read failed: %s", e)
|
||||
|
||||
# Normalize to absolute forms so resolution does NOT depend on the calling
|
||||
# thread's CWD. A relative config like "./Transfer" otherwise only resolves
|
||||
# when os.path.isdir("./Transfer") happens to be true from the current CWD —
|
||||
# which fails in background workers whose CWD isn't the app root, leaving
|
||||
# base_dirs empty and every track "unresolved". For each candidate we try
|
||||
# the raw form first (cheap, preserves an already-absolute path), then its
|
||||
# os.path.abspath() form so "./Transfer" → "/app/Transfer".
|
||||
expanded: list[str] = []
|
||||
for c in candidates:
|
||||
if not c:
|
||||
continue
|
||||
expanded.append(c)
|
||||
if not os.path.isabs(c):
|
||||
expanded.append(os.path.abspath(c))
|
||||
|
||||
# De-duplicate while preserving order, drop empties / non-existent dirs.
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for c in candidates:
|
||||
for c in expanded:
|
||||
if not c or c in seen:
|
||||
continue
|
||||
seen.add(c)
|
||||
|
|
@ -224,10 +239,23 @@ def resolve_library_file_path_with_diagnostic(
|
|||
if not base_dirs:
|
||||
return None, attempt
|
||||
|
||||
# Skip index 0 to avoid drive-letter / leading-slash artifacts
|
||||
# (e.g. "E:" or "" from a leading "/").
|
||||
# Try progressively shorter path suffixes against each base dir.
|
||||
#
|
||||
# Start at index 0 so a clean RELATIVE library path is tried in FULL first.
|
||||
# SoulSync's own library scanner stores paths like
|
||||
# "Asketa/Another Side/01 - Track.flac" (no leading slash) — index 0 is the
|
||||
# artist folder and dropping it (the old range(1, ...)) meant the artist
|
||||
# segment was never joined, so nothing under transfer/ ever resolved and
|
||||
# every track looked unreadable to the quality scanner.
|
||||
#
|
||||
# For ABSOLUTE media-server paths ("/music/Artist/Album/track.flac") index 0
|
||||
# is the empty leading segment and i=0 yields os.path.join(base, "", ...) ==
|
||||
# base/Artist/... which simply won't exist and harmlessly falls through to
|
||||
# i=1 ("music/...") etc. A Windows drive part ("E:") at i=0 likewise just
|
||||
# fails on POSIX and falls through. So starting at 0 is safe for every form
|
||||
# and only ADDS the relative-full-path match that was missing.
|
||||
for base in base_dirs:
|
||||
for i in range(1, len(path_parts)):
|
||||
for i in range(0, len(path_parts)):
|
||||
candidate = os.path.join(base, *path_parts[i:])
|
||||
if os.path.exists(candidate):
|
||||
return candidate, attempt
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from core.runtime_state import (
|
|||
download_tasks,
|
||||
tasks_lock,
|
||||
)
|
||||
from core.metadata.album_tracks import get_album_for_source
|
||||
from core.metadata.registry import (
|
||||
get_deezer_client,
|
||||
get_itunes_client,
|
||||
|
|
@ -26,6 +27,27 @@ from database.music_database import get_database
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _album_data_from_source(full: dict, album_id: str, fallback_name: str) -> dict:
|
||||
"""Build the redownload `album_data` from a primary-source get_album result (#915).
|
||||
|
||||
Mirrors the Spotify branch's album_data shape so iTunes/Deezer redownloads carry the
|
||||
real release_date / album_type / total_tracks — instead of a lean {'name': ...} that
|
||||
drops the $year and forces a manual reorganize afterwards."""
|
||||
images = full.get('images') or []
|
||||
image_url = full.get('image_url') or ''
|
||||
if not image_url and images and isinstance(images[0], dict):
|
||||
image_url = images[0].get('url', '')
|
||||
return {
|
||||
'id': str(full.get('id') or album_id),
|
||||
'name': full.get('name') or fallback_name,
|
||||
'release_date': full.get('release_date', ''),
|
||||
'album_type': full.get('album_type', 'album'),
|
||||
'total_tracks': full.get('total_tracks', 0),
|
||||
'images': images,
|
||||
'image_url': image_url,
|
||||
}
|
||||
|
||||
|
||||
def _get_itunes_client():
|
||||
"""Mirror of web_server._get_itunes_client — delegates to registry."""
|
||||
return get_itunes_client()
|
||||
|
|
@ -141,12 +163,26 @@ def redownload_start(track_id):
|
|||
'images': album_images,
|
||||
'image_url': album_images[0]['url'] if album_images else '',
|
||||
}
|
||||
elif meta_source == 'itunes':
|
||||
track_number = full_track_details.get('trackNumber')
|
||||
disc_number = full_track_details.get('discNumber', 1)
|
||||
elif meta_source == 'deezer':
|
||||
track_number = full_track_details.get('track_position')
|
||||
disc_number = full_track_details.get('disk_number', 1)
|
||||
elif meta_source in ('itunes', 'deezer'):
|
||||
# #915: parity with the Spotify branch + Reorganize — pull the full album from
|
||||
# the primary source so album_data carries release_date/album_type/total_tracks
|
||||
# (was lean {'name': ...}, which dropped the $year on iTunes/Deezer redownloads).
|
||||
if meta_source == 'itunes':
|
||||
track_number = full_track_details.get('trackNumber')
|
||||
disc_number = full_track_details.get('discNumber', 1)
|
||||
_alb_id = full_track_details.get('collectionId')
|
||||
else:
|
||||
track_number = full_track_details.get('track_position')
|
||||
disc_number = full_track_details.get('disk_number', 1)
|
||||
_alb_id = (full_track_details.get('album') or {}).get('id')
|
||||
if _alb_id:
|
||||
try:
|
||||
_full_album = get_album_for_source(meta_source, str(_alb_id))
|
||||
except Exception as _alb_err: # noqa: BLE001 — never let metadata break redownload
|
||||
logger.debug("[Redownload] %s album fetch failed: %s", meta_source, _alb_err)
|
||||
_full_album = None
|
||||
if isinstance(_full_album, dict):
|
||||
album_data = _album_data_from_source(_full_album, str(_alb_id), metadata.get('album', ''))
|
||||
|
||||
track_data = {
|
||||
'id': meta_id,
|
||||
|
|
|
|||
53
core/library/residual_files.py
Normal file
53
core/library/residual_files.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
"""What counts as a *residual* file — a leftover with no value once the audio it
|
||||
accompanied is gone: OS junk, cover/scan images, and lyric/metadata sidecars.
|
||||
|
||||
Single source of truth shared by:
|
||||
* the **Reorganize** cleanup, which strips these from a source dir after every
|
||||
track has moved out (so the empty-dir pruner can take the folder), and
|
||||
* the **Empty Folder Cleaner** job, which can optionally treat a folder holding
|
||||
ONLY residual files as removable (#891).
|
||||
|
||||
Defining "disposable" in one place keeps the two features agreeing on what a "dead
|
||||
folder" is. Pure predicates — no filesystem access — so they're unit-tested in
|
||||
isolation. The whitelist is deliberately conservative: anything NOT recognized here
|
||||
(a booklet ``.pdf``, a video, a ``.txt`` note) is treated as real content and kept.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
# OS / tooling junk.
|
||||
JUNK_FILES = {'.ds_store', 'thumbs.db', 'desktop.ini', '.directory', 'album.nfo~'}
|
||||
# Cover art + booklet scans.
|
||||
IMAGE_EXTS = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.tiff', '.tif'}
|
||||
# Lyric / metadata / playlist sidecars that are worthless without their audio.
|
||||
SIDECAR_EXTS = {'.lrc', '.nfo', '.cue', '.m3u', '.m3u8'}
|
||||
|
||||
|
||||
def _ext(name: str) -> str:
|
||||
return os.path.splitext(name or '')[1].lower()
|
||||
|
||||
|
||||
def is_junk(name: str) -> bool:
|
||||
return (name or '').lower() in JUNK_FILES
|
||||
|
||||
|
||||
def is_image(name: str) -> bool:
|
||||
return _ext(name) in IMAGE_EXTS
|
||||
|
||||
|
||||
def is_sidecar(name: str) -> bool:
|
||||
return _ext(name) in SIDECAR_EXTS
|
||||
|
||||
|
||||
def is_disposable(name: str) -> bool:
|
||||
"""True if this file is junk, a cover/scan image, or a lyric/metadata sidecar —
|
||||
i.e. safe to delete from a folder that has no audio left."""
|
||||
return is_junk(name) or is_image(name) or is_sidecar(name)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'JUNK_FILES', 'IMAGE_EXTS', 'SIDECAR_EXTS',
|
||||
'is_junk', 'is_image', 'is_sidecar', 'is_disposable',
|
||||
]
|
||||
104
core/library/standalone_scan.py
Normal file
104
core/library/standalone_scan.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
"""Decision logic for the SoulSync standalone Deep Scan's untracked → Staging move.
|
||||
|
||||
The standalone deep scan (``_run_soulsync_deep_scan`` in web_server) walks the
|
||||
Transfer folder, diffs it against the ``soulsync`` rows in the DB, and relocates
|
||||
every file it can't find a DB record for into Staging for auto-import. That's fine
|
||||
when Transfer is a scratch/landing area — files arrive, get moved, imported, and
|
||||
recorded, so a later scan only ever sees a few genuinely-new arrivals.
|
||||
|
||||
It is a DATA-LOSS trap when the DB is empty or out of sync with disk (a volume
|
||||
swap, a DB reset, external tag edits) while Transfer holds the user's real library:
|
||||
a path-only diff then flags the *entire* library as "untracked" and the scan
|
||||
relocates all of it (issue #904). The same failure mode the orphan detector and the
|
||||
media-server deep scan already guard against (``core.library.stale_guard``) — this
|
||||
path just never used the guard.
|
||||
|
||||
This module is the pure, testable decision: given the Transfer file set, the DB's
|
||||
known paths, and the user's "Transfer is my permanent library" preference, decide
|
||||
WHICH files are untracked and WHETHER it's safe to relocate them. The web layer does
|
||||
only the I/O (walk/move/delete) based on the returned plan.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable, Set
|
||||
|
||||
from core.library.stale_guard import (
|
||||
DEFAULT_MAX_ORPHAN_FRACTION,
|
||||
DEFAULT_MIN_ORPHANS,
|
||||
is_implausible_orphan_flood,
|
||||
)
|
||||
|
||||
# Block reason codes (web layer turns these into a user-facing warning).
|
||||
BLOCK_NONE = ""
|
||||
BLOCK_TRANSFER_PERMANENT = "transfer_permanent"
|
||||
BLOCK_DESYNC = "desync"
|
||||
|
||||
|
||||
def _norm(path: str) -> str:
|
||||
"""Normalize a path for cross-platform comparison (Windows vs Unix separators)."""
|
||||
return str(path).replace("\\", "/")
|
||||
|
||||
|
||||
def diff_untracked(transfer_files: Iterable[str], db_paths: Iterable[str]) -> Set[str]:
|
||||
"""Files present in ``transfer_files`` but with no matching ``db_paths`` record.
|
||||
|
||||
Comparison is separator-normalized, so a DB path stored with one separator style
|
||||
still matches the on-disk path. Pure — no I/O. Returns the original (un-normalized)
|
||||
transfer paths so the caller can act on the real filesystem entries.
|
||||
"""
|
||||
db_norm = {_norm(p) for p in db_paths if p}
|
||||
return {f for f in transfer_files if _norm(f) not in db_norm}
|
||||
|
||||
|
||||
def plan_standalone_deep_scan(
|
||||
transfer_files: Iterable[str],
|
||||
db_paths: Iterable[str],
|
||||
*,
|
||||
never_move: bool = False,
|
||||
min_untracked: int = DEFAULT_MIN_ORPHANS,
|
||||
max_fraction: float = DEFAULT_MAX_ORPHAN_FRACTION,
|
||||
) -> dict:
|
||||
"""Plan the untracked → Staging move for a standalone deep scan. Pure — no I/O.
|
||||
|
||||
Returns a dict:
|
||||
* ``untracked`` (set[str]) — Transfer files with no DB record.
|
||||
* ``move_blocked`` (bool) — True when the untracked files must NOT be relocated.
|
||||
* ``block_reason`` (str) — ``BLOCK_TRANSFER_PERMANENT`` / ``BLOCK_DESYNC`` / "".
|
||||
|
||||
The move is blocked when either:
|
||||
* ``never_move`` is set (the user marked Transfer as their permanent library), or
|
||||
* the untracked share is implausibly large (> ``min_untracked`` files AND
|
||||
> ``max_fraction`` of the folder) — the empty/desynced-DB signature, where a
|
||||
path-only diff would relocate the whole library. Below that floor a normal
|
||||
batch of new arrivals still moves as before.
|
||||
|
||||
``move_blocked`` is only ever True when there ARE untracked files; an empty scan
|
||||
or a clean library returns ``move_blocked=False`` with no reason.
|
||||
"""
|
||||
transfer_set = set(transfer_files) # concrete (handles generators) + dedups
|
||||
untracked = diff_untracked(transfer_set, db_paths)
|
||||
total = len(transfer_set)
|
||||
n_untracked = len(untracked)
|
||||
|
||||
if n_untracked == 0:
|
||||
return {"untracked": untracked, "move_blocked": False, "block_reason": BLOCK_NONE}
|
||||
|
||||
if never_move:
|
||||
return {"untracked": untracked, "move_blocked": True, "block_reason": BLOCK_TRANSFER_PERMANENT}
|
||||
|
||||
if is_implausible_orphan_flood(
|
||||
n_untracked, total, min_orphans=min_untracked, max_fraction=max_fraction
|
||||
):
|
||||
return {"untracked": untracked, "move_blocked": True, "block_reason": BLOCK_DESYNC}
|
||||
|
||||
return {"untracked": untracked, "move_blocked": False, "block_reason": BLOCK_NONE}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"diff_untracked",
|
||||
"plan_standalone_deep_scan",
|
||||
"BLOCK_NONE",
|
||||
"BLOCK_TRANSFER_PERMANENT",
|
||||
"BLOCK_DESYNC",
|
||||
]
|
||||
|
|
@ -26,14 +26,16 @@ without a source ID are reported back to the caller and skipped
|
|||
entirely.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, List, Optional, Set
|
||||
from typing import Any, Callable, Dict, List, Optional, Set, Tuple
|
||||
|
||||
# Per-album track concurrency. Matches the download workers' per-batch
|
||||
# concurrency (3) so reorganize feels comparable to a fresh download.
|
||||
|
|
@ -99,6 +101,7 @@ _ALBUM_ID_COLUMNS = {
|
|||
'deezer': 'deezer_id',
|
||||
'discogs': 'discogs_id',
|
||||
'hydrabase': 'soul_id',
|
||||
'musicbrainz': 'musicbrainz_release_id',
|
||||
}
|
||||
|
||||
# Human-facing label for each source.
|
||||
|
|
@ -407,6 +410,18 @@ def _differentiators_in(norm_title: str) -> frozenset:
|
|||
return frozenset(t for t in norm_title.split() if t in _VERSION_DIFFERENTIATORS)
|
||||
|
||||
|
||||
# Featured-artist credit: "(feat. X)" / "[ft X]" / a trailing "feat. X". The
|
||||
# parenthesised form is stripped wherever it appears; the bare form only when
|
||||
# something follows it (so a song literally named "The Feat" is left alone, and
|
||||
# "Defeat"/"Lift" never trip the word-boundary). Case-insensitive.
|
||||
_FEAT_RE = re.compile(
|
||||
r"""\s*[\(\[]\s*(?:feat|ft|featuring)\b\.?[^)\]]*[\)\]] # (feat. X) / [ft. X]
|
||||
| \s+(?:feat|ft|featuring)\b\.?\s+\S.*$ # trailing feat. X ...
|
||||
""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_title(value) -> str:
|
||||
"""Lowercase + strip cosmetic punctuation and treat brackets / dashes
|
||||
/ slashes as word separators so the same track named slightly
|
||||
|
|
@ -418,10 +433,17 @@ def _normalize_title(value) -> str:
|
|||
- ``Don't Stop Believin'`` ↔ ``Don’t Stop Believin’``
|
||||
- ``Swimming Pools (Drank) - Extended Version``
|
||||
↔ ``Swimming Pools (Drank) (Extended Version)``
|
||||
- ``The Chase (feat. Big Artist)`` ↔ ``The Chase`` (#914)
|
||||
"""
|
||||
if value is None:
|
||||
return ''
|
||||
out = str(value).strip().lower()
|
||||
out = str(value).strip()
|
||||
# #914: drop featured-artist credits FIRST (while the parens are still here to
|
||||
# bound the group). iTunes appends "(feat. X)" to track titles while a user's
|
||||
# file is often just "The Chase" — the credit is metadata, not the song's
|
||||
# identity, and leaving it in dropped the match ratio below the threshold so
|
||||
# correctly-identified tracks reported as "not in the tracklist".
|
||||
out = _FEAT_RE.sub('', out).lower()
|
||||
# Strip characters that don't carry meaning across providers.
|
||||
for ch in ('"', "'", '‘', '’', '“', '”', '.', ',', '!', '?',
|
||||
'(', ')', '[', ']', '{', '}'):
|
||||
|
|
@ -1080,6 +1102,12 @@ def preview_album_reorganize(
|
|||
'track_number': track.get('track_number', 0),
|
||||
'current_path': _trim_to_transfer(db_path, resolved, transfer_dir),
|
||||
'new_path': '',
|
||||
# Absolute on-disk paths (additive). `current_path`/`new_path` above are
|
||||
# display-trimmed; these carry the real paths so the rename-only executor
|
||||
# acts on EXACTLY what the preview computed — no separate path logic that
|
||||
# could drift from what the user saw (#875).
|
||||
'current_path_abs': resolved or '',
|
||||
'new_path_abs': '',
|
||||
'file_exists': resolved is not None,
|
||||
'unchanged': False,
|
||||
'collision': False,
|
||||
|
|
@ -1127,6 +1155,7 @@ def preview_album_reorganize(
|
|||
new_full, _ok = build_final_path_fn(
|
||||
context, spotify_artist, album_info, file_ext, create_dirs=False
|
||||
)
|
||||
item['new_path_abs'] = new_full or ''
|
||||
item['new_path'] = (
|
||||
os.path.relpath(new_full, transfer_dir)
|
||||
if transfer_dir and new_full and new_full.startswith(transfer_dir)
|
||||
|
|
@ -1786,6 +1815,150 @@ def reorganize_album(
|
|||
return summary
|
||||
|
||||
|
||||
def _rename_track_in_place(current_abs: str, new_abs: str) -> Tuple[bool, Optional[str]]:
|
||||
"""Move ONE file from ``current_abs`` to ``new_abs`` in place — no copy, no re-tag,
|
||||
no post-processing. Creates the destination folder, carries sibling-format files
|
||||
(e.g. a lossy ``.opus`` alongside the ``.flac``) along with the renamed stem, and
|
||||
falls back to a cross-device move when the rename crosses a filesystem boundary.
|
||||
|
||||
Refuses to overwrite a DIFFERENT existing file at the destination (returns an error
|
||||
instead) — never silent data loss. Returns ``(ok, error_message)``.
|
||||
"""
|
||||
try:
|
||||
if current_abs and not os.path.exists(current_abs):
|
||||
return False, 'source file no longer on disk'
|
||||
same = os.path.normpath(current_abs) == os.path.normpath(new_abs)
|
||||
if os.path.exists(new_abs) and not same:
|
||||
return False, 'destination already exists'
|
||||
os.makedirs(os.path.dirname(new_abs), exist_ok=True)
|
||||
# Carry sibling-format audio to the same destination with the renamed stem —
|
||||
# mirrors _finalize_track so lossy-copy pairs don't get orphaned.
|
||||
for sibling_src in _find_sibling_audio_files(current_abs):
|
||||
_move_sibling_to_destination(sibling_src, new_abs)
|
||||
try:
|
||||
os.rename(current_abs, new_abs)
|
||||
except OSError as e:
|
||||
if getattr(e, 'errno', None) == errno.EXDEV:
|
||||
shutil.move(current_abs, new_abs) # crosses a filesystem boundary
|
||||
else:
|
||||
raise
|
||||
return True, None
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def reorganize_album_rename_only(
|
||||
*,
|
||||
album_id: str,
|
||||
db,
|
||||
transfer_dir: str,
|
||||
resolve_file_path_fn: Callable[[Optional[str]], Optional[str]],
|
||||
build_final_path_fn: Callable,
|
||||
update_track_path_fn: Optional[Callable[[object, str], None]] = None,
|
||||
cleanup_empty_dir_fn: Optional[Callable[[str], None]] = None,
|
||||
on_progress: Optional[Callable[[dict], None]] = None,
|
||||
primary_source: Optional[str] = None,
|
||||
strict_source: bool = False,
|
||||
metadata_source: str = 'api',
|
||||
stop_check: Optional[Callable[[], bool]] = None,
|
||||
preview_fn: Optional[Callable] = None,
|
||||
) -> dict:
|
||||
"""RENAME-ONLY reorganize (#875): move each track's file to the path the current
|
||||
naming scheme dictates, and nothing else — no copy-to-staging, no re-tag, no
|
||||
quality/AcoustID checks.
|
||||
|
||||
It acts on EXACTLY what :func:`preview_album_reorganize` computed (injected via
|
||||
``preview_fn`` for testability), so the apply can never disagree with what the user
|
||||
saw, and ONLY files whose path actually changes are touched — files marked
|
||||
``unchanged`` are skipped, which is what keeps a rename from rewriting the whole
|
||||
album (the #875 complaint). Tags and audio are left byte-for-byte alone.
|
||||
|
||||
Returns the same summary shape as :func:`reorganize_album`.
|
||||
"""
|
||||
preview_fn = preview_fn or preview_album_reorganize
|
||||
summary = {
|
||||
'status': 'completed', 'source': None, 'total': 0,
|
||||
'moved': 0, 'skipped': 0, 'failed': 0, 'errors': [],
|
||||
}
|
||||
|
||||
def _emit(**updates):
|
||||
if on_progress is None:
|
||||
return
|
||||
try:
|
||||
on_progress(updates)
|
||||
except Exception as e:
|
||||
logger.debug("[Reorganize/rename] progress emit failed: %s", e)
|
||||
|
||||
preview = preview_fn(
|
||||
album_id=album_id, db=db, transfer_dir=transfer_dir,
|
||||
resolve_file_path_fn=resolve_file_path_fn,
|
||||
build_final_path_fn=build_final_path_fn,
|
||||
primary_source=primary_source, strict_source=strict_source,
|
||||
metadata_source=metadata_source,
|
||||
)
|
||||
summary['source'] = preview.get('source')
|
||||
if not preview.get('success'):
|
||||
summary['status'] = preview.get('status', 'error')
|
||||
return summary
|
||||
|
||||
tracks = preview.get('tracks', [])
|
||||
summary['total'] = len(tracks)
|
||||
src_dirs_touched: Set[str] = set()
|
||||
|
||||
for t in tracks:
|
||||
if stop_check and stop_check():
|
||||
break
|
||||
title = t.get('title', 'Unknown')
|
||||
_emit(current_track=title)
|
||||
|
||||
# Skip anything that isn't a real, changing move. `unchanged` is the key one —
|
||||
# it's why a rename no longer rewrites files whose name didn't change.
|
||||
if (not t.get('matched') or t.get('unchanged')
|
||||
or t.get('collision') or not t.get('new_path_abs')):
|
||||
summary['skipped'] += 1
|
||||
_emit(skipped=summary['skipped'])
|
||||
continue
|
||||
|
||||
current_abs = t.get('current_path_abs')
|
||||
new_abs = t.get('new_path_abs')
|
||||
ok, err = _rename_track_in_place(current_abs, new_abs)
|
||||
if not ok:
|
||||
summary['failed'] += 1
|
||||
summary['errors'].append({
|
||||
'track_id': t.get('track_id'), 'title': title,
|
||||
'error': err or 'rename failed',
|
||||
})
|
||||
_emit(failed=summary['failed'], errors=list(summary['errors']))
|
||||
continue
|
||||
|
||||
# File is at its new home — update the DB directly (authoritative; no need to
|
||||
# round-trip through a server scan to learn what we just did). On DB failure the
|
||||
# file still moved; a library scan reconciles it, so we don't fail the track.
|
||||
if update_track_path_fn:
|
||||
try:
|
||||
update_track_path_fn(t.get('track_id'), new_abs)
|
||||
except Exception as db_err:
|
||||
logger.warning(
|
||||
"[Reorganize/rename] DB path update failed for %s: %s "
|
||||
"(file moved to %s; a scan will reconcile)",
|
||||
t.get('track_id'), db_err, new_abs,
|
||||
)
|
||||
if current_abs:
|
||||
src_dirs_touched.add(os.path.dirname(current_abs))
|
||||
summary['moved'] += 1
|
||||
_emit(moved=summary['moved'],
|
||||
processed=summary['moved'] + summary['skipped'] + summary['failed'])
|
||||
|
||||
if cleanup_empty_dir_fn:
|
||||
for src_dir in src_dirs_touched:
|
||||
try:
|
||||
cleanup_empty_dir_fn(src_dir)
|
||||
except Exception as e:
|
||||
logger.debug("[Reorganize/rename] cleanup of %s failed: %s", src_dir, e)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def _find_artist_dir(dest_path: str, transfer_dir: str) -> Optional[str]:
|
||||
"""Walk up from ``dest_path`` until the parent equals ``transfer_dir``;
|
||||
the directory at that point is the artist folder. Returns None if
|
||||
|
|
@ -1847,14 +2020,8 @@ def _prune_empty_album_dirs(artist_dir: str) -> None:
|
|||
# Sidecars that live alongside ONE audio file (same filename stem).
|
||||
_TRACK_SIDECAR_EXTS = ('.lrc', '.nfo', '.txt', '.cue', '.json')
|
||||
|
||||
# Sidecars that live at the ALBUM level (one per directory).
|
||||
_ALBUM_SIDECARS = (
|
||||
'cover.jpg', 'cover.jpeg', 'cover.png',
|
||||
'folder.jpg', 'folder.png',
|
||||
'front.jpg', 'front.png',
|
||||
'album.jpg', 'album.png',
|
||||
'artwork.jpg', 'artwork.png',
|
||||
)
|
||||
# Album-level leftovers (cover images, .lrc, etc.) are classified by the shared
|
||||
# `core.library.residual_files.is_disposable` predicate — see `_delete_album_sidecars`.
|
||||
|
||||
# Audio extensions used to decide whether a source directory still has
|
||||
# tracks the user might care about (i.e. a per-track failure left audio
|
||||
|
|
@ -1954,16 +2121,30 @@ def _delete_track_sidecars(audio_path: str) -> None:
|
|||
|
||||
|
||||
def _delete_album_sidecars(src_dir: str) -> None:
|
||||
"""Delete album-level sidecars (cover.jpg, folder.jpg, etc.) from
|
||||
`src_dir`. Used during end-of-run cleanup when no audio files remain
|
||||
in the directory. Best-effort — individual failures are debug-logged."""
|
||||
for name in _ALBUM_SIDECARS:
|
||||
sidecar = os.path.join(src_dir, name)
|
||||
if os.path.isfile(sidecar):
|
||||
"""Delete album-level *residual* files from ``src_dir`` — any cover/scan image,
|
||||
lyric/metadata sidecar (.lrc/.nfo/.cue/.m3u), or OS junk. Called during
|
||||
end-of-run cleanup ONLY when no audio remains in the directory, so everything
|
||||
here is leftover from the album that just moved (#891 — previously this only
|
||||
removed a fixed list of cover names, so ``back.jpg`` / ``disc.jpg`` / ``.webp``
|
||||
survived and kept the folder un-prunable).
|
||||
|
||||
Uses the shared ``is_disposable`` predicate so it agrees with the Empty Folder
|
||||
Cleaner on what's a dead leftover; anything unrecognized (a booklet ``.pdf``, a
|
||||
video) is deliberately LEFT. Best-effort — individual failures are debug-logged."""
|
||||
from core.library.residual_files import is_disposable
|
||||
try:
|
||||
entries = os.listdir(src_dir)
|
||||
except OSError:
|
||||
return
|
||||
for name in entries:
|
||||
if not is_disposable(name):
|
||||
continue
|
||||
full = os.path.join(src_dir, name)
|
||||
if os.path.isfile(full):
|
||||
try:
|
||||
os.remove(sidecar)
|
||||
os.remove(full)
|
||||
except OSError as e:
|
||||
logger.debug(f"[Reorganize] Couldn't remove album sidecar {sidecar}: {e}")
|
||||
logger.debug(f"[Reorganize] Couldn't remove residual file {full}: {e}")
|
||||
|
||||
|
||||
def _has_remaining_audio(directory: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -47,7 +47,10 @@ class LidarrDownloadClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
|
||||
self.active_downloads: Dict[str, Dict[str, Any]] = {}
|
||||
self._download_lock = threading.Lock()
|
||||
|
|
|
|||
|
|
@ -158,6 +158,161 @@ class ListenBrainzClient:
|
|||
|
||||
return True
|
||||
|
||||
_MAX_TRACKS_PER_ADD = 100 # ListenBrainz MAX_RECORDINGS_PER_ADD
|
||||
_PLAYLIST_EXT = "https://musicbrainz.org/doc/jspf#playlist"
|
||||
|
||||
def _lb_headers(self) -> Dict:
|
||||
return {"Authorization": f"Token {self.token}", "Content-Type": "application/json"}
|
||||
|
||||
def _add_tracks_in_batches(self, playlist_mbid: str, tracks: List[Dict]) -> int:
|
||||
"""Add JSPF tracks to a playlist in <=100-track batches; return how many were added."""
|
||||
added = 0
|
||||
headers = self._lb_headers()
|
||||
for i in range(0, len(tracks or []), self._MAX_TRACKS_PER_ADD):
|
||||
batch = tracks[i:i + self._MAX_TRACKS_PER_ADD]
|
||||
try:
|
||||
r = self._make_request_with_retry(
|
||||
"POST", f"{self.base_url}/playlist/{playlist_mbid}/item/add",
|
||||
json={"playlist": {"track": batch}}, headers=headers,
|
||||
)
|
||||
if r and r.status_code in (200, 201):
|
||||
added += len(batch)
|
||||
else:
|
||||
logger.warning(f"ListenBrainz item/add batch failed: "
|
||||
f"{r.status_code if r else 'no response'}")
|
||||
except Exception as e:
|
||||
logger.error(f"ListenBrainz item/add error: {e}")
|
||||
return added
|
||||
|
||||
def get_playlist_track_count(self, playlist_mbid: str):
|
||||
"""Current track count of an LB playlist, or None if it can't be fetched (gone/404)."""
|
||||
try:
|
||||
r = self._make_request_with_retry(
|
||||
"GET", f"{self.base_url}/playlist/{playlist_mbid}",
|
||||
params={"fetch_metadata": "false"},
|
||||
headers={"Authorization": f"Token {self.token}"},
|
||||
)
|
||||
if r and r.status_code == 200:
|
||||
return len(((r.json() or {}).get("playlist") or {}).get("track", []))
|
||||
except Exception as e:
|
||||
logger.debug(f"ListenBrainz get playlist count failed: {e}")
|
||||
return None
|
||||
|
||||
def delete_playlist(self, playlist_mbid: str) -> bool:
|
||||
"""Delete an LB playlist. True on success."""
|
||||
try:
|
||||
r = self._make_request_with_retry(
|
||||
"POST", f"{self.base_url}/playlist/{playlist_mbid}/delete", headers=self._lb_headers()
|
||||
)
|
||||
return bool(r and r.status_code in (200, 201))
|
||||
except Exception as e:
|
||||
logger.error(f"ListenBrainz delete playlist error: {e}")
|
||||
return False
|
||||
|
||||
def create_playlist(self, title: str, tracks: List[Dict], public: bool = False) -> Dict:
|
||||
"""Create a NEW playlist on ListenBrainz and add its tracks (#903).
|
||||
|
||||
``tracks`` are JSPF track dicts — each MUST carry an ``identifier`` of the form
|
||||
``https://musicbrainz.org/recording/<mbid>`` (LB rejects text-only tracks). Creates
|
||||
an empty playlist for the MBID, then adds tracks in <=100 batches. Returns
|
||||
``{success, playlist_mbid, playlist_url, added, requested, error, updated}``. Never raises.
|
||||
"""
|
||||
result = {"success": False, "playlist_mbid": None, "playlist_url": None,
|
||||
"added": 0, "requested": len(tracks or []), "error": None, "updated": False}
|
||||
if not self.is_authenticated():
|
||||
result["error"] = "ListenBrainz not authenticated (no token/username)"
|
||||
return result
|
||||
|
||||
create_body = {"playlist": {
|
||||
"title": (title or "SoulSync Export").strip() or "SoulSync Export",
|
||||
"extension": {self._PLAYLIST_EXT: {"public": bool(public)}},
|
||||
}}
|
||||
try:
|
||||
resp = self._make_request_with_retry(
|
||||
"POST", f"{self.base_url}/playlist/create", json=create_body, headers=self._lb_headers()
|
||||
)
|
||||
except Exception as e:
|
||||
result["error"] = f"create request failed: {e}"
|
||||
return result
|
||||
if not resp or resp.status_code not in (200, 201):
|
||||
result["error"] = f"create returned {resp.status_code if resp else 'no response'}"
|
||||
return result
|
||||
try:
|
||||
playlist_mbid = (resp.json() or {}).get("playlist_mbid")
|
||||
except Exception:
|
||||
playlist_mbid = None
|
||||
if not playlist_mbid:
|
||||
result["error"] = "create succeeded but no playlist_mbid in response"
|
||||
return result
|
||||
|
||||
result["playlist_mbid"] = playlist_mbid
|
||||
result["playlist_url"] = f"https://listenbrainz.org/playlist/{playlist_mbid}"
|
||||
result["added"] = self._add_tracks_in_batches(playlist_mbid, tracks)
|
||||
result["success"] = True
|
||||
return result
|
||||
|
||||
def update_playlist(self, playlist_mbid: str, title: str, tracks: List[Dict], public: bool = False) -> Dict:
|
||||
"""Replace an existing LB playlist's contents IN PLACE (stable URL/MBID) (#903).
|
||||
|
||||
Verifies the playlist still exists, clears its current items, re-adds the new tracks,
|
||||
and updates the title. If the playlist is gone (deleted on LB), returns success=False
|
||||
with ``gone=True`` so the caller can fall back to creating a fresh one.
|
||||
Returns the same shape as ``create_playlist`` plus ``updated=True``.
|
||||
"""
|
||||
result = {"success": False, "playlist_mbid": playlist_mbid,
|
||||
"playlist_url": f"https://listenbrainz.org/playlist/{playlist_mbid}",
|
||||
"added": 0, "requested": len(tracks or []), "error": None,
|
||||
"updated": True, "gone": False}
|
||||
if not self.is_authenticated():
|
||||
result["error"] = "ListenBrainz not authenticated (no token/username)"
|
||||
return result
|
||||
|
||||
count = self.get_playlist_track_count(playlist_mbid)
|
||||
if count is None:
|
||||
result["error"] = "playlist not found on ListenBrainz"
|
||||
result["gone"] = True
|
||||
return result
|
||||
|
||||
headers = self._lb_headers()
|
||||
# Clear existing items (one range delete from the top).
|
||||
if count > 0:
|
||||
try:
|
||||
self._make_request_with_retry(
|
||||
"POST", f"{self.base_url}/playlist/{playlist_mbid}/item/delete",
|
||||
json={"index": 0, "count": count}, headers=headers,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ListenBrainz item/delete (clear) failed: {e}")
|
||||
|
||||
result["added"] = self._add_tracks_in_batches(playlist_mbid, tracks)
|
||||
|
||||
# Refresh the title (best-effort — content already replaced).
|
||||
try:
|
||||
self._make_request_with_retry(
|
||||
"POST", f"{self.base_url}/playlist/edit/{playlist_mbid}",
|
||||
json={"playlist": {"title": (title or "SoulSync Export").strip() or "SoulSync Export"}},
|
||||
headers=headers,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"ListenBrainz playlist title edit failed: {e}")
|
||||
|
||||
result["success"] = True
|
||||
return result
|
||||
|
||||
def create_or_update_playlist(self, title: str, tracks: List[Dict],
|
||||
existing_mbid: str = None, public: bool = False) -> Dict:
|
||||
"""Update the existing LB playlist in place when we've pushed this one before, else
|
||||
create a fresh one — so re-exporting the same SoulSync playlist never duplicates it.
|
||||
Falls back to create if the remembered playlist was deleted on LB."""
|
||||
if existing_mbid:
|
||||
res = self.update_playlist(existing_mbid, title, tracks, public)
|
||||
if res.get("success"):
|
||||
return res
|
||||
# Remembered playlist gone/failed -> create a new one instead of erroring out.
|
||||
logger.info(f"ListenBrainz playlist {existing_mbid} unavailable for update "
|
||||
f"({res.get('error')}); creating a new one.")
|
||||
return self.create_playlist(title, tracks, public)
|
||||
|
||||
def get_playlists_created_for_user(self, count: int = 25, offset: int = 0) -> List[Dict]:
|
||||
"""
|
||||
Fetch playlists created FOR the user (recommendations, personalized playlists)
|
||||
|
|
|
|||
|
|
@ -291,6 +291,13 @@ class ListeningStatsWorker:
|
|||
if not (top_artists or top_albums or top_tracks):
|
||||
return
|
||||
|
||||
# Normalize image URLs HERE, at cache-build time, not on every /api/stats/cached
|
||||
# read. normalize_image_url registers each URL in the image cache (a SQLite write
|
||||
# under a lock) — doing that per-request made the "instant" stats endpoint take ~20s
|
||||
# on HDD-backed installs (#935). Done once per background rebuild it's off the hot path,
|
||||
# and the read just returns the already-browser-safe URLs.
|
||||
from core.metadata import normalize_image_url as _fix_image
|
||||
|
||||
conn = None
|
||||
try:
|
||||
conn = self.db._get_connection()
|
||||
|
|
@ -324,7 +331,7 @@ class ListeningStatsWorker:
|
|||
key = (artist.get('name') or '').lower()
|
||||
r = artist_rows.get(key)
|
||||
if r:
|
||||
artist['image_url'] = r[1] or None
|
||||
artist['image_url'] = _fix_image(r[1]) or None
|
||||
artist['id'] = r[2]
|
||||
artist['global_listeners'] = r[3]
|
||||
artist['global_playcount'] = r[4]
|
||||
|
|
@ -356,7 +363,7 @@ class ListeningStatsWorker:
|
|||
key = (album.get('name') or '').lower()
|
||||
r = album_rows.get(key)
|
||||
if r:
|
||||
album['image_url'] = r[1] or None
|
||||
album['image_url'] = _fix_image(r[1]) or None
|
||||
album['id'] = r[2]
|
||||
album['artist_id'] = r[3]
|
||||
|
||||
|
|
@ -395,7 +402,7 @@ class ListeningStatsWorker:
|
|||
(track.get('artist') or '').lower())
|
||||
r = track_rows.get(key)
|
||||
if r:
|
||||
track['image_url'] = r[2] or None
|
||||
track['image_url'] = _fix_image(r[2]) or None
|
||||
track['id'] = r[3]
|
||||
track['artist_id'] = r[4]
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -438,10 +438,19 @@ def embed_album_art_metadata(audio_file, metadata: dict):
|
|||
|
||||
if not image_data:
|
||||
art_url = metadata.get("album_art_url")
|
||||
if not art_url:
|
||||
logger.warning("No album art URL available for embedding.")
|
||||
# Prefer the pinned release's OWN cover over a release-group / provider
|
||||
# representative (usually the standard edition); fall back to art_url
|
||||
# when the release has no art of its own. Keeps embedded art in sync
|
||||
# with cover.jpg (same preference + fetch).
|
||||
from core.metadata.caa_art import fetch_release_preferred_art
|
||||
image_data, mime_type, used_url = fetch_release_preferred_art(
|
||||
release_mbid, art_url, fetch_fn=_fetch_art_bytes)
|
||||
if not image_data:
|
||||
if not art_url and not release_mbid:
|
||||
logger.warning("No album art URL available for embedding.")
|
||||
return False
|
||||
image_data, mime_type = _fetch_art_bytes(art_url)
|
||||
if release_mbid and used_url and used_url != art_url:
|
||||
logger.info("Embedding release-specific art (edition match): %s", release_mbid)
|
||||
|
||||
if not image_data:
|
||||
logger.error("Failed to download album art data.")
|
||||
|
|
@ -560,13 +569,20 @@ def download_cover_art(album_info: dict, target_dir: str, context: dict = None,
|
|||
art_url = images[0].get("url", "")
|
||||
if art_url:
|
||||
logger.info("Using cover art URL from album context")
|
||||
if not art_url:
|
||||
logger.warning("No cover art URL available for download.")
|
||||
# Prefer the pinned release's OWN cover over a release-group / provider
|
||||
# representative (which is usually the standard edition), falling back
|
||||
# to art_url when the release has no art of its own. Upgrades to the
|
||||
# source's highest resolution via _fetch_art_bytes (shared with the
|
||||
# tag-embed path so cover.jpg and embedded art match).
|
||||
from core.metadata.caa_art import fetch_release_preferred_art
|
||||
image_data, _, used_url = fetch_release_preferred_art(
|
||||
release_mbid, art_url, fetch_fn=_fetch_art_bytes)
|
||||
if not image_data:
|
||||
if not art_url and not release_mbid:
|
||||
logger.warning("No cover art URL available for download.")
|
||||
return
|
||||
# Upgrade to the source's highest resolution (Spotify master /
|
||||
# iTunes 3000 / Deezer 1900) with a one-level fallback — shared
|
||||
# with the tag-embed path so cover.jpg and embedded art match.
|
||||
image_data, _ = _fetch_art_bytes(art_url)
|
||||
if release_mbid and used_url and used_url != art_url:
|
||||
logger.info("Using release-specific cover.jpg (edition match): %s", release_mbid)
|
||||
|
||||
if not image_data:
|
||||
return
|
||||
|
|
|
|||
74
core/metadata/caa_art.py
Normal file
74
core/metadata/caa_art.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""Cover Art Archive helper: prefer a pinned release's OWN cover over the
|
||||
release-group representative.
|
||||
|
||||
On the Cover Art Archive a release-group ``front`` is a single REPRESENTATIVE
|
||||
cover — CAA designates one release in the group to stand for the whole thing,
|
||||
which is almost always the standard / most-common edition. So when a download
|
||||
has pinned a SPECIFIC release (e.g. a "Gustave Edition" the user picked), using
|
||||
the release-group cover silently swaps in the standard art.
|
||||
|
||||
This helper tries the specific release's own ``/release/<mbid>/front`` first and
|
||||
only falls back to the caller's existing URL (a release-group representative or a
|
||||
provider cover) when the release has no art of its own — so it can only ever
|
||||
*improve* on today's behaviour, never strip a cover that was already showing.
|
||||
|
||||
Pure: the network fetch is injected, so the preference logic is unit-testable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
COVER_ART_ARCHIVE_URL = "https://coverartarchive.org"
|
||||
|
||||
|
||||
def caa_front_url(mbid: Optional[str], scope: str = "release", size: int = 1200) -> Optional[str]:
|
||||
"""Build a Cover Art Archive front-cover URL, or None for a falsy mbid.
|
||||
``scope`` is 'release' (a specific edition) or 'release-group' (the group's
|
||||
representative). ``size`` selects the CDN thumbnail (e.g. 250/500/1200); 0
|
||||
requests the bare ``/front`` original."""
|
||||
if not mbid:
|
||||
return None
|
||||
if scope not in ("release", "release-group"):
|
||||
scope = "release"
|
||||
suffix = f"-{size}" if size else ""
|
||||
return f"{COVER_ART_ARCHIVE_URL}/{scope}/{mbid}/front{suffix}"
|
||||
|
||||
|
||||
def fetch_release_preferred_art(
|
||||
release_mbid: Optional[str],
|
||||
fallback_url: Optional[str],
|
||||
*,
|
||||
fetch_fn: Callable[[str], Tuple[Optional[bytes], Optional[str]]],
|
||||
size: int = 1200,
|
||||
min_bytes: int = 0,
|
||||
) -> Tuple[Optional[bytes], Optional[str], Optional[str]]:
|
||||
"""Fetch the best cover, preferring the specific release's own art.
|
||||
|
||||
Tries ``/release/<release_mbid>/front`` first; on any miss (no such art,
|
||||
404, or smaller than ``min_bytes``) falls back to ``fallback_url`` (a
|
||||
release-group / provider cover). ``fetch_fn(url) -> (bytes|None, mime|None)``;
|
||||
it is expected to return ``(None, None)`` on a 404, which is how a release
|
||||
with no art of its own advances to the fallback. ``min_bytes`` defaults to 0
|
||||
(accept any non-empty image) to preserve the fallback path's prior behaviour;
|
||||
callers can raise it to reject placeholder/error images. Returns
|
||||
``(bytes|None, mime|None, url_used|None)``. Never raises for a missing cover —
|
||||
a failed candidate just advances to the next, so coverage never regresses."""
|
||||
candidates = []
|
||||
release_url = caa_front_url(release_mbid, "release", size) if release_mbid else None
|
||||
if release_url:
|
||||
candidates.append(release_url)
|
||||
if fallback_url and fallback_url not in candidates:
|
||||
candidates.append(fallback_url)
|
||||
|
||||
for url in candidates:
|
||||
try:
|
||||
data, mime = fetch_fn(url)
|
||||
except Exception:
|
||||
data, mime = None, None
|
||||
if data and len(data) > min_bytes:
|
||||
return data, mime, url
|
||||
return None, None, None
|
||||
|
||||
|
||||
__all__ = ["COVER_ART_ARCHIVE_URL", "caa_front_url", "fetch_release_preferred_art"]
|
||||
|
|
@ -211,7 +211,7 @@ def pick_canonical_release(
|
|||
# core.library_reorganize._ALBUM_ID_COLUMNS — a test pins them in sync). A manual
|
||||
# match on any of these should pin/lock the canonical version (#758); a match on
|
||||
# a source the canonical tools don't read (e.g. lastfm) has no version to pin.
|
||||
CANONICAL_ALBUM_SOURCES = frozenset({'spotify', 'itunes', 'deezer', 'discogs', 'hydrabase'})
|
||||
CANONICAL_ALBUM_SOURCES = frozenset({'spotify', 'itunes', 'deezer', 'discogs', 'hydrabase', 'musicbrainz'})
|
||||
|
||||
|
||||
def should_pin_manual_canonical(entity_type: str, source: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -167,6 +167,7 @@ def track_already_owned(
|
|||
album_name: str,
|
||||
server_source: Optional[str],
|
||||
confidence_threshold: float = 0.7,
|
||||
candidate_tracks: Optional[List[Any]] = None,
|
||||
) -> bool:
|
||||
"""Return True if the track is already in the user's library.
|
||||
|
||||
|
|
@ -186,6 +187,15 @@ def track_already_owned(
|
|||
original deleted) matches just fine — track_name + artist + album
|
||||
don't change with format.
|
||||
|
||||
``candidate_tracks`` (when not None) is the artist's library tracks,
|
||||
pre-fetched ONCE by the caller, so the check scores in-memory instead
|
||||
of firing per-track fuzzy SQL scans against the whole library. Pass an
|
||||
empty list for an artist the user owns nothing of — it still routes
|
||||
through the fast in-memory path (scores against zero candidates →
|
||||
instant "not owned") rather than the slow per-track search. None
|
||||
preserves the original per-track-SQL behaviour for callers that don't
|
||||
pre-fetch.
|
||||
|
||||
Returns False on any exception so a transient DB hiccup doesn't
|
||||
silently nuke a discography fetch — a redundant wishlist add is
|
||||
much cheaper to recover from than a missed track.
|
||||
|
|
@ -198,6 +208,7 @@ def track_already_owned(
|
|||
confidence_threshold=confidence_threshold,
|
||||
server_source=server_source,
|
||||
album=album_name or None,
|
||||
candidate_tracks=candidate_tracks,
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -131,6 +131,14 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf
|
|||
track_num_str = format_track_number_tag(
|
||||
metadata.get('track_number'), metadata.get('total_tracks')
|
||||
)
|
||||
# Disc number is written UNCONDITIONALLY (floored to >=1), like the
|
||||
# track number above. The old code only wrote it when truthy, so a
|
||||
# track whose disc came back 0/None/'' (e.g. matched to a different
|
||||
# edition) lost its disc tag on the clear-then-rewrite and floated
|
||||
# ungrouped above the disc sections in Jellyfin/Plex (Sokhi).
|
||||
from core.imports.track_number import normalize_disc_number
|
||||
_disc_num = normalize_disc_number(metadata.get('disc_number'))
|
||||
disc_num_str = str(_disc_num)
|
||||
write_multi = cfg.get("metadata_enhancement.tags.write_multi_artist", False)
|
||||
artists_list = metadata.get("_artists_list", [])
|
||||
|
||||
|
|
@ -162,8 +170,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf
|
|||
if metadata.get("genre"):
|
||||
audio_file.tags.add(symbols.TCON(encoding=3, text=[metadata["genre"]]))
|
||||
audio_file.tags.add(symbols.TRCK(encoding=3, text=[track_num_str]))
|
||||
if metadata.get("disc_number"):
|
||||
audio_file.tags.add(symbols.TPOS(encoding=3, text=[str(metadata["disc_number"])]))
|
||||
audio_file.tags.add(symbols.TPOS(encoding=3, text=[disc_num_str]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
if metadata.get("title"):
|
||||
audio_file["title"] = [metadata["title"]]
|
||||
|
|
@ -180,8 +187,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf
|
|||
if metadata.get("genre"):
|
||||
audio_file["genre"] = [metadata["genre"]]
|
||||
audio_file["tracknumber"] = [track_num_str]
|
||||
if metadata.get("disc_number"):
|
||||
audio_file["discnumber"] = [str(metadata["disc_number"])]
|
||||
audio_file["discnumber"] = [disc_num_str]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
if metadata.get("title"):
|
||||
audio_file["\xa9nam"] = [metadata["title"]]
|
||||
|
|
@ -198,8 +204,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf
|
|||
audio_file["trkn"] = [format_track_number_tuple(
|
||||
metadata.get("track_number"), metadata.get("total_tracks")
|
||||
)]
|
||||
if metadata.get("disc_number"):
|
||||
audio_file["disk"] = [(metadata["disc_number"], 0)]
|
||||
audio_file["disk"] = [(_disc_num, 0)]
|
||||
|
||||
embed_source_ids(audio_file, metadata, context, runtime=runtime)
|
||||
|
||||
|
|
@ -242,18 +247,25 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf
|
|||
logger.warning("[Metadata Debug] Artist: %s", artist.get("name", "MISSING") if artist else "None")
|
||||
logger.warning("[Metadata Debug] Album info: %s", album_info.get("album_name", "MISSING") if album_info else "None")
|
||||
logger.error("[Metadata Debug] Traceback:\n%s", traceback.format_exc())
|
||||
# We cleared the file's art early; if the rewrite then crashed
|
||||
# before re-embedding, the on-disk file (already saved cleared at
|
||||
# the start) would be left art-less. Best-effort: put the original
|
||||
# art back and persist it so a mid-enrichment crash never destroys
|
||||
# the cover (#764). Guarded so a failure here can't mask the
|
||||
# original error.
|
||||
# The file was saved with tags CLEARED up front (so stale tags never
|
||||
# linger), then the failure-prone enrichment ran. By the time most
|
||||
# failures hit — the external source-id embed / cover-art fetch — the
|
||||
# core tags (album/artist/title/track from the matched context) are
|
||||
# already on the in-memory object but NOT yet on disk; the on-disk
|
||||
# file is still the cleared one. Persist the in-memory tags now (and
|
||||
# restore the original art too, #764) so a mid-enrichment crash leaves
|
||||
# a correctly-tagged file instead of an UNTAGGED one (Sokhi: tracks
|
||||
# landing in Rockbox's 'untagged' bucket after a 'processing failed').
|
||||
#
|
||||
# Previously this save was gated on there being original art to
|
||||
# restore, so an art-less file lost its tags entirely on any crash.
|
||||
# Guarded so a failure here can't mask the original error.
|
||||
try:
|
||||
if audio_file is not None and art_snapshot and restore_embedded_art(
|
||||
audio_file, symbols, art_snapshot
|
||||
):
|
||||
if audio_file is not None:
|
||||
if art_snapshot:
|
||||
restore_embedded_art(audio_file, symbols, art_snapshot)
|
||||
save_audio_file(audio_file, symbols)
|
||||
logger.info("Restored original cover art after enrichment error.")
|
||||
logger.info("Persisted core tags (and restored art) after enrichment error.")
|
||||
except Exception as restore_exc:
|
||||
logger.debug("Art restore after error failed: %s", restore_exc)
|
||||
logger.debug("Tag/art persist after error failed: %s", restore_exc)
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -230,15 +230,20 @@ def get_spotify_client_for_profile(profile_id: Optional[int] = None):
|
|||
return client
|
||||
|
||||
try:
|
||||
from core.spotify_client import SpotifyClient
|
||||
from core.spotify_client import SpotifyClient, normalize_spotify_oauth_config, SPOTIFY_OAUTH_SCOPE
|
||||
from spotipy.oauth2 import SpotifyOAuth
|
||||
import spotipy
|
||||
|
||||
normalized_creds = normalize_spotify_oauth_config({
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"redirect_uri": redirect_uri,
|
||||
})
|
||||
auth_manager = SpotifyOAuth(
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
redirect_uri=redirect_uri,
|
||||
scope="user-library-read user-read-private playlist-read-private playlist-read-collaborative user-read-email user-follow-read",
|
||||
client_id=normalized_creds.get("client_id", client_id),
|
||||
client_secret=normalized_creds.get("client_secret", client_secret),
|
||||
redirect_uri=normalized_creds.get("redirect_uri", redirect_uri),
|
||||
scope=SPOTIFY_OAUTH_SCOPE,
|
||||
cache_path=cache_path,
|
||||
state=f"profile_{profile_id}",
|
||||
)
|
||||
|
|
@ -352,10 +357,26 @@ def get_hydrabase_client(allow_fallback: bool = True, require_enabled: bool = Tr
|
|||
return None
|
||||
|
||||
|
||||
def get_configured_primary_source() -> str:
|
||||
"""Return metadata.fallback_source as stored in config, without runtime downgrade.
|
||||
|
||||
Unlike get_primary_source(), this never probes Spotify auth. Use at boot and
|
||||
anywhere blocking network I/O must be avoided (gunicorn worker import, Docker
|
||||
cold start when Spotify is unreachable).
|
||||
"""
|
||||
_default = METADATA_SOURCE_PRIORITY[0]
|
||||
return _get_config_value("metadata.fallback_source", _default) or _default
|
||||
|
||||
|
||||
def get_primary_source(spotify_client_factory: Optional[MetadataClientFactory] = None) -> str:
|
||||
"""Return configured primary metadata source."""
|
||||
from core.boot_phase import is_boot_phase
|
||||
|
||||
if is_boot_phase():
|
||||
return get_configured_primary_source()
|
||||
|
||||
_default = METADATA_SOURCE_PRIORITY[0]
|
||||
source = _get_config_value("metadata.fallback_source", _default) or _default
|
||||
source = get_configured_primary_source()
|
||||
|
||||
if source == "spotify":
|
||||
try:
|
||||
|
|
@ -368,6 +389,24 @@ def get_primary_source(spotify_client_factory: Optional[MetadataClientFactory] =
|
|||
return source
|
||||
|
||||
|
||||
def get_primary_source_label() -> str:
|
||||
"""Configured primary source for UI that *names* "your primary source" (the
|
||||
import-search fallback banner, etc.).
|
||||
|
||||
Identical to ``get_primary_source()`` except it does NOT downgrade a no-auth
|
||||
Spotify Free user to the working fallback: Spotify Free (fallback_source=
|
||||
'spotify' + metadata.spotify_free) is reported as 'spotify', because that IS
|
||||
their configured source — even though free-text album search itself has no
|
||||
free-path implementation and legitimately falls back to another provider.
|
||||
``get_primary_source()`` keeps the downgrade so client routing always yields a
|
||||
usable client; labels want the user's actual intent, not the fallback."""
|
||||
_default = METADATA_SOURCE_PRIORITY[0]
|
||||
source = _get_config_value("metadata.fallback_source", _default) or _default
|
||||
if source == "spotify" and _get_config_value("metadata.spotify_free", False):
|
||||
return "spotify"
|
||||
return get_primary_source()
|
||||
|
||||
|
||||
def get_spotify_disconnect_source(configured_source: Optional[str] = None) -> str:
|
||||
"""Return the active metadata source after Spotify is disconnected."""
|
||||
_default = METADATA_SOURCE_PRIORITY[0]
|
||||
|
|
@ -424,7 +463,19 @@ def get_primary_source_status(
|
|||
musicbrainz_client_factory: Optional[MetadataClientFactory] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Return a generic status snapshot for the active primary metadata source."""
|
||||
from core.boot_phase import is_boot_phase
|
||||
|
||||
source = _get_config_value("metadata.fallback_source", "deezer") or "deezer"
|
||||
if is_boot_phase():
|
||||
display_source = source
|
||||
if source == "spotify" and _get_config_value("metadata.spotify_free", False):
|
||||
display_source = "spotify_free"
|
||||
return {
|
||||
"source": display_source,
|
||||
"connected": False,
|
||||
"response_time": 0,
|
||||
}
|
||||
|
||||
started = time.time()
|
||||
connected = False
|
||||
|
||||
|
|
@ -492,9 +543,13 @@ def get_client_for_source(
|
|||
musicbrainz_client_factory: Optional[MetadataClientFactory] = None,
|
||||
):
|
||||
"""Return exact client for a source, or None if unavailable."""
|
||||
from core.boot_phase import is_boot_phase
|
||||
|
||||
if source == "spotify":
|
||||
try:
|
||||
client = get_spotify_client(client_factory=spotify_client_factory)
|
||||
if is_boot_phase():
|
||||
return client if client and getattr(client, "sp", None) else None
|
||||
if client and client.is_spotify_authenticated():
|
||||
return client
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -52,6 +52,27 @@ from typing import List, Optional, Sequence
|
|||
from core.metadata.types import Track
|
||||
|
||||
|
||||
def build_combined_search_query(track: str = '', artist: str = '',
|
||||
legacy: str = '') -> str:
|
||||
"""Combine a track + artist into a PLAIN, source-agnostic search query.
|
||||
|
||||
Deliberately NOT field-scoped (``track:"X" artist:"Y"``). That Spotify/Lucene
|
||||
filter syntax leaks to non-Spotify sources when a search falls back: Deezer
|
||||
closed the connection on it outright (``RemoteDisconnected``), and even sources
|
||||
that parse it over-constrain and miss the canonical cut (the iTunes/Deezer search
|
||||
endpoints already dropped it for that reason). The matching ``/search_tracks``
|
||||
endpoints rerank by expected title/artist afterward, so precision is recovered
|
||||
without the brittle field syntax.
|
||||
|
||||
Falls back to ``legacy`` when neither track nor artist is given. Returns ``''``
|
||||
when there's nothing to search (caller decides how to handle empty).
|
||||
"""
|
||||
parts = [p.strip() for p in (track, artist) if p and p.strip()]
|
||||
if parts:
|
||||
return ' '.join(parts)
|
||||
return (legacy or '').strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pattern tables — public so tests can introspect, callers can extend
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -1089,10 +1089,12 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di
|
|||
metadata["track_number"] = 1
|
||||
metadata["total_tracks"] = 1
|
||||
|
||||
disc_num = original_search.get("disc_number")
|
||||
if disc_num is None and album_info:
|
||||
disc_num = album_info.get("disc_number")
|
||||
metadata["disc_number"] = disc_num if disc_num is not None else 1
|
||||
# Resolve via the SHARED resolver so the embedded tag and the "Disc N" folder
|
||||
# (computed in the import pipeline from album_info) can never disagree — same
|
||||
# function, same inputs. Floors to >=1 (a 0/''/non-numeric disc must not read
|
||||
# as disc-less and ungroup the track in Jellyfin/Plex).
|
||||
from core.imports.track_number import resolve_disc_for_track
|
||||
metadata["disc_number"] = resolve_disc_for_track(original_search, album_info)
|
||||
|
||||
if album_ctx and album_ctx.get("release_date"):
|
||||
release_date = _normalize_release_date_tag(album_ctx.get("release_date"))
|
||||
|
|
|
|||
|
|
@ -49,8 +49,10 @@ from core.metadata.registry import (
|
|||
get_discogs_client,
|
||||
get_hydrabase_client,
|
||||
get_itunes_client,
|
||||
get_configured_primary_source,
|
||||
get_primary_client,
|
||||
get_primary_source,
|
||||
get_primary_source_label,
|
||||
get_spotify_client_for_profile,
|
||||
get_registered_runtime_client,
|
||||
get_source_priority,
|
||||
|
|
@ -116,7 +118,9 @@ __all__ = [
|
|||
"get_metadata_service",
|
||||
"get_musicmap_similar_artists",
|
||||
"get_primary_client",
|
||||
"get_configured_primary_source",
|
||||
"get_primary_source",
|
||||
"get_primary_source_label",
|
||||
"get_spotify_client_for_profile",
|
||||
"get_registered_runtime_client",
|
||||
"get_spotify_client",
|
||||
|
|
|
|||
|
|
@ -110,6 +110,7 @@ class NavidromeTrack:
|
|||
self.title = navidrome_data.get('title', 'Unknown Track')
|
||||
self.duration = navidrome_data.get('duration', 0) * 1000 # Convert to milliseconds
|
||||
self.trackNumber = navidrome_data.get('track')
|
||||
self.discNumber = navidrome_data.get('discNumber') # multi-disc: disc number
|
||||
self.year = navidrome_data.get('year')
|
||||
self.userRating = navidrome_data.get('userRating')
|
||||
self.addedAt = self._parse_date(navidrome_data.get('created'))
|
||||
|
|
@ -1022,6 +1023,34 @@ class NavidromeClient(MediaServerClient):
|
|||
logger.error(f"Error {'updating' if playlist_id else 'creating'} Navidrome playlist '{name}': {e}")
|
||||
return False
|
||||
|
||||
def rewrite_playlist_order(self, playlist_id: str, name: str, ordered_song_ids) -> bool:
|
||||
"""Rewrite a playlist's tracks to an exact ordered id list (Subsonic has no
|
||||
per-track move — the only reorder primitive is overwriting the whole song
|
||||
list). Overwrites in place via createPlaylist + playlistId, so the playlist
|
||||
identity (id/name) survives. Used ONLY by the 'Align playlists' path with
|
||||
ids already present in the playlist — never adds a new track.
|
||||
|
||||
NOTE: like every createPlaylist+playlistId overwrite (same as replace-mode
|
||||
sync), Navidrome may not carry the playlist's comment forward — the caller
|
||||
re-applies it if needed."""
|
||||
if not self.ensure_connection():
|
||||
return False
|
||||
ids = [str(i) for i in (ordered_song_ids or []) if str(i)]
|
||||
if not ids:
|
||||
logger.warning(f"rewrite_playlist_order: no song ids for '{name}'")
|
||||
return False
|
||||
try:
|
||||
params = {'name': name, 'songId': ids, 'playlistId': playlist_id}
|
||||
response = self._make_request('createPlaylist', params)
|
||||
if response and response.get('status') == 'ok':
|
||||
logger.info(f"Aligned Navidrome playlist '{name}' order ({len(ids)} tracks)")
|
||||
return True
|
||||
logger.error(f"rewrite_playlist_order failed for '{name}'")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error rewriting Navidrome playlist order '{name}': {e}")
|
||||
return False
|
||||
|
||||
def copy_playlist(self, source_name: str, target_name: str) -> bool:
|
||||
"""Copy a playlist to create a backup"""
|
||||
if not self.ensure_connection():
|
||||
|
|
@ -1192,7 +1221,11 @@ class NavidromeClient(MediaServerClient):
|
|||
|
||||
primary = existing_playlists[0]
|
||||
existing_tracks = self.get_playlist_tracks(primary.id)
|
||||
current_ids = [str(t.id) for t in existing_tracks if getattr(t, 'id', None)]
|
||||
# #905: NavidromeTrack exposes the Subsonic song id as `ratingKey` (NOT `.id`,
|
||||
# which doesn't exist) — same as append_to_playlist reads it. Reading `t.id` here
|
||||
# made current_ids ALWAYS empty, so reconcile thought the playlist was empty and
|
||||
# re-added every track each sync (playlists doubling) while removing nothing.
|
||||
current_ids = [str(t.ratingKey) for t in existing_tracks if getattr(t, 'ratingKey', None)]
|
||||
desired_ids = []
|
||||
for t in tracks:
|
||||
tid = (str(t.ratingKey) if hasattr(t, 'ratingKey')
|
||||
|
|
|
|||
|
|
@ -25,3 +25,4 @@ from core.personalized.generators import daily_mix # noqa: F401
|
|||
from core.personalized.generators import fresh_tape # noqa: F401
|
||||
from core.personalized.generators import archives # noqa: F401
|
||||
from core.personalized.generators import seasonal_mix # noqa: F401
|
||||
from core.personalized.generators import listening_mix # noqa: F401
|
||||
|
|
|
|||
68
core/personalized/generators/listening_mix.py
Normal file
68
core/personalized/generators/listening_mix.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
"""Listening Mix (#913) generator.
|
||||
|
||||
Reads the full, render-ready track dicts the watchlist scan stored under the
|
||||
``listening_recs_tracks_full`` metadata key — built from the recommended artists'
|
||||
top tracks (see ``core.watchlist_scanner._build_listening_recommendations``) — and
|
||||
coerces them into ``Track`` records.
|
||||
|
||||
Unlike Fresh Tape / Archives this needs NO discovery-pool hydration: the stored
|
||||
dicts are already complete, so the snapshot can't shrink when the pool rotates. It
|
||||
also means the generator is a pure read — generation/network all happen during the
|
||||
scan; this just hands the stored tracks to the personalized manager so the mix can
|
||||
participate in the Sync-page mirror + Auto-Sync pipeline like every other kind."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, List
|
||||
|
||||
from core.personalized.specs import PlaylistKindSpec, get_registry
|
||||
from core.personalized.types import PlaylistConfig, Track
|
||||
|
||||
|
||||
KIND = 'listening_mix'
|
||||
METADATA_KEY = 'listening_recs_tracks_full'
|
||||
|
||||
|
||||
def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]:
|
||||
"""Return the stored Listening Mix tracks, trimmed to ``config.limit``.
|
||||
|
||||
Empty (not an error) when the scan hasn't produced a mix yet — the manager
|
||||
preserves any prior snapshot rather than dropping it. Tolerates a missing/garbage
|
||||
metadata blob the same way.
|
||||
"""
|
||||
db = getattr(deps, 'database', None) or (deps.get('database') if isinstance(deps, dict) else None)
|
||||
if db is None:
|
||||
raise RuntimeError("Listening Mix generator deps missing `database`")
|
||||
|
||||
raw = db.get_metadata(METADATA_KEY)
|
||||
if not raw:
|
||||
return []
|
||||
try:
|
||||
rows = json.loads(raw) or []
|
||||
except (ValueError, TypeError):
|
||||
return []
|
||||
|
||||
tracks: List[Track] = []
|
||||
for d in rows:
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
tracks.append(Track.from_dict(d))
|
||||
if len(tracks) >= config.limit:
|
||||
break
|
||||
return tracks
|
||||
|
||||
|
||||
SPEC = PlaylistKindSpec(
|
||||
kind=KIND,
|
||||
name_template='Your Listening Mix',
|
||||
description='Tracks from artists matched to what you actually listen to.',
|
||||
default_config=PlaylistConfig(limit=50, max_per_album=5, max_per_artist=3),
|
||||
generator=generate,
|
||||
requires_variant=False,
|
||||
tags=['curated', 'listening'],
|
||||
)
|
||||
|
||||
|
||||
if get_registry().get(KIND) is None:
|
||||
get_registry().register(SPEC)
|
||||
|
|
@ -111,7 +111,10 @@ class PersonalizedPlaylistsService:
|
|||
Either value can be None to skip that filter for that source.
|
||||
|
||||
- Spotify: 60 / 40 (the existing 0-100 popularity scale)
|
||||
- Deezer: 500000 / 100000 (rank values — ballpark from real data)
|
||||
- Deezer: 60 / 50 (ALSO 0-100 — the discovery pool SYNTHESIZES deezer popularity to a
|
||||
0-100 score at scan time, capped at 100; it is NOT Deezer's raw rank. The old
|
||||
500000/100000 rank thresholds matched nothing, so Popular Picks came back empty for
|
||||
deezer users while Hidden Gems' < 100000 caught the whole pool.)
|
||||
- iTunes / others: None / None (no popularity data; fall back to no
|
||||
threshold filter, just diversity-on-random)
|
||||
|
||||
|
|
@ -121,7 +124,7 @@ class PersonalizedPlaylistsService:
|
|||
if normalized == 'spotify':
|
||||
return 60, 40
|
||||
if normalized == 'deezer':
|
||||
return 500_000, 100_000
|
||||
return 60, 50
|
||||
# iTunes, hydrabase, anything else — no usable popularity data
|
||||
return None, None
|
||||
|
||||
|
|
|
|||
94
core/playlists/item_naming.py
Normal file
94
core/playlists/item_naming.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
"""Optional custom naming for the FILES inside an organize-by-playlist folder.
|
||||
|
||||
By default a playlist entry keeps the real library filename (the materialized
|
||||
folder is a view onto Artist/Album/track.ext). A user can opt into a flat
|
||||
filename template — e.g. ``$position - $artist - $title`` — so the folder sorts
|
||||
and plays the way they want (most commonly: in playlist order on a dumb DAP).
|
||||
|
||||
It is a **filename** template, never a path:
|
||||
- it may NOT contain a path separator (``/`` or ``\\``) — it names the file,
|
||||
not a folder tree, and
|
||||
- it MUST contain ``$title`` — so every file has a real, non-empty name.
|
||||
|
||||
Both rules are validated up front (so the Settings UI can reject a bad value
|
||||
with a reason) AND re-checked at apply time, where an invalid/empty template or
|
||||
an empty render falls back to the library filename. So a bad value can never
|
||||
produce a broken name — the worst case is "no change from today".
|
||||
|
||||
Pure logic: no DB, no config, no filesystem. The caller supplies the metadata.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from core.imports.paths import sanitize_filename
|
||||
|
||||
# Tokens a user may use in the template (for docs / UI hints).
|
||||
PLAYLIST_ITEM_TOKENS = ("$position", "$artist", "$album", "$track", "$title")
|
||||
|
||||
|
||||
def validate_playlist_item_template(template: Optional[str]) -> Tuple[bool, str]:
|
||||
"""Return ``(ok, reason)``. An empty template is VALID and means "feature off"
|
||||
(keep the library filename). ``reason`` is '' when ok."""
|
||||
t = (template or "").strip()
|
||||
if not t:
|
||||
return True, "" # empty == disabled, not an error
|
||||
if "/" in t or "\\" in t:
|
||||
return False, ("Playlist file naming can't contain a folder separator "
|
||||
"( / or \\ ) — it names the file, not a path.")
|
||||
if "$title" not in t:
|
||||
return False, "Playlist file naming must include $title so every file has a name."
|
||||
return True, ""
|
||||
|
||||
|
||||
def render_playlist_item_name(
|
||||
template: Optional[str],
|
||||
*,
|
||||
title: str,
|
||||
artist: str = "",
|
||||
album: str = "",
|
||||
track: object = None,
|
||||
position: object = None,
|
||||
ext: str = "",
|
||||
fallback_name: str = "",
|
||||
) -> str:
|
||||
"""Render ``template`` to a sanitized filename WITH ``ext`` appended.
|
||||
|
||||
Falls back to ``fallback_name`` (the library filename) when the template is
|
||||
empty/invalid or renders to nothing after sanitizing — so the result is
|
||||
never broken. ``position`` is used verbatim (the caller pre-pads it for
|
||||
correct sorting); ``track`` is zero-padded to two digits when numeric."""
|
||||
ok, _ = validate_playlist_item_template(template)
|
||||
t = (template or "").strip()
|
||||
if not ok or not t:
|
||||
return fallback_name
|
||||
|
||||
pos_str = "" if position is None else str(position)
|
||||
if track is None:
|
||||
trk_str = ""
|
||||
else:
|
||||
try:
|
||||
trk_str = f"{int(track):02d}"
|
||||
except (TypeError, ValueError):
|
||||
trk_str = str(track)
|
||||
|
||||
# No token is a prefix of another, so replacement order is irrelevant.
|
||||
out = t
|
||||
out = out.replace("$position", pos_str)
|
||||
out = out.replace("$artist", str(artist or ""))
|
||||
out = out.replace("$album", str(album or ""))
|
||||
out = out.replace("$track", trk_str)
|
||||
out = out.replace("$title", str(title or ""))
|
||||
|
||||
out = sanitize_filename(out).strip()
|
||||
if not out:
|
||||
return fallback_name
|
||||
return out + (ext or "")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PLAYLIST_ITEM_TOKENS",
|
||||
"validate_playlist_item_template",
|
||||
"render_playlist_item_name",
|
||||
]
|
||||
|
|
@ -72,16 +72,25 @@ def playlist_dir_for(playlists_root: str, playlist_name: str) -> str:
|
|||
return candidate
|
||||
|
||||
|
||||
def _desired_entries(playlist_dir: str, real_paths: Sequence[str]) -> "list[tuple[str, str]]":
|
||||
"""Map each real file to a flat destination inside ``playlist_dir``, preserving
|
||||
the source filename. On a basename collision between two *different* sources,
|
||||
disambiguate with a numeric suffix rather than silently overwriting."""
|
||||
def _desired_entries(
|
||||
playlist_dir: str,
|
||||
real_paths: Sequence[str],
|
||||
dest_names: Optional[Sequence[Optional[str]]] = None,
|
||||
) -> "list[tuple[str, str]]":
|
||||
"""Map each real file to a flat destination inside ``playlist_dir``.
|
||||
|
||||
By default the source filename is preserved. ``dest_names`` (parallel to
|
||||
``real_paths``) lets a caller override the name per entry — e.g. a custom
|
||||
playlist file-naming template; a falsy override falls back to the source
|
||||
basename. On a name collision between two *different* sources, disambiguate
|
||||
with a numeric suffix rather than silently overwriting."""
|
||||
entries: list[tuple[str, str]] = []
|
||||
used: dict[str, str] = {} # dest basename -> source real path
|
||||
for real in real_paths:
|
||||
for i, real in enumerate(real_paths):
|
||||
if not real:
|
||||
continue
|
||||
base = os.path.basename(real)
|
||||
override = dest_names[i] if (dest_names is not None and i < len(dest_names)) else None
|
||||
base = override or os.path.basename(real)
|
||||
name = base
|
||||
stem, ext = os.path.splitext(base)
|
||||
counter = 1
|
||||
|
|
@ -156,6 +165,7 @@ def rebuild_playlist_folder(
|
|||
real_paths: Sequence[str],
|
||||
mode: str = DEFAULT_MODE,
|
||||
*,
|
||||
dest_names: Optional[Sequence[Optional[str]]] = None,
|
||||
prune_stale: bool = True,
|
||||
symlink_fn: Callable[[str, str], None] = os.symlink,
|
||||
copy_fn: Callable[[str, str], object] = shutil.copy2,
|
||||
|
|
@ -163,13 +173,15 @@ def rebuild_playlist_folder(
|
|||
"""(Re)build ``playlists_root/<playlist_name>/`` so it contains exactly one
|
||||
entry per real file in ``real_paths`` — adding missing entries, leaving correct
|
||||
ones untouched, and (when ``prune_stale``) removing entries no longer present.
|
||||
``dest_names`` (parallel to ``real_paths``) optionally overrides each entry's
|
||||
filename (custom playlist naming); falsy entries keep the source basename.
|
||||
Idempotent and safe to re-run any time. Filesystem ops are injectable."""
|
||||
mode = normalize_mode(mode)
|
||||
pdir = playlist_dir_for(playlists_root, playlist_name)
|
||||
summary = RebuildSummary(playlist_dir=pdir, mode_requested=mode)
|
||||
os.makedirs(pdir, exist_ok=True)
|
||||
|
||||
entries = _desired_entries(pdir, real_paths)
|
||||
entries = _desired_entries(pdir, real_paths, dest_names)
|
||||
keep = {dest for _real, dest in entries}
|
||||
|
||||
for real, dest in entries:
|
||||
|
|
|
|||
|
|
@ -155,11 +155,20 @@ def _rebuild_one_from_db(db, config_manager, playlist: dict):
|
|||
source IDs), resolving to disk, and rebuilding WITH prune. Because it's driven
|
||||
by current membership, a track that has LEFT the playlist drops out of the set
|
||||
and its symlink is pruned. Returns ``(playlist_name, RebuildSummary)``."""
|
||||
import os as _os
|
||||
|
||||
from core.library.path_resolver import resolve_library_file_path
|
||||
from core.playlists.item_naming import render_playlist_item_name
|
||||
|
||||
root = docker_resolve_path(config_manager.get("playlists.materialize_path", "./Playlists"))
|
||||
mode = normalize_mode(config_manager.get("playlists.materialize_mode", "symlink"))
|
||||
real_paths: List[str] = []
|
||||
item_template = ((config_manager.get("file_organization.templates", {}) or {})
|
||||
.get("playlist_item", "") or "").strip()
|
||||
|
||||
# Resolve owned tracks to real paths IN PLAYLIST ORDER, keeping the metadata
|
||||
# so an optional custom filename template ($position/$artist/$title/...) can
|
||||
# be applied. $position is the playlist index, which is exactly this order.
|
||||
resolved: List[dict] = []
|
||||
seen = set()
|
||||
for t in (db.get_mirrored_playlist_tracks(playlist["id"]) or []):
|
||||
title = (t.get("track_name") or "").strip()
|
||||
|
|
@ -175,9 +184,32 @@ def _rebuild_one_from_db(db, config_manager, playlist: dict):
|
|||
real = resolve_library_file_path(getattr(db_track, "file_path", None), config_manager=config_manager)
|
||||
if real and real not in seen:
|
||||
seen.add(real)
|
||||
real_paths.append(real)
|
||||
resolved.append({
|
||||
"real": real,
|
||||
"title": title,
|
||||
"artist": artist,
|
||||
"album": (t.get("album_name") or t.get("album") or "").strip(),
|
||||
"track": getattr(db_track, "track_number", None),
|
||||
})
|
||||
|
||||
real_paths: List[str] = [r["real"] for r in resolved]
|
||||
|
||||
dest_names = None
|
||||
if item_template:
|
||||
width = max(2, len(str(len(resolved)))) # zero-pad $position for correct sorting
|
||||
dest_names = [
|
||||
render_playlist_item_name(
|
||||
item_template,
|
||||
title=r["title"], artist=r["artist"], album=r["album"], track=r["track"],
|
||||
position=f"{i:0{width}d}",
|
||||
ext=_os.path.splitext(r["real"])[1],
|
||||
fallback_name=_os.path.basename(r["real"]),
|
||||
)
|
||||
for i, r in enumerate(resolved, start=1)
|
||||
]
|
||||
|
||||
name = playlist.get("name") or "Unnamed Playlist"
|
||||
return name, rebuild_playlist_folder(root, name, real_paths, mode) # prune_stale=True
|
||||
return name, rebuild_playlist_folder(root, name, real_paths, mode, dest_names=dest_names) # prune_stale=True
|
||||
|
||||
|
||||
def rebuild_organized_playlists_from_db(db, config_manager, *, profile_id: int = 1):
|
||||
|
|
|
|||
|
|
@ -19,6 +19,35 @@ from urllib.parse import parse_qs, urlparse
|
|||
|
||||
_SPOTIFY_ID_RE = re.compile(r"^[A-Za-z0-9]{16,32}$")
|
||||
|
||||
|
||||
def stable_source_track_id(track: Mapping, existing: Optional[str] = None) -> str:
|
||||
"""A stable per-track id for a mirrored-playlist track.
|
||||
|
||||
Spotify / YouTube / Deezer tracks carry a native id. File-import (CSV / M3U /
|
||||
TXT) and iTunes-only sources don't — they arrive with an empty
|
||||
``source_track_id``. The whole manual-match system (Find & Add ↔ sync) keys on
|
||||
``source_track_id``, and an empty key can neither be recorded (the persist is a
|
||||
no-op) nor looked up — so a manual match on a file-import track is silently
|
||||
dropped and the track re-appears as "extra" (#901).
|
||||
|
||||
When a native id is present it's used verbatim. Otherwise we derive a
|
||||
DETERMINISTIC id from the track's identity (artist|title|album, normalized) so
|
||||
the SAME song gets the SAME id across re-imports and discovery passes — which
|
||||
is exactly what the match lookup needs. Prefixed ``file:`` so it's recognizable
|
||||
and never collides with a real upstream id. Returns '' only when there's no
|
||||
usable identity at all (no title)."""
|
||||
native = (existing if existing is not None else track.get("source_track_id")) or ""
|
||||
native = str(native).strip()
|
||||
if native:
|
||||
return native
|
||||
title = str(track.get("track_name") or track.get("name") or "").strip().lower()
|
||||
if not title:
|
||||
return ""
|
||||
artist = str(track.get("artist_name") or track.get("artist") or "").strip().lower()
|
||||
album = str(track.get("album_name") or track.get("album") or "").strip().lower()
|
||||
digest = hashlib.md5(f"{artist}|{title}|{album}".encode("utf-8")).hexdigest()[:16]
|
||||
return f"file:{digest}"
|
||||
|
||||
# Synthetic batch playlist_id prefixes that wrap a mirrored_playlists PK.
|
||||
# Download/discovery flows build a batch playlist_id as f"{prefix}{pk}" — e.g.
|
||||
# auto_mirror_<pk> (core/automation/handlers/sync_playlist.py), youtube_mirrored_<pk>
|
||||
|
|
|
|||
|
|
@ -714,6 +714,88 @@ class PlexClient(MediaServerClient):
|
|||
logger.error(f"Error reconciling Plex playlist '{playlist_name}': {e}")
|
||||
return False
|
||||
|
||||
def get_playlist_track_ids(self, playlist_id, playlist_name: str = "") -> List[str]:
|
||||
"""The playlist's current track ratingKeys, in current order. [] if missing."""
|
||||
if not self.ensure_connection():
|
||||
return []
|
||||
try:
|
||||
playlist = None
|
||||
try:
|
||||
playlist = self.server.fetchItem(int(playlist_id))
|
||||
except Exception:
|
||||
playlist = None
|
||||
if playlist is None and playlist_name:
|
||||
try:
|
||||
playlist = self.server.playlist(playlist_name)
|
||||
except Exception:
|
||||
playlist = None
|
||||
if playlist is None:
|
||||
return []
|
||||
return [str(i.ratingKey) for i in playlist.items() if hasattr(i, 'ratingKey')]
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting Plex playlist track ids '{playlist_name}': {e}")
|
||||
return []
|
||||
|
||||
def reorder_playlist(self, playlist_id, playlist_name: str, ordered_ids) -> bool:
|
||||
"""In-place reorder a playlist to an exact ordered ratingKey list ('Align
|
||||
playlists'). Moves items into sequence and removes any current item NOT in
|
||||
``ordered_ids`` (so 'Mirror source' drops extras; 'Keep extras' includes
|
||||
them in the list). Uses moveItem/removeItems so the playlist's poster,
|
||||
summary and ratingKey survive — never deletes/recreates. Order-only: every
|
||||
id in ``ordered_ids`` is already in the playlist (the caller validated)."""
|
||||
if not self.ensure_connection():
|
||||
return False
|
||||
try:
|
||||
playlist = None
|
||||
try:
|
||||
playlist = self.server.fetchItem(int(playlist_id))
|
||||
except Exception as e:
|
||||
logger.debug("Plex reorder fetchItem failed: %s", e)
|
||||
if playlist is None and playlist_name:
|
||||
try:
|
||||
playlist = self.server.playlist(playlist_name)
|
||||
except Exception as e:
|
||||
logger.debug("Plex reorder by-name failed: %s", e)
|
||||
if playlist is None:
|
||||
logger.error(f"Plex reorder: playlist not found (id={playlist_id}, name='{playlist_name}')")
|
||||
return False
|
||||
|
||||
ordered = [str(i) for i in (ordered_ids or []) if str(i)]
|
||||
ordered_set = set(ordered)
|
||||
items = [i for i in playlist.items() if hasattr(i, 'ratingKey')]
|
||||
by_rk = {str(i.ratingKey): i for i in items}
|
||||
|
||||
# Drop items not in the desired list (extras, for Mirror source).
|
||||
to_remove = [i for i in items if str(i.ratingKey) not in ordered_set]
|
||||
if to_remove:
|
||||
try:
|
||||
playlist.removeItems(to_remove)
|
||||
except (AttributeError, TypeError):
|
||||
for it in to_remove:
|
||||
try:
|
||||
playlist.removeItem(it)
|
||||
except Exception as one_err:
|
||||
logger.debug("Plex reorder: removeItem failed: %s", one_err)
|
||||
|
||||
# Move each desired item into place: first to the front (after=None),
|
||||
# then each subsequent one after its predecessor.
|
||||
prev = None
|
||||
for sid in ordered:
|
||||
it = by_rk.get(sid)
|
||||
if it is None:
|
||||
continue
|
||||
try:
|
||||
playlist.moveItem(it, after=prev)
|
||||
except Exception as mv_err:
|
||||
logger.warning(f"Plex reorder moveItem failed for {sid}: {mv_err}")
|
||||
return False
|
||||
prev = it
|
||||
logger.info(f"Aligned Plex playlist '{playlist_name}' order ({len(ordered)} tracks)")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error reordering Plex playlist '{playlist_name}': {e}")
|
||||
return False
|
||||
|
||||
def update_playlist(self, playlist_name: str, tracks: List[TrackInfo]) -> bool:
|
||||
if not self.ensure_connection():
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from config.settings import config_manager
|
|||
|
||||
# Import Soulseek data structures for drop-in replacement compatibility
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.quality.source_map import quality_from_qobuz, quality_tier_for_source
|
||||
|
||||
logger = get_logger("qobuz_client")
|
||||
|
||||
|
|
@ -119,7 +120,10 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
|
||||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
|
||||
logger.info(f"Qobuz client using download path: {self.download_path}")
|
||||
|
||||
|
|
@ -160,6 +164,8 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
|
||||
def _restore_session(self):
|
||||
"""Try to restore saved session from config."""
|
||||
from core.boot_phase import is_boot_phase
|
||||
|
||||
saved = config_manager.get('qobuz.session', {})
|
||||
app_id = saved.get('app_id', '')
|
||||
app_secret = saved.get('app_secret', '')
|
||||
|
|
@ -174,6 +180,10 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
'X-User-Auth-Token': self.user_auth_token,
|
||||
})
|
||||
|
||||
if is_boot_phase():
|
||||
logger.info("Loaded Qobuz session from config (verification deferred until after boot)")
|
||||
return
|
||||
|
||||
# Verify the token is still valid
|
||||
try:
|
||||
resp = self.session.get(
|
||||
|
|
@ -711,6 +721,26 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
logger.error(f"Error getting Qobuz track {track_id}: {e}")
|
||||
return None
|
||||
|
||||
def get_track_result(self, track_id):
|
||||
"""Fetch ONE track by ID and convert it to a downloadable TrackResult.
|
||||
|
||||
Used when a pasted Qobuz link must inject the EXACT track: a text search
|
||||
for an obscure track's name often doesn't surface it at all, so bubbling
|
||||
the matching id is a no-op (#932). Returns None on any failure so the
|
||||
caller falls back to the text-search path."""
|
||||
try:
|
||||
track = self.get_track(track_id)
|
||||
if not track:
|
||||
return None
|
||||
quality_key = quality_tier_for_source('qobuz', default='lossless')
|
||||
quality_info = QOBUZ_QUALITY_MAP.get(quality_key, QOBUZ_QUALITY_MAP['lossless'])
|
||||
# require_streamable=False: this is the exact track the user linked — don't
|
||||
# let a missing/absent 'streamable' flag in track/get drop it (#932).
|
||||
return self._qobuz_to_track_result(track, quality_info, require_streamable=False)
|
||||
except Exception as e:
|
||||
logger.debug(f"get_track_result failed for Qobuz {track_id}: {e}")
|
||||
return None
|
||||
|
||||
# ===================== Playlists & Favorites =====================
|
||||
#
|
||||
# Qobuz playlist sync surface — mirrors the Tidal client contract
|
||||
|
|
@ -987,7 +1017,7 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
return ([], [])
|
||||
|
||||
# Get configured quality for display
|
||||
quality_key = config_manager.get('qobuz.quality', 'lossless')
|
||||
quality_key = quality_tier_for_source('qobuz', default='lossless')
|
||||
quality_info = QOBUZ_QUALITY_MAP.get(quality_key, QOBUZ_QUALITY_MAP['lossless'])
|
||||
|
||||
track_results = []
|
||||
|
|
@ -1008,14 +1038,20 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
traceback.print_exc()
|
||||
return ([], [])
|
||||
|
||||
def _qobuz_to_track_result(self, track: Dict, quality_info: dict) -> Optional[TrackResult]:
|
||||
"""Convert Qobuz track dict to TrackResult (Soulseek-compatible format)."""
|
||||
def _qobuz_to_track_result(self, track: Dict, quality_info: dict,
|
||||
require_streamable: bool = True) -> Optional[TrackResult]:
|
||||
"""Convert Qobuz track dict to TrackResult (Soulseek-compatible format).
|
||||
|
||||
``require_streamable`` gates bulk SEARCH results on the ``streamable`` flag
|
||||
(filters noise). For a track the user explicitly pasted a link to, pass
|
||||
False: ``track/get`` may omit the flag (which would default-False and wrongly
|
||||
drop the exact track they asked for), and they should get to try it (#932)."""
|
||||
track_id = track.get('id')
|
||||
if not track_id:
|
||||
return None
|
||||
|
||||
# Check if track is streamable
|
||||
if not track.get('streamable', False):
|
||||
# Check if track is streamable (skipped for explicit by-id link fetches).
|
||||
if require_streamable and not track.get('streamable', False):
|
||||
return None
|
||||
|
||||
performer = track.get('performer', {})
|
||||
|
|
@ -1070,8 +1106,20 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
title=title,
|
||||
album=album_name,
|
||||
track_number=track.get('track_number'),
|
||||
# Stamp the Qobuz track id so a pasted-link manual search can float the
|
||||
# exact track to the top (#932). Without this the bubble never matched
|
||||
# and the linked track stayed buried among fuzzy lookalikes.
|
||||
_source_metadata={'source': 'qobuz', 'track_id': str(track.get('id') or '')},
|
||||
)
|
||||
|
||||
# Stamp real API quality so the global ranker sees actual
|
||||
# sample_rate/bit_depth. Hi-res only when the track itself is
|
||||
# hires-streamable; otherwise it streams as CD-quality FLAC.
|
||||
if hires_streamable and max_bit_depth >= 24:
|
||||
track_result.set_quality(quality_from_qobuz(max_sample_rate, max_bit_depth))
|
||||
else:
|
||||
track_result.set_quality(quality_from_qobuz(44.1, 16))
|
||||
|
||||
return track_result
|
||||
|
||||
# ===================== Download =====================
|
||||
|
|
@ -1181,7 +1229,7 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
|
||||
try:
|
||||
# Determine quality
|
||||
quality_key = config_manager.get('qobuz.quality', 'lossless')
|
||||
quality_key = quality_tier_for_source('qobuz', default='lossless')
|
||||
quality_info = QOBUZ_QUALITY_MAP.get(quality_key, QOBUZ_QUALITY_MAP['lossless'])
|
||||
|
||||
# Quality fallback chain: hires_max → hires → lossless → mp3
|
||||
|
|
|
|||
0
core/quality/__init__.py
Normal file
0
core/quality/__init__.py
Normal file
103
core/quality/lossless.py
Normal file
103
core/quality/lossless.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
"""Single source of truth for "is this audio lossless?" + the lossy-copy
|
||||
overwrite invariant.
|
||||
|
||||
The "create a lossy copy of lossless tracks" feature lives in two places (the
|
||||
import post-processing path and the Lossy Converter repair job). Both used to
|
||||
hardcode ``.flac``, which is exactly how ALAC/WAV/DSD ended up being quality-
|
||||
profile options but NOT lossy-copy sources (#941). The knowledge of which
|
||||
formats are lossless now lives HERE, derived from the same format names the
|
||||
quality model ranks, so adding a format lights it up in both sites at once and
|
||||
they can never drift.
|
||||
|
||||
Two seams, both pure (no I/O) so they're unit-testable without real files:
|
||||
|
||||
* :func:`is_lossless_format` / :func:`is_lossless_audio_path` — eligibility.
|
||||
``.m4a``/``.mp4`` are ambiguous (ALAC=lossless, AAC=lossy) and can only be told
|
||||
apart by codec, so the path check delegates them to an *injected* ``probe_codec``
|
||||
callable — the file I/O stays at the edge, the decision stays pure.
|
||||
* :func:`lossy_output_would_overwrite_source` — the safety invariant: a lossy copy
|
||||
must NEVER be written over its own source (which becomes possible once ``.m4a``
|
||||
is eligible and the target codec is AAC → same ``.m4a`` path).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from core.quality.source_map import AUDIO_EXTENSIONS, format_from_extension
|
||||
|
||||
# Canonical lossless format set. MUST stay in sync with the lossless tiers in
|
||||
# core.quality.model.tier_score and the frontend RT_LOSSLESS_FORMATS — the
|
||||
# consistency test in tests/quality/test_lossless.py pins the tier agreement.
|
||||
LOSSLESS_FORMATS = frozenset({'flac', 'alac', 'wav', 'dsf'})
|
||||
|
||||
# Container extensions that may hold EITHER lossless (ALAC) or lossy (AAC) audio —
|
||||
# only a codec probe can decide, so they're never lossless on extension alone.
|
||||
_AMBIGUOUS_EXTS = frozenset({'m4a', 'mp4'})
|
||||
|
||||
|
||||
def is_lossless_format(fmt: Any) -> bool:
|
||||
"""True when a unified format name (as returned by ``format_from_extension``)
|
||||
is lossless. ``'aac'`` is False here — an ALAC-in-m4a file reports format
|
||||
``'aac'`` by extension and must be resolved by codec, not by name."""
|
||||
return str(fmt or '').lower() in LOSSLESS_FORMATS
|
||||
|
||||
|
||||
# Extensions worth checking as possibly-lossless (a caller can pre-filter SQL by
|
||||
# these, then confirm each via is_lossless_audio_path). Derived from the format
|
||||
# map so it can't drift: every extension whose format is lossless, plus the
|
||||
# ambiguous ALAC containers. Leading dot included (e.g. '.flac', '.m4a').
|
||||
LOSSLESS_CANDIDATE_EXTENSIONS = frozenset(
|
||||
e for e in AUDIO_EXTENSIONS
|
||||
if is_lossless_format(format_from_extension(e)) or e.lstrip('.') in _AMBIGUOUS_EXTS
|
||||
)
|
||||
|
||||
|
||||
def _ext(path: Any) -> str:
|
||||
return os.path.splitext(str(path or ''))[1].lower().lstrip('.')
|
||||
|
||||
|
||||
def is_lossless_audio_path(
|
||||
path: Any,
|
||||
*,
|
||||
probe_codec: Optional[Callable[[str], Optional[str]]] = None,
|
||||
) -> bool:
|
||||
"""True when the file at ``path`` is lossless.
|
||||
|
||||
Unambiguous extensions (flac/wav/aiff/dsf/dff/alac) are decided by extension.
|
||||
``.m4a``/``.mp4`` are decided by ``probe_codec(path)`` (returns the codec, e.g.
|
||||
``'alac'`` or ``'aac'``) — without a probe they're treated as NOT lossless, so
|
||||
a missing probe can never misclassify an AAC file as lossless.
|
||||
"""
|
||||
ext = _ext(path)
|
||||
if is_lossless_format(format_from_extension(ext)):
|
||||
return True
|
||||
if ext in _AMBIGUOUS_EXTS and probe_codec is not None:
|
||||
try:
|
||||
codec = (probe_codec(str(path)) or '').lower()
|
||||
except Exception:
|
||||
return False
|
||||
return 'alac' in codec
|
||||
return False
|
||||
|
||||
|
||||
def lossy_output_would_overwrite_source(source_path: Any, output_path: Any) -> bool:
|
||||
"""True when the computed lossy-copy output path is the source file itself.
|
||||
|
||||
Safety invariant: the converter must skip (never run ffmpeg with ``-y``) when
|
||||
this is True, or it would destroy the original. Happens when an ``.m4a`` ALAC
|
||||
source is converted with the AAC codec (output is also ``.m4a``)."""
|
||||
if not source_path or not output_path:
|
||||
return False
|
||||
a = os.path.normcase(os.path.normpath(str(source_path)))
|
||||
b = os.path.normcase(os.path.normpath(str(output_path)))
|
||||
return a == b
|
||||
|
||||
|
||||
__all__ = [
|
||||
'LOSSLESS_FORMATS',
|
||||
'is_lossless_format',
|
||||
'is_lossless_audio_path',
|
||||
'lossy_output_would_overwrite_source',
|
||||
]
|
||||
285
core/quality/model.py
Normal file
285
core/quality/model.py
Normal file
|
|
@ -0,0 +1,285 @@
|
|||
"""Source-agnostic audio quality model.
|
||||
|
||||
Every download source maps its result into ``AudioQuality``.
|
||||
The ``QualityTarget`` list in the user's profile defines the
|
||||
priority order (1st choice, 2nd choice, …). ``rank_candidate``
|
||||
scores any ``AudioQuality`` against that list so the same
|
||||
logic drives Soulseek, Tidal, Deezer, torrent — no per-source
|
||||
quality pipelines needed.
|
||||
|
||||
Soulseek attribute type codes (Soulseek protocol spec):
|
||||
0 = bitrate (kbps)
|
||||
1 = duration (seconds)
|
||||
2 = VBR flag
|
||||
4 = sample rate (Hz) — FLAC / WAV only
|
||||
5 = bit depth — FLAC / WAV only
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioQuality:
|
||||
"""Unified audio quality descriptor — source-agnostic."""
|
||||
|
||||
format: str # 'flac', 'mp3', 'aac', 'ogg', 'wav', 'unknown'
|
||||
bitrate: Optional[int] = None # kbps
|
||||
sample_rate: Optional[int] = None # Hz (e.g. 44100, 96000, 192000)
|
||||
bit_depth: Optional[int] = None # bits per sample (16, 24, 32)
|
||||
|
||||
def tier_score(self) -> float:
|
||||
"""Continuous score for ranking within a matched target bucket.
|
||||
Higher = better. Used as a tiebreaker after target-list matching.
|
||||
"""
|
||||
# NOTE: this only orders the *fallback* path (nothing matched a ranked
|
||||
# target) and tie-breaks candidates of the SAME format within one
|
||||
# matched target. Cross-format PRIORITY is decided solely by the user's
|
||||
# ranked-target list (target index), never by these numbers.
|
||||
format_base: dict[str, float] = {
|
||||
'dsf': 102.0, # DSD — 1-bit hi-res lossless, ranks at/above FLAC (#939)
|
||||
'flac': 100.0,
|
||||
'alac': 98.0, # lossless (Apple)
|
||||
'wav': 95.0,
|
||||
'ogg': 70.0,
|
||||
'opus': 65.0,
|
||||
'aac': 60.0,
|
||||
'mp3': 50.0,
|
||||
'wma': 30.0,
|
||||
}
|
||||
base = format_base.get(self.format.lower(), 10.0)
|
||||
|
||||
if self.format.lower() in ('flac', 'wav'):
|
||||
sr = self.sample_rate or 44100
|
||||
bd = self.bit_depth or 16
|
||||
# sample-rate contribution: 44.1 kHz = 0, 192 kHz = +20
|
||||
sr_score = min(sr / 192_000, 1.0) * 20
|
||||
# bit-depth contribution: 16-bit = 0, 24-bit = +10
|
||||
bd_score = max(bd - 16, 0) / 8 * 10
|
||||
return base + sr_score + bd_score
|
||||
else:
|
||||
br = self.bitrate or 0
|
||||
return base + min(br / 320, 1.0) * 10
|
||||
|
||||
def matches_target(self, target: QualityTarget) -> bool:
|
||||
"""True when this quality satisfies every constraint in *target*."""
|
||||
if target.format and target.format.lower() != self.format.lower():
|
||||
return False
|
||||
if target.min_bitrate and (self.bitrate or 0) < target.min_bitrate:
|
||||
return False
|
||||
if target.min_sample_rate:
|
||||
if self.sample_rate is not None:
|
||||
if self.sample_rate < target.min_sample_rate:
|
||||
return False
|
||||
else:
|
||||
# No sample-rate metadata (common on slskd FLAC). Use the kbps
|
||||
# heuristic when a bitrate is present; otherwise we CANNOT
|
||||
# confirm the spec, so fail the strict target rather than
|
||||
# over-claim it — an unknown-spec FLAC must not outrank a known
|
||||
# 16/44 FLAC under a hi-res target (#896 review #4). It falls to
|
||||
# the plain-flac bucket instead.
|
||||
# 16-bit/44.1 kHz ≈ 1411 kbps; 24-bit/96 kHz ≈ 4608 kbps.
|
||||
if self.format.lower() == 'flac' and self.bitrate:
|
||||
required_kbps = _sample_rate_to_min_kbps(target.min_sample_rate, target.bit_depth or 24)
|
||||
if self.bitrate < required_kbps:
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
if target.bit_depth:
|
||||
if self.bit_depth is not None:
|
||||
if self.bit_depth < target.bit_depth:
|
||||
return False
|
||||
else:
|
||||
# No bit-depth metadata. A hi-res (>=24-bit) target needs proof:
|
||||
# use the kbps heuristic if a bitrate is present, else fail
|
||||
# rather than over-claim. The 16-bit baseline still matches an
|
||||
# unknown-spec FLAC (any FLAC is at least CD quality). #896 review #4.
|
||||
if self.format.lower() == 'flac' and target.bit_depth >= 24:
|
||||
if self.bitrate:
|
||||
if self.bitrate < 1450:
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
def label(self) -> str:
|
||||
"""Human-readable label, e.g. 'FLAC 24-bit/192kHz' or 'MP3 320kbps'."""
|
||||
fmt = self.format.upper()
|
||||
if self.format.lower() in ('flac', 'wav'):
|
||||
bd = f"{self.bit_depth}-bit/" if self.bit_depth else ""
|
||||
sr = f"{self.sample_rate // 1000}kHz" if self.sample_rate else ""
|
||||
detail = f" {bd}{sr}".rstrip()
|
||||
return f"{fmt}{detail}" if detail.strip() else fmt
|
||||
else:
|
||||
br = f" {self.bitrate}kbps" if self.bitrate else ""
|
||||
return f"{fmt}{br}"
|
||||
|
||||
@classmethod
|
||||
def from_slskd_file(cls, file_data: dict, extension: str) -> 'AudioQuality':
|
||||
"""Build from a raw slskd API file entry.
|
||||
|
||||
slskd exposes Soulseek protocol file attributes as:
|
||||
``{"attributes": [{"type": 4, "value": 96000}, {"type": 5, "value": 24}, ...]}``
|
||||
"""
|
||||
attrs = {a['type']: a['value'] for a in file_data.get('attributes', [])}
|
||||
return cls(
|
||||
format=extension.lower().lstrip('.'),
|
||||
bitrate=file_data.get('bitRate') or attrs.get(0),
|
||||
sample_rate=attrs.get(4),
|
||||
bit_depth=attrs.get(5),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_tier(cls, fmt: str, bitrate: int, sample_rate: Optional[int] = None, bit_depth: Optional[int] = None) -> 'AudioQuality':
|
||||
"""Build from a hardcoded quality tier (Tidal, Deezer, Qobuz)."""
|
||||
return cls(format=fmt, bitrate=bitrate, sample_rate=sample_rate, bit_depth=bit_depth)
|
||||
|
||||
@classmethod
|
||||
def from_extension_and_bitrate(cls, extension: str, bitrate: Optional[int]) -> 'AudioQuality':
|
||||
"""Minimal constructor when only format + bitrate are known (torrent, YouTube)."""
|
||||
return cls(format=extension.lower().lstrip('.'), bitrate=bitrate)
|
||||
|
||||
|
||||
@dataclass
|
||||
class QualityTarget:
|
||||
"""One ranked entry in the user's quality priority list."""
|
||||
|
||||
label: str = ""
|
||||
format: Optional[str] = None # 'flac', 'mp3', 'aac', …
|
||||
bit_depth: Optional[int] = None # 16, 24
|
||||
min_sample_rate: Optional[int] = None # Hz
|
||||
min_bitrate: Optional[int] = None # kbps (lossy)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {k: v for k, v in self.__dict__.items() if v not in (None, "")}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict) -> 'QualityTarget':
|
||||
known = {f.name for f in cls.__dataclass_fields__.values()} # type: ignore[attr-defined]
|
||||
return cls(**{k: v for k, v in d.items() if k in known})
|
||||
|
||||
|
||||
# ── Default priority list ──────────────────────────────────────────────────────
|
||||
|
||||
DEFAULT_RANKED_TARGETS: List[QualityTarget] = [
|
||||
QualityTarget(label='FLAC 24-bit/192kHz', format='flac', bit_depth=24, min_sample_rate=192_000),
|
||||
QualityTarget(label='FLAC 24-bit/96kHz', format='flac', bit_depth=24, min_sample_rate=96_000),
|
||||
QualityTarget(label='FLAC 24-bit/48kHz', format='flac', bit_depth=24, min_sample_rate=48_000),
|
||||
QualityTarget(label='FLAC 24-bit/44.1kHz',format='flac', bit_depth=24, min_sample_rate=44_100),
|
||||
QualityTarget(label='FLAC 16-bit', format='flac', bit_depth=16),
|
||||
QualityTarget(label='MP3 320kbps', format='mp3', min_bitrate=320),
|
||||
QualityTarget(label='MP3 256kbps', format='mp3', min_bitrate=256),
|
||||
QualityTarget(label='MP3 192kbps', format='mp3', min_bitrate=192),
|
||||
]
|
||||
|
||||
|
||||
# ── Ranking helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
def rank_candidate(aq: AudioQuality, targets: List[QualityTarget]) -> Tuple[int, float]:
|
||||
"""Return *(target_index, tier_score)* for sorting.
|
||||
|
||||
Lower ``target_index`` → higher priority match.
|
||||
Candidates that satisfy no target get ``index = len(targets)``
|
||||
(they sort last but are not discarded — the caller decides that).
|
||||
"""
|
||||
for i, target in enumerate(targets):
|
||||
if aq.matches_target(target):
|
||||
return (i, aq.tier_score())
|
||||
return (len(targets), aq.tier_score())
|
||||
|
||||
|
||||
def filter_and_rank(
|
||||
candidates: list,
|
||||
targets: List[QualityTarget],
|
||||
*,
|
||||
fallback_enabled: bool = True,
|
||||
) -> list:
|
||||
"""Sort *candidates* (any objects with an ``audio_quality`` attribute)
|
||||
by quality priority.
|
||||
|
||||
Returns the subset that matched the *highest-priority* satisfied target,
|
||||
sorted by ``tier_score`` descending within that group.
|
||||
Falls back to all candidates sorted by score when ``fallback_enabled``
|
||||
and nothing matches, or when targets list is empty.
|
||||
"""
|
||||
if not targets:
|
||||
candidates_copy = list(candidates)
|
||||
candidates_copy.sort(key=lambda c: c.audio_quality.tier_score(), reverse=True)
|
||||
return candidates_copy
|
||||
|
||||
scored = [(rank_candidate(c.audio_quality, targets), c) for c in candidates]
|
||||
|
||||
# Best target index that any candidate reached
|
||||
best_idx = min((s[0][0] for s in scored), default=len(targets))
|
||||
|
||||
if best_idx < len(targets):
|
||||
winners = [c for (idx, _), c in scored if idx == best_idx]
|
||||
winners.sort(key=lambda c: c.audio_quality.tier_score(), reverse=True)
|
||||
return winners
|
||||
|
||||
if fallback_enabled:
|
||||
all_sorted = list(candidates)
|
||||
all_sorted.sort(key=lambda c: c.audio_quality.tier_score(), reverse=True)
|
||||
return all_sorted
|
||||
|
||||
return []
|
||||
|
||||
|
||||
# ── Migration helper ───────────────────────────────────────────────────────────
|
||||
|
||||
def v2_qualities_to_ranked_targets(qualities: dict) -> List[dict]:
|
||||
"""Convert old v2 ``qualities`` dict to a ranked-targets list.
|
||||
|
||||
Preserves the user's existing priority order while upgrading to the
|
||||
richer target format.
|
||||
"""
|
||||
_FORMAT_MAP = {
|
||||
'flac': {'format': 'flac', 'bit_depth': None},
|
||||
'mp3_320': {'format': 'mp3', 'min_bitrate': 320},
|
||||
'mp3_256': {'format': 'mp3', 'min_bitrate': 256},
|
||||
'mp3_192': {'format': 'mp3', 'min_bitrate': 192},
|
||||
# AAC (#886): opt-in tier. Match on format alone — Soulseek AAC/.m4a
|
||||
# rarely carries a bitrate attribute, so a min_bitrate gate would
|
||||
# reject every bitrate-less AAC. Priority order (above MP3, below FLAC)
|
||||
# is preserved by the caller's priority sort, not by min_bitrate.
|
||||
'aac': {'format': 'aac'},
|
||||
}
|
||||
enabled = [
|
||||
(cfg.get('priority', 999), name, cfg)
|
||||
for name, cfg in qualities.items()
|
||||
if cfg.get('enabled', False)
|
||||
]
|
||||
enabled.sort()
|
||||
targets = []
|
||||
for _, name, cfg in enabled:
|
||||
base = _FORMAT_MAP.get(name, {}).copy()
|
||||
if not base:
|
||||
continue
|
||||
if name == 'flac':
|
||||
bd = cfg.get('bit_depth', 'any')
|
||||
if bd == '24':
|
||||
base['bit_depth'] = 24
|
||||
base['label'] = 'FLAC 24-bit'
|
||||
elif bd == '16':
|
||||
base['bit_depth'] = 16
|
||||
base['label'] = 'FLAC 16-bit'
|
||||
else:
|
||||
base['label'] = 'FLAC (any)'
|
||||
else:
|
||||
base['label'] = name.upper().replace('_', ' ')
|
||||
targets.append(base)
|
||||
return targets
|
||||
|
||||
|
||||
# ── Internal helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
def _sample_rate_to_min_kbps(sample_rate: int, bit_depth: int) -> int:
|
||||
"""Approximate minimum kbps for a lossless file at the given spec.
|
||||
Used as heuristic when actual sample-rate metadata is absent.
|
||||
"""
|
||||
# kbps = sample_rate * channels * bit_depth / 1000 * compression_ratio
|
||||
# Assume stereo (2 ch) and ~0.6 FLAC compression ratio
|
||||
raw_kbps = sample_rate * 2 * bit_depth / 1000
|
||||
return int(raw_kbps * 0.55) # conservative compressed estimate
|
||||
148
core/quality/selection.py
Normal file
148
core/quality/selection.py
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
"""Quality-aware candidate selection shared by the search engine and the
|
||||
download orchestrator.
|
||||
|
||||
``rank_with_targets`` is the pure core: it ranks candidates against a target
|
||||
list and reports whether any candidate met a *real* target (strict, fallback
|
||||
off). The engine uses that ``satisfied`` flag to decide whether the current
|
||||
source is good enough or it should fall through to the next source in the
|
||||
hybrid chain.
|
||||
|
||||
``rank_for_profile`` is the thin DB-backed wrapper that loads the user's
|
||||
quality profile (with v2->v3 migration) and delegates.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Tuple
|
||||
|
||||
from core.quality.model import (
|
||||
QualityTarget,
|
||||
filter_and_rank,
|
||||
v2_qualities_to_ranked_targets,
|
||||
)
|
||||
|
||||
|
||||
def rank_with_targets(
|
||||
candidates: list,
|
||||
targets: List[QualityTarget],
|
||||
*,
|
||||
fallback_enabled: bool = True,
|
||||
) -> Tuple[list, bool]:
|
||||
"""Rank *candidates* against *targets*.
|
||||
|
||||
Returns ``(ranked, satisfied)`` where ``satisfied`` is True when at least
|
||||
one candidate meets a real target. When no targets are configured the
|
||||
profile imposes no constraint, so any non-empty result counts as
|
||||
satisfied (the first source wins, quality-sorted).
|
||||
"""
|
||||
if not candidates:
|
||||
return [], False
|
||||
|
||||
if not targets:
|
||||
ranked = filter_and_rank(candidates, targets, fallback_enabled=True)
|
||||
return ranked, bool(ranked)
|
||||
|
||||
strict = filter_and_rank(candidates, targets, fallback_enabled=False)
|
||||
if strict:
|
||||
return strict, True
|
||||
|
||||
if fallback_enabled:
|
||||
return filter_and_rank(candidates, targets, fallback_enabled=True), False
|
||||
return [], False
|
||||
|
||||
|
||||
def targets_from_profile(profile: dict) -> Tuple[List[QualityTarget], bool]:
|
||||
"""Convert a quality-profile dict into ``(targets, fallback_enabled)`` with
|
||||
v2->v3 migration applied. The single conversion path shared by the import
|
||||
guard, the download ranker and the library quality scanner."""
|
||||
raw_targets = profile.get('ranked_targets')
|
||||
if not raw_targets and 'qualities' in profile:
|
||||
raw_targets = v2_qualities_to_ranked_targets(profile['qualities'])
|
||||
|
||||
targets = [QualityTarget.from_dict(t) for t in (raw_targets or [])]
|
||||
fallback_enabled = profile.get('fallback_enabled', True)
|
||||
return targets, fallback_enabled
|
||||
|
||||
|
||||
def load_profile_targets() -> Tuple[List[QualityTarget], bool]:
|
||||
"""Load the user's quality profile from the DB and return
|
||||
``(targets, fallback_enabled)`` with v2->v3 migration applied.
|
||||
|
||||
Callers that rank across many sources should load once and reuse via
|
||||
:func:`rank_with_targets` rather than calling :func:`rank_for_profile`
|
||||
per source.
|
||||
"""
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
return targets_from_profile(MusicDatabase().get_quality_profile())
|
||||
|
||||
|
||||
def quality_meets_profile(aq, targets: List[QualityTarget]) -> bool:
|
||||
"""Strict: True iff *aq* satisfies at least one ranked *target*.
|
||||
|
||||
The shared definition of "good enough" for both the import guard and the
|
||||
library scanner — bit depth + sample rate are minimums (see
|
||||
:meth:`AudioQuality.matches_target`). Fallback is NOT consulted here; it's a
|
||||
download-time last-resort concession, not part of what counts as meeting the
|
||||
profile. ``targets`` empty → no constraint (True). ``aq`` None (probe
|
||||
failed) → True, so an unreadable file is never falsely flagged.
|
||||
"""
|
||||
if not targets:
|
||||
return True
|
||||
if aq is None:
|
||||
return True
|
||||
from core.quality.model import rank_candidate
|
||||
|
||||
idx, _ = rank_candidate(aq, targets)
|
||||
return idx < len(targets)
|
||||
|
||||
|
||||
_VALID_SEARCH_MODES = ("priority", "best_quality")
|
||||
|
||||
|
||||
def load_search_mode() -> str:
|
||||
"""Return the download search strategy from the user's quality profile.
|
||||
|
||||
``'priority'`` (default) keeps today's behaviour — the first source in the
|
||||
hybrid chain that meets a quality target wins. ``'best_quality'`` pools
|
||||
candidates across all sources and works them best→worst by actual audio
|
||||
quality. Any missing/unknown value resolves to ``'priority'`` so existing
|
||||
installs are unaffected.
|
||||
"""
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
try:
|
||||
profile = MusicDatabase().get_quality_profile()
|
||||
mode = profile.get("search_mode", "priority")
|
||||
except Exception:
|
||||
return "priority"
|
||||
return mode if mode in _VALID_SEARCH_MODES else "priority"
|
||||
|
||||
|
||||
def load_rank_candidates_by_quality() -> bool:
|
||||
"""Opt-in: order the priority-mode download walk (and thus the
|
||||
version-mismatch force-import pick, which takes the first-tried = best)
|
||||
by ranked-target quality instead of confidence-first.
|
||||
|
||||
Best-quality search mode is always quality-first regardless of this flag;
|
||||
this toggle only affects *priority* mode. Default ``False`` keeps the
|
||||
byte-for-byte old behaviour (confidence/peer-speed first), so existing
|
||||
installs are unaffected unless they opt in. Any missing value or DB error
|
||||
resolves to ``False``.
|
||||
"""
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
try:
|
||||
profile = MusicDatabase().get_quality_profile()
|
||||
return bool(profile.get("rank_candidates_by_quality", False))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def rank_for_profile(candidates: list) -> Tuple[list, bool]:
|
||||
"""Load the user's quality profile and rank *candidates* against it.
|
||||
|
||||
Returns ``(ranked, satisfied)`` — see :func:`rank_with_targets`.
|
||||
"""
|
||||
targets, fallback_enabled = load_profile_targets()
|
||||
return rank_with_targets(candidates, targets, fallback_enabled=fallback_enabled)
|
||||
206
core/quality/source_map.py
Normal file
206
core/quality/source_map.py
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
"""Per-source quality mappers — turn each download source's tier string or
|
||||
API values into a unified :class:`~core.quality.model.AudioQuality`.
|
||||
|
||||
Every streaming source describes quality differently (Tidal/HiFi use tier
|
||||
strings, Qobuz reports real kHz + bit depth, Deezer uses config codes,
|
||||
Amazon mixes real values with HD/UHD tiers). Centralising the knowledge
|
||||
here keeps the per-client code to a single call and keeps the tier tables
|
||||
in one auditable place.
|
||||
|
||||
Each value is a *claim*: the download client populates its ``TrackResult``
|
||||
from it so the global ranker can choose a source, and the post-download
|
||||
quality guard later verifies the real file. Over-claiming is the danger —
|
||||
an unknown tier maps to ``format='unknown'`` rather than pretending to be
|
||||
lossless.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from core.quality.model import AudioQuality
|
||||
from core.quality.selection import load_profile_targets
|
||||
|
||||
|
||||
# ── Extension → format string (source-agnostic) ────────────────────────────
|
||||
#
|
||||
# The single source of truth for mapping a file extension to the unified
|
||||
# AudioQuality ``format``. Every extension-based download source (Soulseek,
|
||||
# torrent/usenet file lists, …) classifies through this, so the ranked-target
|
||||
# system behaves identically across sources and adding a format here lights it
|
||||
# up everywhere at once. Unknown extensions → 'unknown' (never matches a
|
||||
# target, so it only ever comes through via the fallback toggle).
|
||||
#
|
||||
# AIFF/AIF are uncompressed PCM like WAV → the same 'wav' tier. ``.m4a``
|
||||
# defaults to 'aac'; an ALAC-in-m4a file can't be told apart by extension
|
||||
# alone, so probe_audio_quality corrects it from the real codec post-download.
|
||||
_EXTENSION_FORMAT_MAP = {
|
||||
'flac': 'flac',
|
||||
'alac': 'alac',
|
||||
'wav': 'wav', 'wave': 'wav',
|
||||
'aiff': 'wav', 'aif': 'wav', 'aifc': 'wav',
|
||||
'mp3': 'mp3',
|
||||
'm4a': 'aac', 'mp4': 'aac', 'aac': 'aac',
|
||||
'ogg': 'ogg', 'oga': 'ogg',
|
||||
'opus': 'opus',
|
||||
'wma': 'wma',
|
||||
# DSD (DSD Stream File / DSDIFF) — 1-bit hi-res lossless (e.g. DSD64 ≈ 11 Mbps).
|
||||
# Both container types map to the single 'dsf' tier (#939).
|
||||
'dsf': 'dsf', 'dff': 'dsf',
|
||||
}
|
||||
|
||||
# Audio extensions worth probing/classifying at all — derived from the map so
|
||||
# the allow-list and the classifier never drift apart.
|
||||
AUDIO_EXTENSIONS = {f'.{e}' for e in _EXTENSION_FORMAT_MAP}
|
||||
|
||||
|
||||
def format_from_extension(ext: str) -> str:
|
||||
"""Map a file extension (with or without leading dot) to the unified
|
||||
AudioQuality format string. Unknown → 'unknown'."""
|
||||
return _EXTENSION_FORMAT_MAP.get(str(ext or '').lower().lstrip('.'), 'unknown')
|
||||
|
||||
|
||||
# ── Tidal / HiFi (Monochrome is Tidal-backed) ──────────────────────────────
|
||||
#
|
||||
# Tidal exposes UPPER_SNAKE tier strings (``HI_RES_LOSSLESS``); HiFi's config
|
||||
# uses lowercase keys (``hires``/``lossless``). We normalise both into the
|
||||
# same lookup so one mapper serves both sources.
|
||||
|
||||
_TIDAL_HIRES = AudioQuality(format='flac', sample_rate=96000, bit_depth=24)
|
||||
_TIDAL_LOSSLESS = AudioQuality(format='flac', sample_rate=44100, bit_depth=16)
|
||||
_TIDAL_HIGH = AudioQuality(format='aac', bitrate=320)
|
||||
_TIDAL_LOW = AudioQuality(format='aac', bitrate=96)
|
||||
|
||||
TIDAL_TIER_MAP = {
|
||||
'HI_RES_LOSSLESS': _TIDAL_HIRES,
|
||||
'HI_RES': _TIDAL_HIRES,
|
||||
'HIRES': _TIDAL_HIRES,
|
||||
'LOSSLESS': _TIDAL_LOSSLESS,
|
||||
'HIGH': _TIDAL_HIGH,
|
||||
'LOW': _TIDAL_LOW,
|
||||
}
|
||||
|
||||
|
||||
def quality_from_tidal_tier(tier: str) -> AudioQuality:
|
||||
"""Map a Tidal/HiFi quality tier string to an AudioQuality.
|
||||
|
||||
Case-insensitive; accepts both ``HI_RES`` and ``hires`` spellings.
|
||||
Unrecognised tiers map to ``format='unknown'`` so they never
|
||||
over-claim lossless quality.
|
||||
"""
|
||||
key = (tier or '').strip().upper()
|
||||
return TIDAL_TIER_MAP.get(key, AudioQuality(format='unknown'))
|
||||
|
||||
|
||||
# ── Qobuz (real API values) ────────────────────────────────────────────────
|
||||
|
||||
def quality_from_qobuz(sampling_rate_khz: float, bit_depth: int) -> AudioQuality:
|
||||
"""Qobuz reports ``maximum_sampling_rate`` in kHz (e.g. 44.1, 96, 192)
|
||||
and ``maximum_bit_depth``. These are real values from the API.
|
||||
"""
|
||||
sample_rate = int(round(sampling_rate_khz * 1000)) if sampling_rate_khz else None
|
||||
return AudioQuality(format='flac', sample_rate=sample_rate, bit_depth=bit_depth)
|
||||
|
||||
|
||||
# ── Deezer (config code) ───────────────────────────────────────────────────
|
||||
|
||||
DEEZER_CODE_MAP = {
|
||||
'flac': AudioQuality(format='flac', sample_rate=44100, bit_depth=16),
|
||||
'mp3_320': AudioQuality(format='mp3', bitrate=320),
|
||||
'mp3_128': AudioQuality(format='mp3', bitrate=128),
|
||||
}
|
||||
|
||||
|
||||
def quality_from_deezer(code: str) -> AudioQuality:
|
||||
"""Map a Deezer download quality code to AudioQuality.
|
||||
|
||||
Deezer FLAC is always CD-quality (16-bit/44.1 kHz).
|
||||
"""
|
||||
return DEEZER_CODE_MAP.get((code or '').lower(), AudioQuality(format='unknown'))
|
||||
|
||||
|
||||
# ── Amazon Music (real sampleRate preferred, HD/UHD tier fallback) ─────────
|
||||
|
||||
_AMAZON_TIER_MAP = {
|
||||
'UHD': AudioQuality(format='flac', sample_rate=96000, bit_depth=24),
|
||||
'HD': AudioQuality(format='flac', sample_rate=44100, bit_depth=16),
|
||||
}
|
||||
|
||||
|
||||
def quality_from_amazon(
|
||||
tier: str,
|
||||
sample_rate: Optional[int] = None,
|
||||
bit_depth: Optional[int] = None,
|
||||
) -> AudioQuality:
|
||||
"""Amazon Music is FLAC; prefer the real ``sampleRate``/``bitDepth`` from
|
||||
the stream info when present, otherwise fall back to the HD/UHD tier.
|
||||
"""
|
||||
base = _AMAZON_TIER_MAP.get((tier or '').strip().upper(), AudioQuality(format='flac'))
|
||||
return AudioQuality(
|
||||
format='flac',
|
||||
sample_rate=sample_rate if sample_rate is not None else base.sample_rate,
|
||||
bit_depth=bit_depth if bit_depth is not None else base.bit_depth,
|
||||
)
|
||||
|
||||
|
||||
# ── Profile-driven download tier (replaces per-source quality settings) ─────
|
||||
#
|
||||
# Each source's selectable download tiers, ordered best → worst, with the
|
||||
# AudioQuality the tier delivers. ``quality_tier_for_source`` walks these to
|
||||
# request the LOWEST tier that satisfies the user's top global target — so the
|
||||
# global quality profile, not a per-source dropdown, decides what each source
|
||||
# fetches.
|
||||
|
||||
_SOURCE_TIER_LADDERS: dict[str, list[tuple[str, AudioQuality]]] = {
|
||||
'tidal': [
|
||||
('hires', AudioQuality('flac', sample_rate=96000, bit_depth=24)),
|
||||
('lossless', AudioQuality('flac', sample_rate=44100, bit_depth=16)),
|
||||
('high', AudioQuality('aac', bitrate=320)),
|
||||
('low', AudioQuality('aac', bitrate=96)),
|
||||
],
|
||||
'hifi': [
|
||||
('hires', AudioQuality('flac', sample_rate=96000, bit_depth=24)),
|
||||
('lossless', AudioQuality('flac', sample_rate=44100, bit_depth=16)),
|
||||
('high', AudioQuality('aac', bitrate=320)),
|
||||
('low', AudioQuality('aac', bitrate=96)),
|
||||
],
|
||||
'qobuz': [
|
||||
('hires_max', AudioQuality('flac', sample_rate=192000, bit_depth=24)),
|
||||
('hires', AudioQuality('flac', sample_rate=96000, bit_depth=24)),
|
||||
('lossless', AudioQuality('flac', sample_rate=44100, bit_depth=16)),
|
||||
('mp3', AudioQuality('mp3', bitrate=320)),
|
||||
],
|
||||
'deezer': [
|
||||
('flac', AudioQuality('flac', sample_rate=44100, bit_depth=16)),
|
||||
('mp3_320', AudioQuality('mp3', bitrate=320)),
|
||||
('mp3_128', AudioQuality('mp3', bitrate=128)),
|
||||
],
|
||||
'amazon': [
|
||||
('flac', AudioQuality('flac', sample_rate=48000, bit_depth=24)),
|
||||
('opus', AudioQuality('aac', bitrate=320)),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def quality_tier_for_source(source_name: str, *, default: Optional[str] = None) -> Optional[str]:
|
||||
"""Return the source tier key to request, derived from the global profile.
|
||||
|
||||
Picks the lowest tier in the source's ladder that satisfies the user's
|
||||
top (most-preferred) target — respecting the quality ceiling and saving
|
||||
bandwidth. Falls back to the source's max tier when none can satisfy it
|
||||
(best effort), or to the source's max when no targets are configured.
|
||||
Returns *default* for an unknown source.
|
||||
"""
|
||||
ladder = _SOURCE_TIER_LADDERS.get(source_name)
|
||||
if not ladder:
|
||||
return default
|
||||
|
||||
targets, _ = load_profile_targets()
|
||||
if not targets:
|
||||
return ladder[0][0]
|
||||
|
||||
top = targets[0]
|
||||
for key, aq in reversed(ladder): # low → high
|
||||
if aq.matches_target(top):
|
||||
return key
|
||||
return ladder[0][0] # best effort: max tier
|
||||
|
|
@ -70,6 +70,9 @@ class QueueItem:
|
|||
# 'tags' = read each file's embedded tags as the source
|
||||
# of truth (issue #592). Zero API calls.
|
||||
metadata_source: str = 'api'
|
||||
# Rename-only mode (#875): move files to the current naming scheme WITHOUT the
|
||||
# copy + post-processing (re-tag / quality / AcoustID) the full flow runs.
|
||||
rename_only: bool = False
|
||||
status: str = 'queued' # queued | running | done | failed | cancelled
|
||||
started_at: Optional[float] = None
|
||||
finished_at: Optional[float] = None
|
||||
|
|
@ -96,6 +99,7 @@ class QueueItem:
|
|||
'artist_name': self.artist_name,
|
||||
'source': self.source,
|
||||
'metadata_source': self.metadata_source,
|
||||
'rename_only': self.rename_only,
|
||||
'enqueued_at': self.enqueued_at,
|
||||
'started_at': self.started_at,
|
||||
'finished_at': self.finished_at,
|
||||
|
|
@ -161,6 +165,7 @@ class ReorganizeQueue:
|
|||
artist_name: str,
|
||||
source: Optional[str] = None,
|
||||
metadata_source: str = 'api',
|
||||
rename_only: bool = False,
|
||||
) -> dict:
|
||||
"""Add an album to the queue. Returns a result dict:
|
||||
|
||||
|
|
@ -190,6 +195,7 @@ class ReorganizeQueue:
|
|||
source=source,
|
||||
enqueued_at=time.time(),
|
||||
metadata_source=metadata_source or 'api',
|
||||
rename_only=bool(rename_only),
|
||||
)
|
||||
self._items.append(item)
|
||||
position = sum(1 for i in self._items if i.status == 'queued')
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ def build_runner(
|
|||
is_shutting_down_fn: Callable[[], bool],
|
||||
get_download_path: Callable[[], str],
|
||||
get_transfer_path: Callable[[], str],
|
||||
build_final_path_fn: Optional[Callable] = None,
|
||||
) -> Callable[[object], dict]:
|
||||
"""Return the closure the queue worker invokes per item.
|
||||
|
||||
|
|
@ -59,7 +60,7 @@ def build_runner(
|
|||
A callable ``runner(item)`` suitable for
|
||||
:meth:`core.reorganize_queue.ReorganizeQueue.set_runner`.
|
||||
"""
|
||||
from core.library_reorganize import reorganize_album
|
||||
from core.library_reorganize import reorganize_album, reorganize_album_rename_only
|
||||
from core.reorganize_queue import get_queue
|
||||
|
||||
def _update_track_path(track_id, new_path):
|
||||
|
|
@ -80,17 +81,6 @@ def build_runner(
|
|||
# server restart.
|
||||
download_dir = get_download_path()
|
||||
transfer_dir = get_transfer_path()
|
||||
staging_root = os.path.join(download_dir, 'ssync_staging')
|
||||
try:
|
||||
os.makedirs(staging_root, exist_ok=True)
|
||||
except OSError as mk_err:
|
||||
logger.error(f"[Reorganize] Cannot create staging dir {staging_root}: {mk_err}")
|
||||
return {
|
||||
'status': 'setup_failed',
|
||||
'source': None,
|
||||
'total': 0, 'moved': 0, 'skipped': 0, 'failed': 0,
|
||||
'errors': [{'error': f'Could not create staging dir: {mk_err}'}],
|
||||
}
|
||||
|
||||
def _cleanup_empty(src_dir):
|
||||
try:
|
||||
|
|
@ -105,6 +95,42 @@ def build_runner(
|
|||
# Progress fan-out failures must never break a run.
|
||||
logger.debug("reorganize progress fan-out: %s", e)
|
||||
|
||||
# Rename-only mode (#875): just move files to the current scheme — no staging,
|
||||
# no copy, no post-processing. Falls through to the full pipeline otherwise.
|
||||
if getattr(item, 'rename_only', False):
|
||||
if build_final_path_fn is None:
|
||||
return {
|
||||
'status': 'setup_failed', 'source': None,
|
||||
'total': 0, 'moved': 0, 'skipped': 0, 'failed': 0,
|
||||
'errors': [{'error': 'Rename-only mode unavailable (no path builder)'}],
|
||||
}
|
||||
return reorganize_album_rename_only(
|
||||
album_id=item.album_id,
|
||||
db=get_database(),
|
||||
transfer_dir=transfer_dir,
|
||||
resolve_file_path_fn=resolve_file_path_fn,
|
||||
build_final_path_fn=build_final_path_fn,
|
||||
update_track_path_fn=_update_track_path,
|
||||
cleanup_empty_dir_fn=_cleanup_empty,
|
||||
on_progress=_on_progress,
|
||||
primary_source=item.source,
|
||||
strict_source=bool(item.source),
|
||||
metadata_source=getattr(item, 'metadata_source', 'api') or 'api',
|
||||
stop_check=is_shutting_down_fn,
|
||||
)
|
||||
|
||||
staging_root = os.path.join(download_dir, 'ssync_staging')
|
||||
try:
|
||||
os.makedirs(staging_root, exist_ok=True)
|
||||
except OSError as mk_err:
|
||||
logger.error(f"[Reorganize] Cannot create staging dir {staging_root}: {mk_err}")
|
||||
return {
|
||||
'status': 'setup_failed',
|
||||
'source': None,
|
||||
'total': 0, 'moved': 0, 'skipped': 0, 'failed': 0,
|
||||
'errors': [{'error': f'Could not create staging dir: {mk_err}'}],
|
||||
}
|
||||
|
||||
return reorganize_album(
|
||||
album_id=item.album_id,
|
||||
db=get_database(),
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ _JOB_MODULES = [
|
|||
'core.repair_jobs.metadata_gap_filler',
|
||||
'core.repair_jobs.album_completeness',
|
||||
'core.repair_jobs.fake_lossless_detector',
|
||||
'core.repair_jobs.quality_upgrade_scanner',
|
||||
'core.repair_jobs.library_reorganize',
|
||||
'core.repair_jobs.mbid_mismatch_detector',
|
||||
'core.repair_jobs.single_album_dedup',
|
||||
|
|
@ -51,6 +52,7 @@ _JOB_MODULES = [
|
|||
'core.repair_jobs.canonical_version_resolve',
|
||||
'core.repair_jobs.library_retag',
|
||||
'core.repair_jobs.quality_upgrade',
|
||||
'core.repair_jobs.short_preview_track',
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -389,14 +389,43 @@ class AcoustIDScannerJob(RepairJob):
|
|||
cur.execute(
|
||||
"UPDATE tracks SET verification_status = ? WHERE id = ?",
|
||||
(status, track_id))
|
||||
matched = 0
|
||||
exp = expected or {}
|
||||
# Find the canonical history row for this file. The stored path is frozen at
|
||||
# import time while the file has since moved (media-server import / reorganize),
|
||||
# so an exact-path match alone misses it — then the status never lands and a
|
||||
# duplicate row gets inserted every scan (#934). Match exact path first, then
|
||||
# filename guarded by title, and HEAL the row's path so future scans match cleanly.
|
||||
from core.downloads.history_match import pick_history_row, like_filename_filter
|
||||
current = fpath or db_path
|
||||
basename = os.path.basename(current) if current else ''
|
||||
clauses, params = [], []
|
||||
for p in {p for p in (fpath, db_path) if p}:
|
||||
clauses.append("file_path = ?")
|
||||
params.append(p)
|
||||
if basename:
|
||||
clauses.append("file_path LIKE ? ESCAPE '\\'")
|
||||
params.append(like_filename_filter(basename))
|
||||
row_id = None
|
||||
if clauses:
|
||||
cur.execute(
|
||||
"UPDATE library_history SET verification_status = ? WHERE file_path = ?",
|
||||
(status, p))
|
||||
matched += max(getattr(cur, 'rowcount', 0) or 0, 0)
|
||||
if status == 'unverified' and matched == 0:
|
||||
exp = expected or {}
|
||||
"SELECT id, file_path, title, download_source FROM library_history WHERE "
|
||||
+ " OR ".join(clauses),
|
||||
params)
|
||||
row_id = pick_history_row(
|
||||
cur.fetchall(),
|
||||
current_paths=(fpath, db_path),
|
||||
basename=basename, title=exp.get('title') or '')
|
||||
if row_id is not None:
|
||||
cur.execute(
|
||||
"UPDATE library_history SET verification_status = ?, file_path = ? WHERE id = ?",
|
||||
(status, current, row_id))
|
||||
# Drop synthetic scan-created duplicates for this exact file (the #934
|
||||
# leftovers). Exact path → collision-free; never touches a real download row.
|
||||
cur.execute(
|
||||
"DELETE FROM library_history WHERE id != ? AND download_source = 'acoustid_scan' "
|
||||
"AND file_path = ?",
|
||||
(row_id, current))
|
||||
elif status == 'unverified':
|
||||
cur.execute(
|
||||
"""INSERT INTO library_history
|
||||
(event_type, title, artist_name, album_name, file_path,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue