Compare commits
296 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3ab9301a18 | ||
|
|
4b8ddad9ff | ||
|
|
68d52a1b3f | ||
|
|
81bd85d639 | ||
|
|
f746fb4fc4 | ||
|
|
8f1ffa7632 | ||
|
|
a8520bc9b2 | ||
|
|
79648a4f5f | ||
|
|
9fc3628062 | ||
|
|
9b18e99419 | ||
|
|
b05b641521 | ||
|
|
efefdd64ff | ||
|
|
f3672c7ab4 | ||
|
|
6f451a34e1 | ||
|
|
633aa82b22 | ||
|
|
bd6db37624 | ||
|
|
8d549eb4fa | ||
|
|
1e68e339ca | ||
|
|
84d6208bc9 | ||
|
|
5d5aac9be3 | ||
|
|
60e7193539 | ||
|
|
9bb73c4cb4 | ||
|
|
050ce79d51 | ||
|
|
4743dfd644 | ||
|
|
91eaaeabb2 | ||
|
|
37c8b06a27 | ||
|
|
6e1a377f37 | ||
|
|
614deba0af | ||
|
|
465a77f01d | ||
|
|
0db72bf48d | ||
|
|
37c2b9b569 | ||
|
|
ede62824ad | ||
|
|
c593a17ac2 | ||
|
|
62aa2bef2d | ||
|
|
5df873655f | ||
|
|
92c101e300 | ||
|
|
ad715dddda | ||
|
|
34adb6fb32 | ||
|
|
79383df6d8 | ||
|
|
a62d2d4310 | ||
|
|
3207310448 | ||
|
|
6a388c43fc | ||
|
|
33fe92a525 | ||
|
|
512ec227be | ||
|
|
b07359cdb5 | ||
|
|
caee0fc3e2 | ||
|
|
969700674c | ||
|
|
8abf470018 | ||
|
|
551df0c3ca | ||
|
|
1382cb6117 | ||
|
|
8149f35fae | ||
|
|
1da677ee2d | ||
|
|
b43e44219a | ||
|
|
0276aa8764 | ||
|
|
60dee1b4d8 | ||
|
|
2fb142dded | ||
|
|
e2317de0a4 | ||
|
|
70dba77711 | ||
|
|
a1b0e787bc | ||
|
|
d4e2dccd73 | ||
|
|
d871899451 | ||
|
|
b72febbf1c | ||
|
|
8aa8fcb94a | ||
|
|
4af7600fd5 | ||
|
|
c96135ee60 | ||
|
|
b62d9b5b08 | ||
|
|
df0b4d3595 | ||
|
|
579617f861 | ||
|
|
bcf99d76d3 | ||
|
|
77e3673c9c | ||
|
|
d62afbb2df | ||
|
|
36986e4668 | ||
|
|
d72f6396f0 | ||
|
|
2b8f6f8611 | ||
|
|
b543db147a | ||
|
|
d30273985f | ||
|
|
4487a9d2dc | ||
|
|
d84fba14ba | ||
|
|
50c17ec9f7 | ||
|
|
88b68b8073 | ||
|
|
80d84c6de3 | ||
|
|
60b0022122 | ||
|
|
b683c7fb50 | ||
|
|
4c53a8f3a2 | ||
|
|
0be1952222 | ||
|
|
27ea2ee735 | ||
|
|
034a19b24f | ||
|
|
bd3c5860f6 | ||
|
|
116edeb477 | ||
|
|
319b6483a2 | ||
|
|
b85e9b40d9 | ||
|
|
b4583f23be | ||
|
|
f383a62c4f | ||
|
|
de9e78cf95 | ||
|
|
5a54ffe14a | ||
|
|
6802399805 | ||
|
|
240dce0c1b | ||
|
|
c6caaaa599 | ||
|
|
9faaf5c50c | ||
|
|
bce6a91aa2 | ||
|
|
28a539a840 | ||
|
|
027cf74d47 | ||
|
|
a42cf3b865 | ||
|
|
3a92571c71 | ||
|
|
866c8a6400 | ||
|
|
eddaea2f93 | ||
|
|
b1f061a2a8 | ||
|
|
7e2d2db08d | ||
|
|
e96d62432f | ||
|
|
3a95fc45a4 | ||
|
|
31637e0096 | ||
|
|
18f390b879 | ||
|
|
85d5904846 | ||
|
|
a6364ac283 | ||
|
|
8a6b02b3fd | ||
|
|
c8accf4e28 | ||
|
|
580d9eb0f5 | ||
|
|
92025f5fb3 | ||
|
|
c033656fdf | ||
|
|
30ff0bde49 | ||
|
|
086d153d77 | ||
|
|
3c33e31985 | ||
|
|
dc813d67c1 | ||
|
|
9847d6f0a9 | ||
|
|
602b035bad | ||
|
|
e3915b63e6 | ||
|
|
528aedbdfe | ||
|
|
e29cc641cb | ||
|
|
7a8b66fd2e | ||
|
|
2668980872 | ||
|
|
ab05508acc | ||
|
|
05b36704c3 | ||
|
|
88ff47e115 | ||
|
|
9cb5c4d40d | ||
|
|
9c91ba29bf | ||
|
|
71aa3397bf | ||
|
|
ad657f02a8 | ||
|
|
729a06c6d7 | ||
|
|
ed0a2079cf | ||
|
|
b4dde43b45 | ||
|
|
f010fbc487 | ||
|
|
79101e1847 | ||
|
|
5b7f99c30b | ||
|
|
d0966ec262 | ||
|
|
5cad2c02b0 | ||
|
|
e148f859e7 | ||
|
|
81a8b57dba | ||
|
|
df4ef99389 | ||
|
|
4d287c9699 | ||
|
|
551d12dba3 | ||
|
|
124e8bb21c | ||
|
|
774d0f6c65 | ||
|
|
49d3c77808 | ||
|
|
bac5da9177 | ||
|
|
8afbfbfeab | ||
|
|
606d1f951d | ||
|
|
17137cea5b | ||
|
|
2ae9ad3c6f | ||
|
|
d310b090e2 | ||
|
|
13bfdaeda2 | ||
|
|
8934c6418c | ||
|
|
5d5ed486c4 | ||
|
|
ecd2500c39 | ||
|
|
462a722cce | ||
|
|
0f2d6ddb63 | ||
|
|
4c47c01076 | ||
|
|
1005c7e306 | ||
|
|
ff12d8bbf2 | ||
|
|
94637cbe6f | ||
|
|
1f14fb4d5e | ||
|
|
ef0b68a973 | ||
|
|
e4ba27d8b3 | ||
|
|
bdbeff89d6 | ||
|
|
5f5bf4b24e | ||
|
|
83f21b248d | ||
|
|
8f66432592 | ||
|
|
1770cae04f | ||
|
|
842bc0ab34 | ||
|
|
55f7176c34 | ||
|
|
ec99d686cc | ||
|
|
d647fc8ad1 | ||
|
|
9d091207f6 | ||
|
|
03926bd6e2 | ||
|
|
c21031b9bc | ||
|
|
9ad5188610 | ||
|
|
d4e80fdaa0 | ||
|
|
2b17ed8451 | ||
|
|
cce7df4f3d | ||
|
|
600a744f7f | ||
|
|
2934903874 | ||
|
|
e32b4ec727 | ||
|
|
e8cc7ca2c8 | ||
|
|
16cb29c9ee | ||
|
|
15fa64248c | ||
|
|
10a1e1337f | ||
|
|
9d16abf952 | ||
|
|
fc2c38ad97 | ||
|
|
9f5bc0de89 | ||
|
|
e301877e64 | ||
|
|
65f73fae92 | ||
|
|
69cb51cc13 | ||
|
|
f2f4f8ccee | ||
|
|
b42ce3e0ca | ||
|
|
674b80972a | ||
|
|
d169818043 | ||
|
|
7186d24120 | ||
|
|
63374b32f1 | ||
|
|
dcd68e3631 | ||
|
|
cefddd73b7 | ||
|
|
ab8f82af2e | ||
|
|
5f1ec9ed7e | ||
|
|
c06ef6bb34 | ||
|
|
5ca88d96c1 | ||
|
|
d430dd8b71 | ||
|
|
42ff13d517 | ||
|
|
c6b5cd9763 | ||
|
|
fb9d88ea6a | ||
|
|
01c51a3c0e | ||
|
|
df6815c2cc | ||
|
|
b73389adea | ||
|
|
4b58d8079e | ||
|
|
b5b71df3fa | ||
|
|
9454970a83 | ||
|
|
49592f898c | ||
|
|
b95a8f539e | ||
|
|
89018bb6b3 | ||
|
|
203142c4a9 | ||
|
|
c11a742e58 | ||
|
|
1ad80d77a6 | ||
|
|
6e622d30f1 | ||
|
|
0df1d55ed6 | ||
|
|
41f5b29c27 | ||
|
|
ad0eda3575 | ||
|
|
2a7259b296 | ||
|
|
458658de86 | ||
|
|
9aaafaf341 | ||
|
|
15ea87a154 | ||
|
|
1c93a5640d | ||
|
|
3aaf19a357 | ||
|
|
d58da7a3db | ||
|
|
7159b3dc35 | ||
|
|
3496bb1800 | ||
|
|
1782e36c78 | ||
|
|
cf2b4d7d38 | ||
|
|
b6cb244cbd | ||
|
|
4bce1932ba | ||
|
|
1249160ff5 | ||
|
|
7bbd069147 | ||
|
|
75f3b0aa71 | ||
|
|
f9dfbd8fd5 | ||
|
|
bfc207f21a | ||
|
|
b761229a00 | ||
|
|
b928f4df43 | ||
|
|
0f7e15363b | ||
|
|
526ed227c7 | ||
|
|
bb1a1222f8 | ||
|
|
1734a1b3c4 | ||
|
|
16673f4559 | ||
|
|
0e14ff03ee | ||
|
|
8bb749de9c | ||
|
|
973d28f61d | ||
|
|
5b3061ee2e | ||
|
|
4cb1937810 | ||
|
|
59858b033b | ||
|
|
bb632f6564 | ||
|
|
e82f3ab04f | ||
|
|
cf8f562e61 | ||
|
|
8a22b3f8ba | ||
|
|
ff061324ba | ||
|
|
9af31c3706 | ||
|
|
446465a833 | ||
|
|
501868d9c9 | ||
|
|
d2abec4a92 | ||
|
|
55f3dd427c | ||
|
|
949070ce73 | ||
|
|
f55a4fcf72 | ||
|
|
e594ac9799 | ||
|
|
d484046523 | ||
|
|
31d85e59c9 | ||
|
|
11d2fa9ad6 | ||
|
|
62d5821d26 | ||
|
|
975cf4cf3d | ||
|
|
b6be680d23 | ||
|
|
7421839da5 | ||
|
|
c32fe219fe | ||
|
|
fe78a3cdc3 | ||
|
|
6046a814cb | ||
|
|
d717f06afe | ||
|
|
95a7b51966 | ||
|
|
e0c55342bc | ||
|
|
71f7c4a5e1 | ||
|
|
310a5fe1bd | ||
|
|
12341f006b | ||
|
|
97242cbd8d | ||
|
|
4cc2401332 | ||
|
|
2c91af0062 |
261 changed files with 25417 additions and 3430 deletions
4
.github/workflows/docker-publish.yml
vendored
4
.github/workflows/docker-publish.yml
vendored
|
|
@ -9,9 +9,9 @@ on:
|
|||
workflow_dispatch:
|
||||
inputs:
|
||||
version_tag:
|
||||
description: 'Version tag (e.g. 2.7.3)'
|
||||
description: 'Version tag (e.g. 2.8.2)'
|
||||
required: true
|
||||
default: '2.7.3'
|
||||
default: '2.8.2'
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
|
|
|
|||
10
.gitignore
vendored
10
.gitignore
vendored
|
|
@ -12,10 +12,12 @@ __pycache__/
|
|||
|
||||
# User-specific files (auto-created by the app if missing)
|
||||
config/config.json
|
||||
database/music_library.db
|
||||
database/music_library.db-shm
|
||||
database/music_library.db-wal
|
||||
database/music_library.db.backup_*
|
||||
config/youtube_cookies.txt
|
||||
# All app databases are live user data — never commit (music_library, video_library, …)
|
||||
database/*.db
|
||||
database/*.db-shm
|
||||
database/*.db-wal
|
||||
database/*.db.backup_*
|
||||
database/api_call_history.json
|
||||
storage/image_cache/
|
||||
logs/*.log
|
||||
|
|
|
|||
74
DISCOVER_BEST_IN_CLASS_PLAN.md
Normal file
74
DISCOVER_BEST_IN_CLASS_PLAN.md
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
# discover page — best in class plan (#913 + full generator audit)
|
||||
|
||||
morning notes. did the work overnight. tl;dr at top, details below, all of it `break nothing` + tested.
|
||||
|
||||
## what i shipped tonight (done, tested, safe)
|
||||
|
||||
### 1. listening recommendations (#913) — went from BROKEN to best-in-class
|
||||
|
||||
the feature was silently producing **zero** recs on real data. dug in and found three stacked bugs in the generation:
|
||||
|
||||
- **wrong id key (the killer).** `similar_artists.source_artist_id` is a *source* id (spotify/itunes/deezer), but the scanner built its id→name map from `artists.id` (the internal row id). so every edge resolved to nothing → 0 recs. proved it on your live db: internal-id join = 0 rows, spotify-id join = 71,636 rows.
|
||||
- **consensus could never fire.** it fed the ranker `get_top_similar_artists`, which does `GROUP BY similar_artist_name` + `MAX(source_artist_id)` — collapsing every similar artist down to a *single* seed. the whole point of the ranker is "artist X is similar to 3 of your seeds = strong signal," and that signal was being flattened away before it ever reached the ranker.
|
||||
- **similarity strength thrown away.** each edge stores a 1-10 closeness rank; it was ignored (everything weighted equally).
|
||||
|
||||
the fix (all in the pure, tested core + thin scan wiring):
|
||||
- build id→name from the **source-id columns**, query the **raw per-seed edges** (consensus preserved), and thread **similarity_rank** into the score so a seed's closest matches count for more.
|
||||
- **recency-weighted seeds**: `weight = lifetime_plays + 1.5 × recent_30d_plays`. picks now track what you're into *now*, not just all-time totals.
|
||||
|
||||
result on your actual library (simulated through the real code path): **40 recommendations, 13 with multi-seed consensus, all 40 with cached art.** top picks: Arcangel (Bad Bunny + Ozuna + J Balvin), Melanie Martinez (Ariana + Billie), Maluma, De La Ghetto — all coherent, all explainable.
|
||||
|
||||
### 2. its own row on the discover page
|
||||
|
||||
new row **"Based On Your Listening"** — play-weighted, consensus-ranked artist cards with a **"Because you listen to X, Y"** line. sits right above the library-driven "Recommended For You" row. purely additive: new endpoint `/api/discover/listening-recommendations`, new loader, hides itself when empty.
|
||||
|
||||
**you need to run one watchlist scan** for the row to populate (the data regenerates during the scan — i did NOT touch your live db). before that scan the row just stays hidden; after it, it fills in.
|
||||
|
||||
> note: this is deliberately different from the existing "Recommended For You" row. that one is driven by your *whole library / watchlist*. this one is driven by your *actual listening intensity* — the ~30 artists you really play, not the thousands you happen to own.
|
||||
|
||||
### 3. Fresh Tape "only 5-10 tracks" — fixed
|
||||
|
||||
root cause: `get_discovery_recent_albums` orders `release_date DESC`, so announced-but-unreleased albums sort to the *top* and ate the 50-album budget. the scanner skipped them *after* the budget was already spent → only a handful of released albums left → 5-10 tracks. fixed by fetching a generous budget (300) **and** excluding next-year albums at the query, so released albums fill the budget. the precise same-year `is_future_release` skip stays as a second guard. downstream caps (6/artist, top 75, take 50) unchanged.
|
||||
|
||||
**tests:** 25 pure-core cases (consensus/similarity/recency) + 2 Fresh Tape regression tests, all green. full discovery suite (255) green. nothing else touched.
|
||||
|
||||
---
|
||||
|
||||
## best-in-class roadmap for listening recs (next phases — your call)
|
||||
|
||||
these are the levers to take it further. ordered by value-to-risk. none are required; tonight's work stands on its own.
|
||||
|
||||
| phase | what | value | risk | notes |
|
||||
|---|---|---|---|---|
|
||||
| **3** | **playable track row** ✅ DONE | high | low-med | shipped: "🎧 Your Listening Mix" row — a track playlist (play/queue/download/sync) right under the artist row. stored as full render-ready dicts (not pool-hydrated, so it can't shrink on pool rotation like Fresh Tape does). |
|
||||
| **4** | **direct top-tracks fetch** ✅ DONE | high | med | shipped: scan fetches each recommended artist's top tracks (Spotify/Deezer), resolving the artist id by name-search when the similar-artist row lacks one — guarded by a strict name-match so it never pulls the wrong artist. bounded (top 20 recs), per-call guarded, fail-soft to the pool. iTunes has no top-tracks API → pool-only there. needs a live scan to populate. |
|
||||
| **5** | **genre-affinity boost** | med | low | we already compute your genre breakdown. boost recs whose genres match your top genres → tighter taste alignment. pure scoring add. |
|
||||
| **6** | **adventurousness dial** | med | low | the ranker already supports `min_seed_count` (consensus floor). expose it as a "Safe ↔ Adventurous" slider on the row. |
|
||||
| **7** | **diversity pass** | low-med | low | avoid 40 recs all orbiting your single heaviest seed — cap picks-per-seed so the row spans your taste. |
|
||||
|
||||
the core is built to absorb all of these without re-plumbing — `similarity_from_rank`, `build_recency_weighted_seeds`, and the scoring formula are all pure + tested.
|
||||
|
||||
---
|
||||
|
||||
## full discover-page generator audit (every soulsync-built row, excluding last.fm + listenbrainz)
|
||||
|
||||
how each one is generated today, and whether it can be elevated. "clear win" = safe + additive. "product call" = needs your decision (changes the row's character).
|
||||
|
||||
### curated (built during the scan, then hydrated)
|
||||
- **Fresh Tape / Release Radar** — new releases from watchlist+similar artists. **FIXED tonight** (see above). one more *clear win* available: hydration silently drops any curated id no longer in the discovery pool — could fall back to the stored `track_data_json` blob so the row can't shrink at read time.
|
||||
- **The Archives / Discovery Weekly** — strong already. nice 3-tier popularity split + serendipity scoring (boost never-played artists, penalize overplayed). same hydration-drop caveat as Fresh Tape; same cheap fallback fix.
|
||||
- **Seasonal Mix** — cleanest of the bunch. hydrates from a dedicated `seasonal_tracks` table (carries its own data), so it doesn't suffer the pool-drop problem. no bug.
|
||||
|
||||
### discovery-pool generators (live queries)
|
||||
- **Popular Picks** — ranks by popularity DESC. solid. only nit: on iTunes (no popularity scale) it silently degrades to random — indistinguishable from Shuffle there. UI-label thing at most.
|
||||
- **Hidden Gems** — *clear win*. currently `ORDER BY RANDOM()` over low-popularity tracks — so it's "random obscure," not "*best* obscure." a light ranking (popularity just under the threshold, or genre-affinity to you) would make it feel curated instead of arbitrary. (a deeper *product call*: add personalization like Archives has — bigger lift, changes its "pure underground" character.)
|
||||
- **Genre Playlists** — good. pushes the genre match into SQL. `RANDOM()` ordering is fine for a browse; a popularity/affinity tiebreak (*clear win*) would make thin genres feel less arbitrary.
|
||||
- **Discovery Shuffle** — random by design, correct. only possible add: exclude tracks already shown in other rows this refresh (needs a cross-section seen-set — medium plumbing).
|
||||
- **Time Machine (by decade)** — *clear win, low risk*: decades are hardcoded, so a modern-only library shows 7 decade tabs, 5 empty. filter the tabs to decades that actually have pool data.
|
||||
- **Daily Mix** — the weakest row. the "50% your library" half permanently returns nothing (library tracks have no source ids to play), so each Daily Mix is really just a relabeled Genre Playlist. real fix = backfill source ids into library rows (*schema-level, higher risk*) — worth a dedicated pass, not a quick tweak. also silently falls back to "top artists as pseudo-genres" when genre data is missing → "Daily Mix 1" becomes mislabeled artist-radio. gate/label that (*clear win*).
|
||||
|
||||
### cross-cutting
|
||||
- **hydration fragility** (Fresh Tape + Archives): both depend on curated ids still living in the pool at read time; misses are dropped silently. Seasonal already solved this with a dedicated table. giving the two spotify-style rows the same data-blob fallback is the single most robust cross-cutting fix. low risk, clear win.
|
||||
- **RANDOM-ordering pattern** (Hidden Gems, Shuffle, Genre, Decade): intentional for variety, but leaves quality signal on the table for the non-shuffle rows. adding a light ranking pass to Hidden Gems + Genre is the biggest "best-in-class" lever after tonight's work.
|
||||
|
||||
want me to take any of these? the Hidden Gems ranking + Time Machine empty-decade filter + the Fresh Tape/Archives hydration fallback are all safe, additive, same-shape-as-tonight wins i can knock out next.
|
||||
13
RELEASE_2.7.9_discord.md
Normal file
13
RELEASE_2.7.9_discord.md
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
**SoulSync 2.7.9** is out 🎉 a big one.
|
||||
|
||||
🎚️ **Best-quality downloads** — downloads now follow a ranked quality profile you drag to order (FLAC 24/192 → mp3). best-quality mode grabs the highest-quality copy across *every* source; priority mode gets an opt-in rank-based order toggle. quarantine is folded into the Downloads page + safer imports (AcoustID fail-closed, silence/truncation guards).
|
||||
|
||||
🎧 **Discover got smart** — "Based On Your Listening" ranks artists from who you *actually play*, and "Your Listening Mix" is a playable track playlist of their top tracks (works on any source). Fresh Tape fills properly now.
|
||||
|
||||
⚡ **Wing It Pool** — a new spot next to Discovery Pool to review + re-match the tracks Wing It guessed at (they used to be invisible).
|
||||
|
||||
🔁 **Auto-Sync redesign** — the scheduling board is now clean horizontal lanes instead of a side-scrolling column wall.
|
||||
|
||||
🐛 **Fixes** — multi-disc albums no longer show disc-2 as "missing" (#927), playlists no longer stuck on "Never Synced" (#925), and tracks can't import while quarantined (#928). thanks @ramonskie + @nick2000713 🙏
|
||||
|
||||
⚠️ **re-scan your library once** so the multi-disc fix can backfill disc numbers on existing tracks. enjoy! 🎶
|
||||
13
RELEASE_2.8.0_discord.md
Normal file
13
RELEASE_2.8.0_discord.md
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
**SoulSync 2.8.0** is out 🎉 a quality + reliability release.
|
||||
|
||||
🧹 **The Unverified queue, finally under control** — if you saw thousands of "unverified" rows piling up, this is for you. the AcoustID scan stops duplicating history rows, a one-time reconcile on startup clears the existing backlog from your library (no re-scan), and a new 🧹 *Clean orphaned* button sweeps dead rows whose file is gone. (#934 — thanks @nick2000713 for #938)
|
||||
|
||||
✂️ **Preview Clip Cleanup** — a new Tools job that finds the ~30s preview clips the HiFi source sometimes hands back instead of the full song, then deletes them and re-wishlists the real version. each finding has a ▶ Play button so you can confirm before approving.
|
||||
|
||||
💿 **Album Completeness handles split albums** — an album split across multiple library rows no longer shows every fragment as falsely "incomplete"; it groups the validated fragments into one correct finding. (#936 — thanks @ragnarlotus)
|
||||
|
||||
🐛 **Fixes** — pasted YouTube cookies no longer throw `unsupported browser: "custom"` on Docker (thanks HellRa1SeR); longer remasters aren't quarantined as "truncated" anymore (#937, thanks @diegocade1); "Add to Wishlist" from a discography went from ~15–30s *per track* to instant; wishlist art renders for re-downloads; and **Clear Completed** is back on the Downloads page.
|
||||
|
||||
⚡ **Performance** — trimmed the dashboard GPU usage that was hammering Firefox/Zen (and Background Particles are OFF by default now), plus bounded the runaway memory growth that could lock the app up on big libraries. (#935 / #802)
|
||||
|
||||
enjoy! 🎶
|
||||
15
RELEASE_2.8.1_discord.md
Normal file
15
RELEASE_2.8.1_discord.md
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
**SoulSync 2.8.1** is out 🎉 a feature + reliability release.
|
||||
|
||||
🎧 **Export playlists to Spotify & Deezer** — the mirrored-playlist export now has **Sync to Spotify** and **Sync to Deezer** next to the ListenBrainz / JSPF options. it builds a playlist in your account from the IDs soulsync already has (the discovery cache first, then your library), so an already-discovered playlist exports **instantly with zero API calls**. re-exporting updates the same playlist instead of duplicating it, and an optional *"match missing tracks"* toggle confidently searches for the stragglers — a wrong-artist or karaoke version is left out, never guessed. the first Spotify export asks permission once. (#945)
|
||||
|
||||
🏷️ **Library Reorganize — Rename only** — a lighter action that just **renames your files** to your naming scheme: no re-tag, no quality/AcoustID re-check, no copy-to-staging. much faster on a NAS, and only touches files whose path actually changes. pick it from the new Action dropdown. (#875 — thanks @tsoulard / @Tacobell444)
|
||||
|
||||
💿 **Broader lossless handling** — lossy-copy now covers **all lossless formats**, not just FLAC (#941); and **DSD** (`.dsf`/`.dff`) is recognized as lossless instead of false-flagged "truncated" (#939).
|
||||
|
||||
🐛 **Download + search fixes** — an unbalanced bracket no longer false-fails as "file not found"; a file we couldn't quarantine is left for retry instead of deleted; "file not found" errors are actionable now; pasted Qobuz/Tidal links inject the exact track into manual search (#932); the Wing It pool "Fix Match" works again.
|
||||
|
||||
⚡ **Reduce visual effects, refined** — it no longer freezes functional motion (spinners, progress), only the expensive GPU stuff (blur, shadows, glow). worker orbs default OFF on Firefox and run at ~30fps under reduce-effects. plus a jellyfin scan watchdog fix for big libraries.
|
||||
|
||||
🔧 **Under the hood** — settings cleanup (#943, @nick2000713), spotify oauth hardening (#942) + npm security fixes (#944, HellRa1SeR).
|
||||
|
||||
enjoy! 🎶
|
||||
9
RELEASE_2.8.2_discord.md
Normal file
9
RELEASE_2.8.2_discord.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
**SoulSync 2.8.2** is out 🎉 a stability + performance release.
|
||||
|
||||
🎧 **Spotify, reliably** — the Docker boot hang is fixed: with Spotify as your primary source, an unreachable Spotify API could block startup so the container bound `:8008` but never served the UI. auth probes are now deferred during boot + capped with a timeout. the "re-auth didn't stick" bug is fixed too (the OAuth callback and the app were reading different token caches), and **Sync to Spotify** now works — it asks for playlist-write permission once, on-demand, leaving your normal login untouched. (#949 — thanks HellRa1SeR)
|
||||
|
||||
⚡ **The "slow after update" fix** — the post-update lag wasn't SoulSync, it was browser password managers (Bitwarden/1Password/etc.) rebuilding their autofill overlay on *every* DOM change. non-credential fields are now marked so they skip them — **~110× less main-thread blocking** in the reporter's benchmark. plus a new **Max Performance** mode (Settings → Appearance) that kills every effect for no-GPU / Docker setups. (#948 — thanks @nick2000713)
|
||||
|
||||
📥 **Large-library imports no longer time out** — dropping a whole library into staging used to make the import page scan every file synchronously and never load. the scan runs in the background now with a live "Scanning N of M…" progress, and fills in when done. (#947 — thanks @ramonskie)
|
||||
|
||||
enjoy! 🎶
|
||||
|
|
@ -705,6 +705,12 @@ class ConfigManager:
|
|||
},
|
||||
"import": {
|
||||
"staging_path": "./Staging",
|
||||
# Master toggle for quality-filtering on import. On by default:
|
||||
# downloaded files that don't meet the quality profile are
|
||||
# quarantined instead of imported (same gate the download
|
||||
# pipeline uses). Off → import everything regardless of quality;
|
||||
# the library Quality Upgrade Scanner still flags them.
|
||||
"quality_filter_enabled": True,
|
||||
"replace_lower_quality": False,
|
||||
# Use the top Staging folder as the artist (Artist/Album layouts,
|
||||
# mixtapes). On by default to preserve the long-standing import
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ from config.settings import config_manager
|
|||
from core.amazon_client import AmazonClient, AmazonClientError
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
from core.quality.model import AudioQuality
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("amazon_download_client")
|
||||
|
|
@ -76,9 +78,12 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get("soulseek.download_path", "./downloads")
|
||||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
|
||||
self._quality = config_manager.get("amazon_download.quality", "flac")
|
||||
self._quality = quality_tier_for_source("amazon", default="flac")
|
||||
self._allow_fallback = config_manager.get("amazon_download.allow_fallback", True)
|
||||
|
||||
self._client = AmazonClient(preferred_codec=self._quality)
|
||||
|
|
@ -133,11 +138,17 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
album_map: Dict[str, AlbumResult] = {}
|
||||
album_order: List[str] = []
|
||||
preferred = self._client.preferred_codec
|
||||
# Search results only carry the codec (real sample_rate arrives at
|
||||
# stream time). Claim the format honestly — FLAC for the lossless
|
||||
# codec, lossy otherwise — so audio_quality derives a real format
|
||||
# instead of the display label ("Lossless"), and the post-download
|
||||
# probe pins the actual sample_rate/bit_depth.
|
||||
amazon_q = AudioQuality(format='flac' if _codec_key(preferred) == 'flac' else 'aac')
|
||||
|
||||
for item in items:
|
||||
quality = _quality_label(preferred)
|
||||
if item.is_track:
|
||||
track_results.append(TrackResult(
|
||||
tr = TrackResult(
|
||||
username="amazon",
|
||||
filename=f"{item.asin}||{item.artist_name} - {item.title}",
|
||||
size=0,
|
||||
|
|
@ -155,7 +166,9 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
"album_asin": item.album_asin,
|
||||
"isrc": item.isrc,
|
||||
},
|
||||
))
|
||||
)
|
||||
tr.set_quality(amazon_q)
|
||||
track_results.append(tr)
|
||||
elif item.is_album:
|
||||
album_asin = item.album_asin or item.asin
|
||||
if album_asin not in album_map:
|
||||
|
|
@ -173,6 +186,7 @@ class AmazonDownloadClient(DownloadSourcePlugin):
|
|||
title=item.title,
|
||||
album=item.album_name,
|
||||
)
|
||||
placeholder.set_quality(amazon_q)
|
||||
album_map[album_asin] = AlbumResult(
|
||||
username="amazon",
|
||||
album_path=album_asin,
|
||||
|
|
|
|||
27
core/boot_phase.py
Normal file
27
core/boot_phase.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
"""Boot-phase guard for non-blocking container startup.
|
||||
|
||||
While the gunicorn worker is importing ``web_server`` (module-level client and
|
||||
worker initialization), external provider API probes must not block startup.
|
||||
Network validation is deferred until ``mark_boot_complete()`` runs at the end
|
||||
of that import pass.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
_boot_lock = threading.Lock()
|
||||
_boot_active = True
|
||||
|
||||
|
||||
def is_boot_phase() -> bool:
|
||||
"""Return True while module import must avoid blocking provider API calls."""
|
||||
with _boot_lock:
|
||||
return _boot_active
|
||||
|
||||
|
||||
def mark_boot_complete() -> None:
|
||||
"""End the boot phase — provider clients may perform network probes again."""
|
||||
global _boot_active
|
||||
with _boot_lock:
|
||||
_boot_active = False
|
||||
|
|
@ -117,6 +117,48 @@ def _is_full_track_payload(payload: Optional[Dict[str, Any]]) -> bool:
|
|||
return 'track_position' in payload and 'contributors' in payload
|
||||
|
||||
|
||||
def resolve_album_track_positions(session, base_url, album_ids, cache=None, sleep_s=0.2):
|
||||
"""Build ``{str(track_id): track_position}`` for a set of Deezer album ids.
|
||||
|
||||
Deezer PLAYLIST and SEARCH track objects (and even the album object's embedded
|
||||
``tracks.data``) omit ``track_position`` — only ``/album/<id>/tracks`` and
|
||||
``/track/<id>`` carry it. So numbering playlist tracks by their playlist index
|
||||
silently poisons the real album track number, which then rides onto the
|
||||
downloaded file's tag. This resolves the authoritative position per album
|
||||
(cache-first, best-effort — a failed album just isn't in the map)."""
|
||||
import time as _time
|
||||
positions: Dict[str, int] = {}
|
||||
for aid in album_ids:
|
||||
aid = str(aid)
|
||||
at_list = None
|
||||
if cache:
|
||||
try:
|
||||
ct = cache.get_entity('deezer', 'album_tracks', aid)
|
||||
if ct and ct.get('data'):
|
||||
at_list = ct['data']
|
||||
except Exception: # noqa: BLE001 - cache is best-effort
|
||||
at_list = None
|
||||
if at_list is None:
|
||||
try:
|
||||
if sleep_s:
|
||||
_time.sleep(sleep_s) # respect Deezer rate limits
|
||||
r = session.get(f"{base_url}/album/{aid}/tracks", params={'limit': 500}, timeout=10)
|
||||
if getattr(r, 'ok', False):
|
||||
at_list = (r.json() or {}).get('data', [])
|
||||
if cache and at_list is not None:
|
||||
try:
|
||||
cache.store_entity('deezer', 'album_tracks', aid, {'data': at_list})
|
||||
except Exception as _cache_err: # noqa: BLE001
|
||||
logger.debug("album_tracks cache store failed for %s: %s", aid, _cache_err)
|
||||
except Exception: # noqa: BLE001 - never let metadata resolution break the fetch
|
||||
at_list = None
|
||||
for at in (at_list or []):
|
||||
tp = at.get('track_position')
|
||||
if at.get('id') and tp:
|
||||
positions[str(at['id'])] = tp
|
||||
return positions
|
||||
|
||||
|
||||
# ==================== Dataclasses (match iTunesClient / SpotifyClient format) ====================
|
||||
|
||||
@dataclass
|
||||
|
|
@ -1357,6 +1399,16 @@ class DeezerClient:
|
|||
|
||||
raw_tracks.extend(page_tracks)
|
||||
|
||||
# Real album track positions — playlist tracks don't carry track_position,
|
||||
# so numbering by playlist index would poison the downloaded file's tag.
|
||||
album_ids = {str(t.get('album', {}).get('id')) for t in raw_tracks if t.get('album', {}).get('id')}
|
||||
try:
|
||||
from core.metadata.cache import get_metadata_cache
|
||||
_cache = get_metadata_cache()
|
||||
except Exception:
|
||||
_cache = None
|
||||
track_positions = resolve_album_track_positions(self.session, self.BASE_URL, album_ids, _cache)
|
||||
|
||||
# Normalize tracks
|
||||
tracks: List[Dict[str, Any]] = []
|
||||
for i, t in enumerate(raw_tracks, start=1):
|
||||
|
|
@ -1368,7 +1420,8 @@ class DeezerClient:
|
|||
'artists': [artist_name],
|
||||
'album': t.get('album', {}).get('title', ''),
|
||||
'duration_ms': t.get('duration', 0) * 1000,
|
||||
'track_number': i,
|
||||
# REAL album position; the playlist index is a last resort only.
|
||||
'track_number': track_positions.get(str(t.get('id'))) or i,
|
||||
})
|
||||
|
||||
result = {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
|||
import requests
|
||||
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
from core.quality.source_map import quality_from_deezer, quality_tier_for_source
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("deezer_download")
|
||||
|
|
@ -92,7 +93,10 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
|
||||
# Engine reference is populated by set_engine() at registration
|
||||
# time. None until orchestrator wires the registry.
|
||||
|
|
@ -117,14 +121,20 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
self._license_token = None
|
||||
self._user_data = None
|
||||
self._authenticated = False
|
||||
self._pending_arl: Optional[str] = None
|
||||
|
||||
# Quality preference
|
||||
self._quality = config_manager.get('deezer_download.quality', 'flac')
|
||||
self._quality = quality_tier_for_source('deezer', default='flac')
|
||||
|
||||
# Try to authenticate on init if ARL is configured
|
||||
arl = config_manager.get('deezer_download.arl', '')
|
||||
if arl:
|
||||
self._authenticate(arl)
|
||||
from core.boot_phase import is_boot_phase
|
||||
if is_boot_phase():
|
||||
self._pending_arl = arl
|
||||
logger.debug("Deezer ARL present — authentication deferred until after boot")
|
||||
else:
|
||||
self._authenticate(arl)
|
||||
|
||||
logger.info(f"Deezer download client initialized (download path: {self.download_path})")
|
||||
|
||||
|
|
@ -223,12 +233,66 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
return self._authenticated
|
||||
|
||||
def is_authenticated(self) -> bool:
|
||||
if self._pending_arl and not self._authenticated:
|
||||
from core.boot_phase import is_boot_phase
|
||||
if not is_boot_phase():
|
||||
self._authenticate(self._pending_arl)
|
||||
self._pending_arl = None
|
||||
return self._authenticated
|
||||
|
||||
async def check_connection(self) -> bool:
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(None, self.is_available)
|
||||
|
||||
# ─── Playlist export (#945) ──────────────────────────────────
|
||||
#
|
||||
# UNOFFICIAL: rides the private gw-light gateway with the ARL session already used
|
||||
# for downloads. Deezer shut their public developer API, so this is the only write
|
||||
# path — and it's fragile by nature (breaks when Deezer changes internals).
|
||||
|
||||
def create_or_update_playlist(self, name, track_ids, *, existing_id=None,
|
||||
public=False, description=""):
|
||||
"""Create a Deezer playlist (or append to an existing one) from a mirrored
|
||||
playlist's tracks. ``track_ids`` are stored ``deezer_id`` values per library track.
|
||||
``existing_id`` set → add to that playlist (idempotent re-export reuses the stored
|
||||
target); unset → create a new one. Returns
|
||||
``{success, playlist_id, url, added, error}``."""
|
||||
if not self._authenticated:
|
||||
return {"success": False, "error": "Deezer is not connected (ARL)"}
|
||||
song_ids = [str(t) for t in (track_ids or []) if t]
|
||||
if not song_ids:
|
||||
return {"success": False, "error": "No matching Deezer tracks to export"}
|
||||
try:
|
||||
songs = [[sid, i] for i, sid in enumerate(song_ids)]
|
||||
if existing_id:
|
||||
res = self._gw_call("playlist.addSongs",
|
||||
{"playlist_id": int(existing_id), "songs": songs})
|
||||
if res is None:
|
||||
return {"success": False, "error": "Deezer rejected the playlist update"}
|
||||
playlist_id = existing_id
|
||||
else:
|
||||
res = self._gw_call("playlist.create", {
|
||||
"title": name, "description": description,
|
||||
"is_public": bool(public), "songs": songs,
|
||||
})
|
||||
if res is None:
|
||||
return {"success": False, "error": "Deezer rejected the playlist create"}
|
||||
# gw 'playlist.create' returns the new playlist id (int) as `results`.
|
||||
if isinstance(res, dict):
|
||||
playlist_id = res.get("PLAYLIST_ID") or res.get("id")
|
||||
else:
|
||||
playlist_id = res
|
||||
if not playlist_id:
|
||||
return {"success": False, "error": "Deezer did not return a playlist id"}
|
||||
return {
|
||||
"success": True,
|
||||
"playlist_id": str(playlist_id),
|
||||
"url": f"https://www.deezer.com/playlist/{playlist_id}",
|
||||
"added": len(song_ids),
|
||||
}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def reconnect(self, arl: str = None) -> bool:
|
||||
"""Re-authenticate with a new or existing ARL."""
|
||||
if arl is None:
|
||||
|
|
@ -417,6 +481,12 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
if aid:
|
||||
album_ids.add(str(aid))
|
||||
album_release_dates = {}
|
||||
# Deezer PLAYLIST tracks do NOT carry `track_position` (only `/track/<id>`
|
||||
# and `/album/<id>/tracks` do), so numbering them by their playlist index
|
||||
# poisons the real album track number — which then rides into the wishlist
|
||||
# and onto the downloaded file's tag (e.g. 'Apologize' tagged track 1 instead
|
||||
# of 16). Resolve the REAL position from each album's track list (cache-first).
|
||||
track_positions: Dict[str, int] = {} # str(track_id) -> album track_position
|
||||
try:
|
||||
from core.metadata.cache import get_metadata_cache
|
||||
cache = get_metadata_cache()
|
||||
|
|
@ -429,24 +499,32 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
cached = cache.get_entity('deezer', 'album', aid)
|
||||
if cached and cached.get('release_date'):
|
||||
album_release_dates[aid] = cached['release_date']
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("cache get_entity album release_date: %s", e)
|
||||
# Cache miss — fetch from API
|
||||
try:
|
||||
time.sleep(0.3) # Respect rate limits
|
||||
a_resp = self._session.get(f'https://api.deezer.com/album/{aid}', timeout=10)
|
||||
if a_resp.ok:
|
||||
a_data = a_resp.json()
|
||||
album_release_dates[aid] = a_data.get('release_date', '')
|
||||
# Store in metadata cache for future use
|
||||
if cache:
|
||||
try:
|
||||
cache.store_entity('deezer', 'album', aid, a_data)
|
||||
except Exception as e:
|
||||
logger.debug("cache store_entity album release_date: %s", e)
|
||||
except Exception as e:
|
||||
logger.debug("fetch deezer album release_date %s: %s", aid, e)
|
||||
if aid not in album_release_dates:
|
||||
try:
|
||||
time.sleep(0.3) # Respect rate limits
|
||||
a_resp = self._session.get(f'https://api.deezer.com/album/{aid}', timeout=10)
|
||||
if a_resp.ok:
|
||||
a_data = a_resp.json()
|
||||
album_release_dates[aid] = a_data.get('release_date', '')
|
||||
# Store in metadata cache for future use
|
||||
if cache:
|
||||
try:
|
||||
cache.store_entity('deezer', 'album', aid, a_data)
|
||||
except Exception as e:
|
||||
logger.debug("cache store_entity album release_date: %s", e)
|
||||
except Exception as e:
|
||||
logger.debug("fetch deezer album release_date %s: %s", aid, e)
|
||||
# Real album track positions (separate endpoint — playlist tracks AND the
|
||||
# album object's embedded tracks both omit track_position). Cache-first.
|
||||
try:
|
||||
from core.deezer_client import resolve_album_track_positions
|
||||
track_positions = resolve_album_track_positions(
|
||||
self._session, 'https://api.deezer.com', album_ids, cache)
|
||||
except Exception as e:
|
||||
logger.debug("resolve deezer album track positions: %s", e)
|
||||
|
||||
tracks = []
|
||||
for i, t in enumerate(raw_tracks, start=1):
|
||||
|
|
@ -467,7 +545,9 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
'id': album_id,
|
||||
},
|
||||
'duration_ms': t.get('duration', 0) * 1000,
|
||||
'track_number': i,
|
||||
# REAL album position (resolved above); the playlist index is a last
|
||||
# resort only when the album lookup failed, never the default.
|
||||
'track_number': track_positions.get(str(t.get('id'))) or t.get('track_position') or i,
|
||||
})
|
||||
|
||||
return {
|
||||
|
|
@ -581,7 +661,7 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
bitrate = 128
|
||||
quality = 'mp3'
|
||||
|
||||
results.append(TrackResult(
|
||||
tr = TrackResult(
|
||||
username='deezer_dl',
|
||||
filename=f"{track_id}||{artist} - {title}",
|
||||
size=est_size,
|
||||
|
|
@ -595,7 +675,10 @@ class DeezerDownloadClient(DownloadSourcePlugin):
|
|||
title=title,
|
||||
album=album,
|
||||
track_number=item.get('track_position'),
|
||||
))
|
||||
)
|
||||
# Stamp CD-quality FLAC (16/44.1) so lossless ranks correctly.
|
||||
tr.set_quality(quality_from_deezer(self._quality))
|
||||
results.append(tr)
|
||||
|
||||
logger.info(f"Deezer search for '{query}' returned {len(results)} results")
|
||||
return results, []
|
||||
|
|
|
|||
340
core/discovery/listening_recommendations.py
Normal file
340
core/discovery/listening_recommendations.py
Normal file
|
|
@ -0,0 +1,340 @@
|
|||
"""Listening-driven recommendation core (#913).
|
||||
|
||||
PURE, side-effect-free ranking that turns "the artists you listen to most" plus
|
||||
"who's similar to each" into:
|
||||
|
||||
1. a consensus-ranked list of artists you'd probably love but don't own, and
|
||||
2. an aggregated candidate-track list for a generated playlist.
|
||||
|
||||
No DB / network / config here. The caller (the watchlist scanner) supplies the
|
||||
seeds (top-played artists), the ``similar_artists`` rows per seed, and the
|
||||
owned-artist set, then fetches top tracks for the winners. Keeping the decision
|
||||
logic in one pure place makes it fully unit-testable without the live stack and
|
||||
keeps the scan wiring thin — and additive, so it can't disturb existing flows.
|
||||
|
||||
Scoring rationale (the "best in class" bit): a recommended artist's score is
|
||||
``Σ over the seeds that recommend it of (seed_weight × similarity)``. That single
|
||||
sum rewards all three signals at once — **consensus** (an artist endorsed by many
|
||||
of your seeds accumulates more terms), your **play weight** (heavier seeds push
|
||||
harder), and **similarity strength** — instead of a flat "appears in N lists".
|
||||
``seed_count`` is exposed separately for display ("because you like A, B, C") and
|
||||
as the adventurousness dial's lever (``min_seed_count``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Sequence, Set
|
||||
|
||||
|
||||
def _norm(name: object) -> str:
|
||||
return str(name or "").strip().lower()
|
||||
|
||||
|
||||
def _positive_float(value: object, default: float = 1.0) -> float:
|
||||
try:
|
||||
f = float(value) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return f if f > 0 else default
|
||||
|
||||
|
||||
def _get(row: object, attr: str):
|
||||
"""Read a field from a dataclass row or a dict row."""
|
||||
if isinstance(row, dict):
|
||||
return row.get(attr)
|
||||
return getattr(row, attr, None)
|
||||
|
||||
|
||||
def choose_mix_fetch_source(active_source: object, active_can_fetch: bool) -> str:
|
||||
"""Pick which source to fetch the "Listening Mix" top tracks from.
|
||||
|
||||
The mix is a list of (artist, title) pairs acquired via Soulseek, so the fetch source need
|
||||
NOT match the user's active metadata source. Use the active source when it can fetch top
|
||||
tracks itself (Spotify/Deezer); otherwise fall back to Deezer, whose public ``artist/{id}/top``
|
||||
needs no auth and is available to every user — so iTunes / Discogs / MusicBrainz users still
|
||||
get a full mix without switching sources. Pure.
|
||||
"""
|
||||
if str(active_source or "").lower() in ("spotify", "deezer") and active_can_fetch:
|
||||
return str(active_source).lower()
|
||||
return "deezer"
|
||||
|
||||
|
||||
def names_match(a: object, b: object) -> bool:
|
||||
"""Strict artist-name equality after stripping case + non-alphanumerics.
|
||||
|
||||
Used to verify a name-search result before fetching that artist's top tracks, so the
|
||||
"Listening Mix" can never pull the WRONG artist's songs (e.g. a same-name act). Exact
|
||||
alphanumeric match: "Tyler, The Creator" == "Tyler The Creator", but "Drake" != "Drake Bell".
|
||||
Pure.
|
||||
"""
|
||||
def _alnum(x: object) -> str:
|
||||
return "".join(ch for ch in str(x or "").lower() if ch.isalnum())
|
||||
na, nb = _alnum(a), _alnum(b)
|
||||
return bool(na) and na == nb
|
||||
|
||||
|
||||
def similarity_from_rank(rank: object, max_rank: int = 10) -> float:
|
||||
"""Turn a stored ``similarity_rank`` (1 = most similar … 10 = least) into a 0–1 weight.
|
||||
|
||||
SoulSync stores each ``(seed → similar)`` edge with a 1–10 rank (``1`` is the closest
|
||||
match). The ranker multiplies this into the score so a seed's *closest* matches count
|
||||
for more than its long-tail ones. Linear decay over the documented range: rank 1 → 1.0,
|
||||
rank 5 → 0.6, rank 10 → 0.1, with a 0.1 floor so a far match still contributes. A
|
||||
missing/garbage rank falls back to 1.0 (treat as "no rank info, full weight"). Pure.
|
||||
"""
|
||||
try:
|
||||
r = int(rank)
|
||||
except (TypeError, ValueError):
|
||||
return 1.0
|
||||
floor = round(1.0 / max_rank, 4)
|
||||
if r <= 1:
|
||||
return 1.0
|
||||
if r >= max_rank:
|
||||
return floor
|
||||
return round((max_rank - r + 1) / max_rank, 4)
|
||||
|
||||
|
||||
def build_recency_weighted_seeds(
|
||||
top_artists: Sequence[dict],
|
||||
recent_play_counts: Optional[Dict[str, float]] = None,
|
||||
*,
|
||||
recency_factor: float = 1.5,
|
||||
) -> List[dict]:
|
||||
"""Blend lifetime + recent play counts into seed weights — "what you're into NOW".
|
||||
|
||||
``weight = lifetime_plays + recency_factor × recent_plays``. An artist you've played a
|
||||
lot *recently* outranks one you played a lot years ago, so the recommendations track
|
||||
your current taste instead of your all-time history. ``recency_factor`` is the dial
|
||||
(0 = pure lifetime). Returns ``[{'name', 'weight'}]`` for :func:`rank_recommended_artists`.
|
||||
Pure — the caller supplies both play-count maps from the listening history.
|
||||
"""
|
||||
recent = {_norm(k): _positive_float(v, 0.0) for k, v in (recent_play_counts or {}).items()}
|
||||
out: List[dict] = []
|
||||
for a in top_artists or ():
|
||||
name = str(a.get("name") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
lifetime = _positive_float(a.get("play_count", a.get("weight", 1.0)))
|
||||
boost = recency_factor * recent.get(_norm(name), 0.0)
|
||||
out.append({"name": name, "weight": lifetime + boost})
|
||||
return out
|
||||
|
||||
|
||||
def group_similars_by_seed(
|
||||
seeds: Sequence[dict],
|
||||
similar_rows: Sequence,
|
||||
id_to_name: Dict[str, str],
|
||||
*,
|
||||
source_id_attr: str = "source_artist_id",
|
||||
similar_name_attr: str = "similar_artist_name",
|
||||
rank_attr: Optional[str] = None,
|
||||
) -> Dict[str, List[dict]]:
|
||||
"""Reshape flat ``similar_artists`` rows into ``{seed_name_lower: [{'name', 'score'?}]}``.
|
||||
|
||||
The stored rows key the similar artist by the SEED's source id (``source_artist_id``),
|
||||
not its name, so :func:`rank_recommended_artists` can't consume them directly. This
|
||||
resolves each row's source id to a name via ``id_to_name`` (``{source_artist_id:
|
||||
artist_name}`` for the library, built by the caller) and keeps only rows that resolve
|
||||
to one of the ``seeds``. Rows may be dataclass objects or dicts. Pure — no I/O.
|
||||
|
||||
``id_to_name`` MUST be keyed by whatever id the edges actually store — for SoulSync that
|
||||
is the artist's SOURCE id (Spotify/iTunes/Deezer/MusicBrainz), NOT the internal row id.
|
||||
When ``rank_attr`` is given, each row's rank is converted via :func:`similarity_from_rank`
|
||||
and carried as ``score`` so closer matches weigh more; without it every similar comes out
|
||||
score-less (the ranker then treats similarity as 1.0 — original behavior).
|
||||
"""
|
||||
seed_names = {_norm(s.get("name")) for s in seeds}
|
||||
seed_names.discard("")
|
||||
id_to_norm = {str(k): _norm(v) for k, v in (id_to_name or {}).items()}
|
||||
|
||||
out: Dict[str, List[dict]] = {}
|
||||
for row in similar_rows or ():
|
||||
seed_name = id_to_norm.get(str(_get(row, source_id_attr) or ""), "")
|
||||
if not seed_name or seed_name not in seed_names:
|
||||
continue
|
||||
sim_name = str(_get(row, similar_name_attr) or "").strip()
|
||||
if not sim_name:
|
||||
continue
|
||||
entry = {"name": sim_name}
|
||||
if rank_attr is not None:
|
||||
entry["score"] = similarity_from_rank(_get(row, rank_attr))
|
||||
out.setdefault(seed_name, []).append(entry)
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecommendedArtist:
|
||||
"""One artist recommended from your listening, with the why."""
|
||||
name: str # display name (first-seen casing)
|
||||
score: float # Σ seed_weight × similarity
|
||||
seed_count: int # distinct seeds endorsing it (consensus)
|
||||
seeds: List[str] = field(default_factory=list) # display names of those seeds
|
||||
|
||||
|
||||
def rank_recommended_artists(
|
||||
seeds: Sequence[dict],
|
||||
similars_by_seed: Dict[str, Sequence[dict]],
|
||||
owned_artist_names: Optional[Set[str]] = None,
|
||||
*,
|
||||
limit: int = 30,
|
||||
min_seed_count: int = 1,
|
||||
) -> List[RecommendedArtist]:
|
||||
"""Rank artists similar to your most-played by consensus + play weight + similarity.
|
||||
|
||||
Args:
|
||||
seeds: ``[{'name': str, 'weight': float}]`` — your top-played artists.
|
||||
``weight`` (play count or any positive number) defaults to 1.0.
|
||||
similars_by_seed: ``{seed_name_lower: [{'name': str, 'score': float}]}`` — the
|
||||
similar-artist rows for each seed. ``score`` is optional (defaults 1.0).
|
||||
owned_artist_names: lowercased names already in the library — excluded so the
|
||||
result is artists you DON'T have. The seeds themselves are always excluded.
|
||||
limit: max results.
|
||||
min_seed_count: drop recommendations endorsed by fewer than N seeds — the
|
||||
adventurousness dial's "Safer" end raises this for higher-confidence picks.
|
||||
|
||||
Returns up to ``limit`` :class:`RecommendedArtist`, highest score first.
|
||||
"""
|
||||
owned = {_norm(a) for a in (owned_artist_names or set())}
|
||||
seed_norms = {_norm(s.get("name")) for s in seeds}
|
||||
seed_norms.discard("")
|
||||
exclude = owned | seed_norms
|
||||
|
||||
acc: Dict[str, dict] = {}
|
||||
for seed in seeds:
|
||||
s_name = _norm(seed.get("name"))
|
||||
if not s_name:
|
||||
continue
|
||||
s_display = str(seed.get("name") or "").strip()
|
||||
weight = _positive_float(seed.get("weight", 1.0))
|
||||
for sim in similars_by_seed.get(s_name, ()) or ():
|
||||
a_norm = _norm(sim.get("name"))
|
||||
if not a_norm or a_norm in exclude:
|
||||
continue
|
||||
sim_score = _positive_float(sim.get("score", 1.0))
|
||||
row = acc.setdefault(
|
||||
a_norm, {"name": str(sim.get("name") or "").strip(), "score": 0.0, "seeds": {}}
|
||||
)
|
||||
row["score"] += weight * sim_score
|
||||
row["seeds"].setdefault(s_name, s_display) # one seed counts once
|
||||
|
||||
out: List[RecommendedArtist] = []
|
||||
floor = max(1, int(min_seed_count))
|
||||
for row in acc.values():
|
||||
seed_count = len(row["seeds"])
|
||||
if seed_count < floor:
|
||||
continue
|
||||
out.append(RecommendedArtist(
|
||||
name=row["name"],
|
||||
score=round(row["score"], 6),
|
||||
seed_count=seed_count,
|
||||
seeds=list(row["seeds"].values()),
|
||||
))
|
||||
out.sort(key=lambda r: (-r.score, -r.seed_count, r.name.lower()))
|
||||
return out[:limit]
|
||||
|
||||
|
||||
def aggregate_candidate_tracks(
|
||||
recommended_artists: Sequence[RecommendedArtist],
|
||||
top_tracks_by_artist: Dict[str, Sequence[dict]],
|
||||
owned_track_keys: Optional[Set] = None,
|
||||
*,
|
||||
per_artist: int = 3,
|
||||
limit: int = 50,
|
||||
exclude_owned: bool = True,
|
||||
) -> List[dict]:
|
||||
"""Build the candidate track list for the generated playlist.
|
||||
|
||||
Takes the top ``per_artist`` tracks from each recommended artist **in artist-rank
|
||||
order**, dedups by ``(artist, title)``, optionally drops owned tracks (the
|
||||
"discovery" flavor) and caps at ``limit``. Each returned track dict is the source
|
||||
track plus ``_seed_artist`` (which recommended artist it came from).
|
||||
|
||||
Args:
|
||||
recommended_artists: ranked output of :func:`rank_recommended_artists`.
|
||||
top_tracks_by_artist: ``{artist_name_lower: [track_dict, ...]}`` — fetched by
|
||||
the caller (Last.fm / source top tracks), NOT limited to a curated pool.
|
||||
owned_track_keys: set of ``(artist_lower, title_lower)`` already in the library.
|
||||
exclude_owned: drop tracks in ``owned_track_keys`` (discovery flavor). Set False
|
||||
for a "replay" playlist of tracks you already own.
|
||||
"""
|
||||
owned = owned_track_keys or set()
|
||||
seen: Set = set()
|
||||
out: List[dict] = []
|
||||
for art in recommended_artists:
|
||||
tracks = top_tracks_by_artist.get(_norm(art.name), ()) or ()
|
||||
taken = 0
|
||||
for t in tracks:
|
||||
if taken >= per_artist:
|
||||
break
|
||||
title = str(t.get("name") or t.get("title") or "").strip()
|
||||
if not title:
|
||||
continue
|
||||
key = (_norm(art.name), _norm(title))
|
||||
if key in seen:
|
||||
continue
|
||||
if exclude_owned and key in owned:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append({**t, "_seed_artist": art.name})
|
||||
taken += 1
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out[:limit]
|
||||
|
||||
|
||||
def to_mix_track(track: object, source: str) -> Optional[dict]:
|
||||
"""Shape one source "top tracks" API dict into the flat dict the Discover compact
|
||||
playlist row renders + syncs (the "Listening Mix" #913 playlist).
|
||||
|
||||
Spotify's ``artist_top_tracks`` and Deezer's ``get_artist_top_tracks`` both return the
|
||||
same Spotify-shape object (``id, name, artists[], album{name,images[]}, duration_ms``).
|
||||
This flattens that into the renderer's field names (``track_name/artist_name/album_name/
|
||||
album_cover_url/duration_ms``), keeps the original under ``track_data_json`` for sync, and
|
||||
sets the source-specific id field. Returns None for anything without a usable id/title so
|
||||
the caller can filter. A ``name`` key is kept so :func:`aggregate_candidate_tracks` can
|
||||
dedup by title. Pure — no I/O.
|
||||
"""
|
||||
if not isinstance(track, dict):
|
||||
return None
|
||||
tid = track.get("id")
|
||||
name = str(track.get("name") or "").strip()
|
||||
if not tid or not name:
|
||||
return None
|
||||
artists = track.get("artists") or []
|
||||
artist_name = ""
|
||||
if artists and isinstance(artists[0], dict):
|
||||
artist_name = str(artists[0].get("name") or "").strip()
|
||||
album = track.get("album") if isinstance(track.get("album"), dict) else {}
|
||||
album_name = str(album.get("name") or "").strip()
|
||||
images = album.get("images") or []
|
||||
cover = images[0].get("url") if images and isinstance(images[0], dict) else None
|
||||
out = {
|
||||
"track_id": str(tid),
|
||||
"name": name, # for aggregate_candidate_tracks dedup
|
||||
"track_name": name, # for the renderer
|
||||
"artist_name": artist_name,
|
||||
"album_name": album_name,
|
||||
"album_cover_url": cover,
|
||||
"duration_ms": track.get("duration_ms") or 0,
|
||||
"track_data_json": track, # full payload for sync/download
|
||||
"source": source,
|
||||
}
|
||||
id_field = {"spotify": "spotify_track_id", "deezer": "deezer_track_id",
|
||||
"itunes": "itunes_track_id"}.get(source)
|
||||
if id_field:
|
||||
out[id_field] = str(tid)
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RecommendedArtist",
|
||||
"choose_mix_fetch_source",
|
||||
"names_match",
|
||||
"similarity_from_rank",
|
||||
"build_recency_weighted_seeds",
|
||||
"to_mix_track",
|
||||
"group_similars_by_seed",
|
||||
"rank_recommended_artists",
|
||||
"aggregate_candidate_tracks",
|
||||
]
|
||||
|
|
@ -21,7 +21,11 @@ from core.metadata.registry import get_client_for_source
|
|||
from core.metadata.types import Album
|
||||
from core.wishlist.payloads import ensure_wishlist_track_format
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
# Use the project logger namespace ("soulsync.*") so the scanner's progress and
|
||||
# diagnostics actually surface in the app log — plain getLogger(__name__) lands
|
||||
# under "core.discovery.quality_scanner", which the app log view doesn't show.
|
||||
from utils.logging_config import get_logger
|
||||
logger = get_logger("discovery.quality_scanner")
|
||||
|
||||
|
||||
# Per-source typed converter dispatch — same registry pattern as
|
||||
|
|
|
|||
|
|
@ -173,6 +173,32 @@ async def _database_only_find_track(spotify_track, candidate_pool=None):
|
|||
logger.debug("sync match cache fast-path failed: %s", e)
|
||||
# --- End cache fast-path ---
|
||||
|
||||
# Durable manual library match (#787) — survives a library rescan (the
|
||||
# sync_match_cache above does not), so a user's Find & Add pairing keeps
|
||||
# sticking across auto-syncs instead of being re-matched from scratch (#895
|
||||
# follow-up). Self-heals a stale library id via the stored file path.
|
||||
if spotify_id:
|
||||
try:
|
||||
from core.artists.map import get_current_profile_id
|
||||
m = db.find_manual_library_match_by_source_track_id(
|
||||
get_current_profile_id(), str(spotify_id), active_server)
|
||||
if m:
|
||||
lib_id = m.get('library_track_id')
|
||||
dt = db.get_track_by_id(lib_id) if lib_id is not None else None
|
||||
if not dt and m.get('library_file_path'):
|
||||
new_id = db.find_track_id_by_file_path(m['library_file_path'])
|
||||
dt = db.get_track_by_id(new_id) if new_id else None
|
||||
if dt:
|
||||
class DatabaseTrackDurable:
|
||||
def __init__(self, db_t):
|
||||
self.ratingKey = db_t.id
|
||||
self.title = db_t.title
|
||||
self.id = db_t.id
|
||||
logger.debug(f"Durable manual match hit: '{original_title}' → {lib_id}")
|
||||
return DatabaseTrackDurable(dt), 1.0
|
||||
except Exception as e:
|
||||
logger.debug("durable manual match fast-path failed: %s", e)
|
||||
|
||||
# Try each artist
|
||||
for artist in spotify_track.artists:
|
||||
if isinstance(artist, str):
|
||||
|
|
@ -281,35 +307,11 @@ def run_sync_task(
|
|||
# This avoids needing to re-fetch it from Spotify
|
||||
logger.info("Converting JSON tracks to SpotifyTrack objects...")
|
||||
|
||||
# Store original track data with full album objects (for wishlist with cover art)
|
||||
# Normalize formats for wishlist: album must be dict {'name': ...}, artists must be [{'name': ...}]
|
||||
# Important: copy data — don't mutate tracks_json since SpotifyTrack expects List[str] artists
|
||||
original_tracks_map = {}
|
||||
for t in tracks_json:
|
||||
track_id = t.get('id', '')
|
||||
if track_id:
|
||||
normalized = dict(t)
|
||||
# Normalize album to dict format, preserving images and metadata
|
||||
raw_album = normalized.get('album', '')
|
||||
if isinstance(raw_album, str):
|
||||
normalized['album'] = {
|
||||
'name': raw_album or normalized.get('name', 'Unknown Album'),
|
||||
'images': [], 'album_type': 'single', 'total_tracks': 1, 'release_date': ''
|
||||
}
|
||||
elif not isinstance(raw_album, dict):
|
||||
normalized['album'] = {
|
||||
'name': str(raw_album) if raw_album else normalized.get('name', 'Unknown Album'),
|
||||
'images': [], 'album_type': 'single', 'total_tracks': 1, 'release_date': ''
|
||||
}
|
||||
else:
|
||||
# Dict — ensure required keys exist
|
||||
raw_album.setdefault('name', 'Unknown Album')
|
||||
raw_album.setdefault('images', [])
|
||||
# Normalize artists to list of dicts
|
||||
raw_artists = normalized.get('artists', [])
|
||||
if raw_artists and isinstance(raw_artists[0], str):
|
||||
normalized['artists'] = [{'name': a} for a in raw_artists]
|
||||
original_tracks_map[track_id] = normalized
|
||||
# Store original track data with full album objects (for wishlist with cover art).
|
||||
# Shared with the sync-detail "re-add to wishlist" action so both build the
|
||||
# IDENTICAL payload (album→dict + images, artists→dicts). Copy-safe.
|
||||
from core.sync.wishlist_readd import build_original_tracks_map
|
||||
original_tracks_map = build_original_tracks_map(tracks_json)
|
||||
|
||||
tracks = []
|
||||
for i, t in enumerate(tracks_json):
|
||||
|
|
|
|||
|
|
@ -32,6 +32,26 @@ from typing import Any, Callable
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_UNKNOWN_ARTIST = 'Unknown Artist'
|
||||
|
||||
|
||||
def resolve_display_artist(yt_artist: str, matched_artist: str) -> str:
|
||||
"""The artist to show in the 'YT Artist' column (#909).
|
||||
|
||||
YouTube's flat playlist data carries no artist, so a track starts as
|
||||
"Unknown Artist" and only gains a real name if per-video recovery succeeds.
|
||||
When recovery comes up empty but the track still matched confidently, show
|
||||
the matched artist instead of a misleading "Unknown Artist". Returns the
|
||||
original ``yt_artist`` whenever it's already a real name (recovery worked) or
|
||||
when there's no matched artist to fall back to — purely a display choice, the
|
||||
match itself is unaffected.
|
||||
"""
|
||||
current = (yt_artist or '').strip()
|
||||
if current and current != _UNKNOWN_ARTIST:
|
||||
return current # recovery already gave a real name — keep it
|
||||
fallback = (matched_artist or '').strip()
|
||||
return fallback or _UNKNOWN_ARTIST # backfill from the match, else honest Unknown
|
||||
|
||||
|
||||
@dataclass
|
||||
class YoutubeDiscoveryDeps:
|
||||
|
|
@ -131,14 +151,15 @@ def run_youtube_discovery_worker(url_hash, deps: YoutubeDiscoveryDeps):
|
|||
cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source)
|
||||
if cached_match and deps.validate_discovery_cache_artist(cleaned_artist, cached_match):
|
||||
logger.debug(f"CACHE HIT [{i+1}/{len(tracks)}]: {cleaned_artist} - {cleaned_title}")
|
||||
_match_artist = deps.extract_artist_name(cached_match.get('artists', [''])[0]) if cached_match.get('artists') else ''
|
||||
result = {
|
||||
'index': i,
|
||||
'yt_track': cleaned_title,
|
||||
'yt_artist': cleaned_artist,
|
||||
'yt_artist': resolve_display_artist(cleaned_artist, _match_artist),
|
||||
'status': 'Found',
|
||||
'status_class': 'found',
|
||||
'spotify_track': cached_match.get('name', ''),
|
||||
'spotify_artist': deps.extract_artist_name(cached_match.get('artists', [''])[0]) if cached_match.get('artists') else '',
|
||||
'spotify_artist': _match_artist,
|
||||
'spotify_album': cached_match.get('album', {}).get('name', '') if isinstance(cached_match.get('album'), dict) else cached_match.get('album', ''),
|
||||
'duration': f"{int(track['duration_ms']) // 60000}:{(int(track['duration_ms']) % 60000) // 1000:02d}" if track['duration_ms'] else '0:00',
|
||||
'discovery_source': discovery_source,
|
||||
|
|
@ -265,15 +286,17 @@ def run_youtube_discovery_worker(url_hash, deps: YoutubeDiscoveryDeps):
|
|||
best_confidence = confidence
|
||||
logger.info(f"Strategy 4 YouTube match (extended): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})")
|
||||
|
||||
# Create result entry
|
||||
# Create result entry. yt_artist falls back to the matched artist when
|
||||
# YouTube/recovery left it "Unknown Artist" but we matched confidently (#909).
|
||||
_match_artist = deps.extract_artist_name(matched_track.artists[0]) if matched_track else ''
|
||||
result = {
|
||||
'index': i,
|
||||
'yt_track': cleaned_title,
|
||||
'yt_artist': cleaned_artist,
|
||||
'yt_artist': resolve_display_artist(cleaned_artist, _match_artist),
|
||||
'status': 'Found' if matched_track else 'Not Found',
|
||||
'status_class': 'found' if matched_track else 'not-found',
|
||||
'spotify_track': matched_track.name if matched_track else '',
|
||||
'spotify_artist': deps.extract_artist_name(matched_track.artists[0]) if matched_track else '',
|
||||
'spotify_artist': _match_artist,
|
||||
'spotify_album': matched_track.album if matched_track else '',
|
||||
'duration': f"{int(track['duration_ms']) // 60000}:{(int(track['duration_ms']) % 60000) // 1000:02d}" if track['duration_ms'] else '0:00',
|
||||
'discovery_source': discovery_source,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ big-bang switchover.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
||||
|
||||
|
|
@ -391,6 +392,16 @@ class DownloadEngine:
|
|||
(tracks, albums) tuple, or ``([], [])`` when every source
|
||||
in the chain is exhausted.
|
||||
|
||||
Priority mode is deliberately quality-AGNOSTIC at search time — source
|
||||
order is king and the first source that returns any tracks wins, exactly
|
||||
matching pre-quality-system behaviour byte-for-byte (#896 review #3).
|
||||
Quality-gating the priority path would deprioritise e.g. a soulseek
|
||||
mp3 whose bitrate slskd omitted (``bitrate=None`` → "unsatisfied"),
|
||||
changing which source wins and adding latency for users who never opted
|
||||
in. Cross-source quality pooling is the job of best_quality mode
|
||||
(``search_all_sources``); final per-result ranking still happens in the
|
||||
orchestrator's match/quality filter. RAW tracks are returned.
|
||||
|
||||
Replaces orchestrator's hand-rolled hybrid search loop. The
|
||||
chain is ordered (most-preferred first).
|
||||
"""
|
||||
|
|
@ -406,9 +417,10 @@ class DownloadEngine:
|
|||
try:
|
||||
logger.info(f"Trying {source_name} (priority {i+1}): {query}")
|
||||
tracks, albums = await plugin.search(query, timeout, progress_callback)
|
||||
if tracks:
|
||||
logger.info(f"{source_name} found {len(tracks)} tracks")
|
||||
return (tracks, albums)
|
||||
if not tracks:
|
||||
continue
|
||||
logger.info(f"{source_name} found {len(tracks)} tracks")
|
||||
return (tracks, albums)
|
||||
except Exception as e:
|
||||
logger.warning(f"{source_name} search failed: {e}")
|
||||
|
||||
|
|
@ -418,6 +430,75 @@ class DownloadEngine:
|
|||
)
|
||||
return ([], [])
|
||||
|
||||
async def search_all_sources(self, query: str, source_chain,
|
||||
timeout=None, progress_callback=None,
|
||||
exclude_sources=None):
|
||||
"""Best-quality mode: pool RAW tracks from EVERY configured source in
|
||||
``source_chain`` instead of stopping at the first satisfying one.
|
||||
|
||||
Unlike :meth:`search_with_fallback`, no source short-circuits the
|
||||
search — the caller (orchestrator/worker) ranks the combined pool
|
||||
best→worst by actual audio quality. ``exclude_sources`` drops sources
|
||||
whose per-source retry budget is already spent (so their candidates
|
||||
never re-enter the pool). Unconfigured / unregistered / raising sources
|
||||
are skipped exactly like the fallback path. Returns
|
||||
``(combined_tracks, combined_albums)``.
|
||||
"""
|
||||
excluded = {s.lower() for s in (exclude_sources or []) if s}
|
||||
pooled_tracks = []
|
||||
pooled_albums = []
|
||||
# Per-source contribution for an honest pool log — e.g. a release-level
|
||||
# source like usenet/torrent that returns nothing for a track-title
|
||||
# query should read "usenet=0", not silently hide behind the chain name.
|
||||
contributions = []
|
||||
|
||||
# Decide which sources to actually query, recording why the rest were
|
||||
# skipped. Searches then run CONCURRENTLY so the pool waits only for the
|
||||
# slowest source (e.g. usenet/Prowlarr, which can be slow) rather than
|
||||
# the sum of every source's latency.
|
||||
to_search = [] # (source_name, plugin)
|
||||
for source_name in source_chain:
|
||||
if source_name.lower() in excluded:
|
||||
contributions.append(f"{source_name}=excluded")
|
||||
continue
|
||||
plugin = self._plugins.get(source_name)
|
||||
if plugin is None:
|
||||
logger.info(f"Skipping {source_name} (not available)")
|
||||
contributions.append(f"{source_name}=unavailable")
|
||||
continue
|
||||
if hasattr(plugin, 'is_configured') and not plugin.is_configured():
|
||||
logger.info(f"Skipping {source_name} (not configured)")
|
||||
contributions.append(f"{source_name}=unconfigured")
|
||||
continue
|
||||
to_search.append((source_name, plugin))
|
||||
|
||||
async def _one(plugin):
|
||||
return await plugin.search(query, timeout, progress_callback)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*[_one(plugin) for _, plugin in to_search],
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
for (source_name, _), result in zip(to_search, results, strict=True):
|
||||
if isinstance(result, Exception):
|
||||
logger.warning(f"{source_name} search failed: {result}")
|
||||
contributions.append(f"{source_name}=error")
|
||||
continue
|
||||
tracks, albums = result
|
||||
n = len(tracks) if tracks else 0
|
||||
if tracks:
|
||||
pooled_tracks.extend(tracks)
|
||||
if albums:
|
||||
pooled_albums.extend(albums)
|
||||
contributions.append(f"{source_name}={n}")
|
||||
|
||||
logger.info(
|
||||
"Best-quality pool: %d candidates [%s] for: %s",
|
||||
len(pooled_tracks), ', '.join(contributions), query,
|
||||
)
|
||||
return (pooled_tracks, pooled_albums)
|
||||
|
||||
async def download_with_fallback(self, username: str, filename: str,
|
||||
file_size: int, source_chain) -> Optional[str]:
|
||||
"""Try each source in ``source_chain`` until one accepts the
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from config.settings import config_manager
|
|||
from core.download_engine import DownloadEngine
|
||||
from core.download_plugins.registry import DownloadPluginRegistry, build_default_registry
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.quality.selection import load_search_mode
|
||||
|
||||
logger = get_logger("download_orchestrator")
|
||||
|
||||
|
|
@ -102,12 +103,14 @@ class DownloadOrchestrator:
|
|||
deezer_dl = self.client('deezer_dl')
|
||||
if deezer_arl and deezer_dl:
|
||||
deezer_dl.reconnect(deezer_arl)
|
||||
deezer_dl._quality = config_manager.get('deezer_download.quality', 'flac')
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
deezer_dl._quality = quality_tier_for_source('deezer', default='flac')
|
||||
|
||||
# Reload Amazon quality preference (T2Tunes needs no reconnect — public proxy)
|
||||
amazon = self.client('amazon')
|
||||
if amazon:
|
||||
quality = config_manager.get('amazon_download.quality', 'flac')
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
quality = quality_tier_for_source('amazon', default='flac')
|
||||
amazon._quality = quality
|
||||
amazon._allow_fallback = config_manager.get('amazon_download.allow_fallback', True)
|
||||
if hasattr(amazon, '_client') and amazon._client:
|
||||
|
|
@ -140,7 +143,10 @@ class DownloadOrchestrator:
|
|||
continue
|
||||
if hasattr(client, 'download_path') and client.download_path != new_path:
|
||||
client.download_path = new_path
|
||||
client.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
client.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {new_path}: {e}")
|
||||
# YouTube also caches path in yt-dlp opts
|
||||
if hasattr(client, 'download_opts') and 'outtmpl' in client.download_opts:
|
||||
client.download_opts['outtmpl'] = str(new_path / '%(title)s.%(ext)s')
|
||||
|
|
@ -342,6 +348,11 @@ class DownloadOrchestrator:
|
|||
if not chain:
|
||||
logger.warning("Hybrid search exhausted: no eligible sources after exclusion filter")
|
||||
return [], []
|
||||
if load_search_mode() == 'best_quality':
|
||||
logger.info(f"Best-quality search ({' → '.join(chain)}): {query}")
|
||||
return await self.engine.search_all_sources(
|
||||
query, chain, timeout, progress_callback,
|
||||
)
|
||||
logger.info(f"Hybrid search ({' → '.join(chain)}): {query}")
|
||||
return await self.engine.search_with_fallback(query, chain, timeout, progress_callback)
|
||||
|
||||
|
|
@ -412,9 +423,17 @@ class DownloadOrchestrator:
|
|||
|
||||
if scored:
|
||||
scored.sort(key=lambda x: x._match_confidence, reverse=True)
|
||||
filtered_results = scored
|
||||
# Match filter done (right track); now prefer the best quality
|
||||
# among the confidence-passing survivors so streaming isn't
|
||||
# quality-blind like Soulseek already isn't. Stable ranking
|
||||
# keeps confidence order within an equal quality tier; the
|
||||
# `or scored` fail-safe never leaves us with nothing to try.
|
||||
from core.quality.selection import rank_for_profile
|
||||
ranked, _ = rank_for_profile(scored)
|
||||
filtered_results = ranked or scored
|
||||
logger.info(f"Streaming validation: {len(scored)}/{len(tracks)} passed "
|
||||
f"(best: {scored[0]._match_confidence:.2f})")
|
||||
f"(best: {scored[0]._match_confidence:.2f}, "
|
||||
f"quality pick: {filtered_results[0].audio_quality.label()})")
|
||||
else:
|
||||
logger.warning(f"No streaming results passed validation for: {query}")
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -13,10 +13,11 @@ import from a neutral package per Cin's contract-first standard.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from core.imports.filename import parse_filename_metadata
|
||||
from core.quality.model import AudioQuality
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -32,6 +33,36 @@ class SearchResult:
|
|||
upload_speed: int
|
||||
queue_length: int
|
||||
result_type: str = "track" # "track" or "album"
|
||||
# Rich quality metadata — populated by sources that provide it.
|
||||
# None means "unknown", not "absent".
|
||||
sample_rate: Optional[int] = None # Hz (e.g. 44100, 96000, 192000)
|
||||
bit_depth: Optional[int] = None # bits per sample (16, 24)
|
||||
|
||||
@property
|
||||
def audio_quality(self) -> AudioQuality:
|
||||
"""Unified quality descriptor derived from this result's fields."""
|
||||
return AudioQuality(
|
||||
format=self.quality.lower() if self.quality else 'unknown',
|
||||
bitrate=self.bitrate,
|
||||
sample_rate=self.sample_rate,
|
||||
bit_depth=self.bit_depth,
|
||||
)
|
||||
|
||||
def set_quality(self, aq: AudioQuality) -> None:
|
||||
"""Merge a mapped :class:`AudioQuality` onto this result's fields.
|
||||
|
||||
Used by streaming sources to stamp their claimed tier (Tidal/HiFi
|
||||
tier strings, Qobuz API values, …) so ``audio_quality`` ranks
|
||||
correctly. Mapper-provided fields win; a ``None`` from the mapper
|
||||
leaves any already-reported value (e.g. a probed bitrate) intact.
|
||||
"""
|
||||
self.quality = aq.format
|
||||
if aq.bitrate is not None:
|
||||
self.bitrate = aq.bitrate
|
||||
if aq.sample_rate is not None:
|
||||
self.sample_rate = aq.sample_rate
|
||||
if aq.bit_depth is not None:
|
||||
self.bit_depth = aq.bit_depth
|
||||
|
||||
@property
|
||||
def quality_score(self) -> float:
|
||||
|
|
@ -127,6 +158,19 @@ class AlbumResult:
|
|||
queue_length: int = 0
|
||||
result_type: str = "album"
|
||||
|
||||
@property
|
||||
def audio_quality(self) -> AudioQuality:
|
||||
"""Unified quality descriptor derived from dominant track quality."""
|
||||
sample_rates = [t.sample_rate for t in self.tracks if t.sample_rate]
|
||||
bit_depths = [t.bit_depth for t in self.tracks if t.bit_depth]
|
||||
bitrates = [t.bitrate for t in self.tracks if t.bitrate]
|
||||
return AudioQuality(
|
||||
format=self.dominant_quality.lower() if self.dominant_quality else 'unknown',
|
||||
bitrate=max(bitrates) if bitrates else None,
|
||||
sample_rate=max(sample_rates) if sample_rates else None,
|
||||
bit_depth=max(bit_depths) if bit_depths else None,
|
||||
)
|
||||
|
||||
@property
|
||||
def quality_score(self) -> float:
|
||||
"""Calculate album quality score based on dominant quality and track count"""
|
||||
|
|
|
|||
|
|
@ -83,9 +83,26 @@ def clear_completed_local() -> int:
|
|||
"""
|
||||
cleared = 0
|
||||
with tasks_lock:
|
||||
# Protect tasks belonging to a still-active batch. A batch is "active"
|
||||
# while any of its queued tasks is non-terminal (still searching /
|
||||
# downloading / queued / post-processing). Pruning a batch's completed
|
||||
# or failed tasks mid-run would yank them out of the Downloads page —
|
||||
# and failed/cancelled rows aren't recoverable from library_history —
|
||||
# so the user would never see them until the batch ended. Keep the whole
|
||||
# active batch intact; it gets cleaned by a later run once it finishes.
|
||||
protected_task_ids: set = set()
|
||||
for batch in download_batches.values():
|
||||
queue = batch.get('queue', []) if isinstance(batch, dict) else []
|
||||
batch_active = any(
|
||||
download_tasks.get(tid, {}).get('status') not in _TERMINAL_STATUSES
|
||||
for tid in queue if tid in download_tasks
|
||||
)
|
||||
if batch_active:
|
||||
protected_task_ids.update(queue)
|
||||
|
||||
task_ids_to_remove = [
|
||||
tid for tid, task in download_tasks.items()
|
||||
if task.get('status') in _TERMINAL_STATUSES
|
||||
if task.get('status') in _TERMINAL_STATUSES and tid not in protected_task_ids
|
||||
]
|
||||
for tid in task_ids_to_remove:
|
||||
del download_tasks[tid]
|
||||
|
|
|
|||
|
|
@ -47,6 +47,55 @@ from core.runtime_state import (
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _priority_sort_key(r):
|
||||
"""Today's confidence-first key: never download a high-quality WRONG file."""
|
||||
return (
|
||||
getattr(r, 'confidence', 0) or 0,
|
||||
getattr(r, 'quality_score', 0) or 0,
|
||||
getattr(r, 'upload_speed', 0) or 0,
|
||||
-(getattr(r, 'queue_length', 0) or 0),
|
||||
getattr(r, 'free_upload_slots', 0) or 0,
|
||||
getattr(r, 'size', 0) or 0,
|
||||
)
|
||||
|
||||
|
||||
def _quality_first_sort_key(r, targets):
|
||||
"""Best-quality key: the user's profile quality rank dominates; all the
|
||||
priority-mode signals (confidence, speed, …) become tiebreakers.
|
||||
|
||||
Every candidate reaching this point already passed match filtering, so it
|
||||
is "correct enough" — ordering by quality among correct candidates is safe.
|
||||
Candidates with no usable quality info, or that match no target, sort last
|
||||
(never dropped). Lower target index = better target, so it's negated to fit
|
||||
the descending (reverse=True) sort.
|
||||
"""
|
||||
from core.quality.model import rank_candidate
|
||||
|
||||
aq = getattr(r, 'audio_quality', None)
|
||||
if aq is None or not targets:
|
||||
target_idx, tier = (len(targets) if targets else 0), 0.0
|
||||
else:
|
||||
try:
|
||||
target_idx, tier = rank_candidate(aq, targets)
|
||||
except Exception:
|
||||
target_idx, tier = len(targets), 0.0
|
||||
return (-target_idx, tier) + _priority_sort_key(r)
|
||||
|
||||
|
||||
def order_candidates(candidates, *, quality_first=False, targets=None):
|
||||
"""Return *candidates* ordered best-first for the download walk.
|
||||
|
||||
``quality_first=False`` (priority mode) → confidence-first, byte-for-byte
|
||||
today's behaviour. ``quality_first=True`` (best-quality mode) → the user's
|
||||
profile quality rank dominates, confidence/peer signals break ties.
|
||||
"""
|
||||
if quality_first:
|
||||
key = lambda r: _quality_first_sort_key(r, targets or [])
|
||||
else:
|
||||
key = _priority_sort_key
|
||||
return sorted(candidates, key=key, reverse=True)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CandidatesDeps:
|
||||
"""Bundle of cross-cutting deps the candidate-fallback logic needs."""
|
||||
|
|
@ -59,25 +108,25 @@ class CandidatesDeps:
|
|||
on_download_completed: Callable
|
||||
|
||||
|
||||
def attempt_download_with_candidates(task_id, candidates, track, batch_id=None, deps: CandidatesDeps = None):
|
||||
def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
|
||||
deps: CandidatesDeps = None, *,
|
||||
quality_first=False, quality_targets=None):
|
||||
"""
|
||||
Attempts to download with fallback candidate logic (matches GUI's retry_parallel_download_with_fallback).
|
||||
Returns True if successful, False if all candidates fail.
|
||||
|
||||
``quality_first`` (best-quality search mode) orders the walk by the user's
|
||||
profile quality rank instead of confidence-first; ``quality_targets`` is the
|
||||
profile target list used for that ranking. Defaults preserve priority-mode
|
||||
behaviour exactly.
|
||||
"""
|
||||
# Sort candidates by match confidence first, then peer quality. Upstream
|
||||
# Soulseek validation already considers peer speed/slots/queue when scores
|
||||
# are close; preserve that signal here instead of flattening ties back to
|
||||
# arbitrary slskd response order.
|
||||
candidates.sort(
|
||||
key=lambda r: (
|
||||
getattr(r, 'confidence', 0) or 0,
|
||||
getattr(r, 'quality_score', 0) or 0,
|
||||
getattr(r, 'upload_speed', 0) or 0,
|
||||
-(getattr(r, 'queue_length', 0) or 0),
|
||||
getattr(r, 'free_upload_slots', 0) or 0,
|
||||
getattr(r, 'size', 0) or 0,
|
||||
),
|
||||
reverse=True,
|
||||
# Sort candidates. Priority mode: confidence-first, then peer quality —
|
||||
# upstream Soulseek validation already considers peer speed/slots/queue when
|
||||
# scores are close; preserve that signal instead of flattening ties back to
|
||||
# arbitrary slskd response order. Best-quality mode: profile quality rank
|
||||
# dominates (all candidates here already passed match filtering).
|
||||
candidates = order_candidates(
|
||||
candidates, quality_first=quality_first, targets=quality_targets,
|
||||
)
|
||||
|
||||
with tasks_lock:
|
||||
|
|
@ -205,6 +254,21 @@ def attempt_download_with_candidates(task_id, candidates, track, batch_id=None,
|
|||
'artists': _fallback_album_artists
|
||||
}
|
||||
|
||||
# #915: parity with Reorganize / manual Enrich. If the album context is lean
|
||||
# (no release_date) and the user's PRIMARY metadata source isn't Spotify, hydrate
|
||||
# it from that source — the same place a reorganize reads — so the download's
|
||||
# $year folder, release_date and album_type match instead of dropping the year /
|
||||
# defaulting to YYYY-01-01 and forcing a manual reorganize afterwards.
|
||||
try:
|
||||
from core.downloads.track_metadata_backfill import backfill_album_context_from_source
|
||||
from core.metadata import registry as _meta_registry
|
||||
from core.metadata.album_tracks import get_album_for_source as _get_album_for_source
|
||||
backfill_album_context_from_source(
|
||||
spotify_album_context, _meta_registry.get_primary_source(), _get_album_for_source,
|
||||
)
|
||||
except Exception as _bf_err: # noqa: BLE001 — never let backfill break a download
|
||||
logger.debug("[Context] primary-source album backfill skipped: %s", _bf_err)
|
||||
|
||||
download_payload = candidate.__dict__
|
||||
|
||||
username = download_payload.get('username')
|
||||
|
|
|
|||
|
|
@ -77,7 +77,16 @@ def _normalize_for_finding(text: str) -> str:
|
|||
return ""
|
||||
text = unidecode(text).lower()
|
||||
text = re.sub(r'[._/]', ' ', text)
|
||||
text = re.sub(r'[\[\(].*?[\]\)]', '', text)
|
||||
# Strip ONLY balanced bracket pairs (tags like "[FLAC]", "(Remastered 2016)").
|
||||
# The old combined pattern r'[\[\(].*?[\]\)]' allowed MISMATCHED delimiters, so a
|
||||
# lone unbalanced '[' — slskd reports "[34 - You & Me (Flume Remix)" but saves the
|
||||
# file as "34 - You & Me (Flume Remix)" — matched from that '[' all the way to the
|
||||
# next ')', eating the entire title and collapsing the search target to "flac". The
|
||||
# file then scored 0.40 against the real on-disk name and was reported "not found"
|
||||
# despite sitting right there. Per-delimiter pairs can't over-consume; a stray
|
||||
# unbalanced bracket simply survives to the alphanumeric strip below.
|
||||
text = re.sub(r'\[[^\]]*\]', '', text)
|
||||
text = re.sub(r'\([^)]*\)', '', text)
|
||||
text = re.sub(r'[^a-z0-9\s-]', '', text)
|
||||
return ' '.join(text.split()).strip()
|
||||
|
||||
|
|
|
|||
62
core/downloads/history_match.py
Normal file
62
core/downloads/history_match.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
"""Match a file back to its download-history row when its path has drifted (#934).
|
||||
|
||||
``library_history.file_path`` is frozen at import time, but the file moves afterward
|
||||
(media-server import, library reorganize) and ``tracks.file_path`` — what the AcoustID
|
||||
scanner reads — no longer equals it. Matching on the exact path alone then fails twice:
|
||||
the verification status never reaches the history row (verified tracks read "unverified"),
|
||||
and a fresh ``acoustid_scan`` row gets inserted every run (thousands of duplicates).
|
||||
|
||||
This module picks the canonical history row by exact path first, then by FILENAME guarded
|
||||
by a title check — so a shared filename ("01 - Intro.flac") can never heal the wrong song.
|
||||
Pure (no DB) so the matching rules are unit-testable; the caller does the SQL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Iterable, Optional, Sequence, Tuple
|
||||
|
||||
|
||||
def _norm_title(value) -> str:
|
||||
"""Alphanumeric-only lowercase form, so "Song (Remaster)" vs "song remaster"
|
||||
style drift between the download tag and the media-server tag still agrees."""
|
||||
return ''.join(ch for ch in str(value or '').lower() if ch.isalnum())
|
||||
|
||||
|
||||
def like_filename_filter(basename: str) -> str:
|
||||
"""A ``LIKE ... ESCAPE '\\'`` pattern that coarsely matches rows whose path ends
|
||||
in ``basename``. Escapes the LIKE metacharacters (``%`` ``_`` ``\\``) — filenames
|
||||
routinely contain underscores. Callers MUST still confirm with an exact basename
|
||||
compare (``pick_history_row`` does), since ``'%name'`` also matches ``'xname'``."""
|
||||
esc = basename.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_')
|
||||
return '%' + esc
|
||||
|
||||
|
||||
def pick_history_row(candidates: Sequence[Tuple], *, current_paths: Iterable[str],
|
||||
basename: str, title: str) -> Optional[int]:
|
||||
"""Return the id of the history row to update for this file, or None.
|
||||
|
||||
``candidates``: ``(id, file_path, title, download_source)`` rows the DB pre-filtered
|
||||
(exact path or filename LIKE). A row matches when its path equals the current path OR
|
||||
its filename matches AND its title agrees — the title guard prevents a shared filename
|
||||
("01 - Intro.flac") from healing a different song's row. Among matches a REAL download
|
||||
row is preferred over a synthetic ``acoustid_scan`` row, so the scanner heals the
|
||||
genuine record and the caller can delete the synthetic duplicate. None when nothing
|
||||
matches safely (caller then inserts a fresh row — the "file SoulSync never downloaded"
|
||||
intent)."""
|
||||
paths = {p for p in current_paths if p}
|
||||
want = _norm_title(title)
|
||||
matches: list = [] # (id, is_exact, is_real)
|
||||
for cid, cpath, ctitle, csource in candidates:
|
||||
is_real = csource != 'acoustid_scan'
|
||||
if cpath and cpath in paths:
|
||||
matches.append((cid, True, is_real))
|
||||
elif (basename and cpath and os.path.basename(cpath) == basename
|
||||
and (not want or not _norm_title(ctitle) or _norm_title(ctitle) == want)):
|
||||
matches.append((cid, False, is_real))
|
||||
if not matches:
|
||||
return None
|
||||
# Prefer a REAL download row over a synthetic acoustid_scan row; within that, prefer an
|
||||
# exact-path match over a filename match. Stable, so ties keep DB order (first/oldest id).
|
||||
matches.sort(key=lambda m: (m[2], m[1]), reverse=True)
|
||||
return matches[0][0]
|
||||
|
|
@ -26,6 +26,7 @@ Lifted verbatim from web_server.py. Dependencies injected via
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import traceback
|
||||
|
|
@ -44,6 +45,27 @@ from core.runtime_state import (
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# A task that has been in 'post_processing' longer than this is treated as stuck.
|
||||
# Post-processing (AcoustID + quality + import) is serialized, so a large batch
|
||||
# legitimately backs up — keep this generous so genuinely-slow imports aren't
|
||||
# cut off mid-flight (the old 5-min cutoff falsely "completed" queued tasks).
|
||||
_POST_PROCESSING_STUCK_TIMEOUT = 1800 # 30 minutes
|
||||
|
||||
|
||||
def _resolve_stuck_post_processing_status(task: dict) -> str:
|
||||
"""Decide the terminal status for a task stuck in post_processing.
|
||||
|
||||
Only call it 'completed' if the import actually produced a file on disk
|
||||
(``final_file_path`` is set at the end of successful post-processing). Without
|
||||
a real file, force-completing is a lie — the task shows as a downloaded track
|
||||
that isn't anywhere. Mark those 'failed' so they're retryable and honest.
|
||||
"""
|
||||
final_path = task.get('final_file_path')
|
||||
if final_path and os.path.exists(final_path):
|
||||
return 'completed'
|
||||
return 'failed'
|
||||
|
||||
|
||||
def _safe_batch_dirname(batch_id: str) -> str:
|
||||
safe = ''.join(ch if ch.isalnum() or ch in ('-', '_') else '_' for ch in str(batch_id or 'batch'))
|
||||
return safe or 'batch'
|
||||
|
|
@ -435,9 +457,15 @@ def on_download_completed(batch_id: str, task_id: str, success: bool, deps: Life
|
|||
retrying_count += 1
|
||||
elif task_status == 'post_processing':
|
||||
task_age = current_time - task.get('status_change_time', current_time)
|
||||
if task_age > 300: # 5 minutes (post-processing should be fast)
|
||||
logger.info(f"⏰ [Stuck Detection] Task {queue_task_id} stuck in post_processing for {task_age:.0f}s - forcing completion")
|
||||
task['status'] = 'completed' # Assume it worked if file verification is taking too long
|
||||
if task_age > _POST_PROCESSING_STUCK_TIMEOUT:
|
||||
new_status = _resolve_stuck_post_processing_status(task)
|
||||
if new_status == 'completed':
|
||||
logger.info(f"⏰ [Stuck Detection] Task {queue_task_id} stuck in post_processing for {task_age:.0f}s but file exists — completing")
|
||||
task['status'] = 'completed'
|
||||
else:
|
||||
logger.warning(f"⏰ [Stuck Detection] Task {queue_task_id} stuck in post_processing for {task_age:.0f}s with no output file — marking failed")
|
||||
task['status'] = 'failed'
|
||||
task['error_message'] = 'Post-processing timed out without producing a file'
|
||||
finished_count += 1
|
||||
else:
|
||||
retrying_count += 1
|
||||
|
|
@ -667,9 +695,15 @@ def check_batch_completion_v2(batch_id: str, deps: LifecycleDeps) -> Optional[bo
|
|||
retrying_count += 1
|
||||
elif task_status == 'post_processing':
|
||||
task_age = current_time - task.get('status_change_time', current_time)
|
||||
if task_age > 300: # 5 minutes (post-processing should be fast)
|
||||
logger.info(f"⏰ [Stuck Detection V2] Task {task_id} stuck in post_processing for {task_age:.0f}s - forcing completion")
|
||||
task['status'] = 'completed' # Assume it worked if file verification is taking too long
|
||||
if task_age > _POST_PROCESSING_STUCK_TIMEOUT:
|
||||
new_status = _resolve_stuck_post_processing_status(task)
|
||||
if new_status == 'completed':
|
||||
logger.info(f"⏰ [Stuck Detection V2] Task {task_id} stuck in post_processing for {task_age:.0f}s but file exists — completing")
|
||||
task['status'] = 'completed'
|
||||
else:
|
||||
logger.warning(f"⏰ [Stuck Detection V2] Task {task_id} stuck in post_processing for {task_age:.0f}s with no output file — marking failed")
|
||||
task['status'] = 'failed'
|
||||
task['error_message'] = 'Post-processing timed out without producing a file'
|
||||
finished_count += 1
|
||||
else:
|
||||
retrying_count += 1
|
||||
|
|
|
|||
|
|
@ -760,7 +760,8 @@ class WebUIDownloadMonitor:
|
|||
# used_sources keys are formatted as "{username}_{filename}", so startswith is exact.
|
||||
is_tidal = any(s.startswith('tidal_') for s in tried_sources)
|
||||
if is_tidal:
|
||||
tidal_quality = config_manager.get('tidal_download.quality', 'lossless')
|
||||
from core.quality.source_map import quality_tier_for_source
|
||||
tidal_quality = quality_tier_for_source('tidal', default='lossless')
|
||||
allow_fb = config_manager.get('tidal_download.allow_fallback', True)
|
||||
if tidal_quality == 'hires' and not allow_fb:
|
||||
task['error_message'] = (
|
||||
|
|
|
|||
55
core/downloads/orphan_history.py
Normal file
55
core/downloads/orphan_history.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
"""Identify dead review-queue history rows whose file is gone (#934 follow-up).
|
||||
|
||||
The Unverified/Quarantine review queue is fed from ``library_history`` — an
|
||||
append-only log that is never pruned. When a file is deleted, replaced, or
|
||||
re-downloaded elsewhere, its old ``unverified`` row lingers forever and can
|
||||
never be healed (there's no file left to confirm). Those are *orphans*.
|
||||
|
||||
This decides which rows are orphans, given a ``resolve(row) -> path | None``
|
||||
the caller wires to the real filesystem lookup. Pure (no DB, no filesystem) so
|
||||
the rules — including the safety gate — are unit-testable.
|
||||
|
||||
Safety gate: a filesystem check mass-false-positives when the library mount is
|
||||
down (every file looks missing). So if EVERY reviewed file is unreachable and
|
||||
there are enough rows to judge, we flag it ``suspicious`` and the caller refuses
|
||||
to delete — better to clean nothing than to wipe a healthy log during an outage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Sequence
|
||||
|
||||
|
||||
def find_orphan_history_ids(
|
||||
rows: Sequence[dict],
|
||||
resolve: Callable[[dict], Any],
|
||||
*,
|
||||
min_for_safety: int = 5,
|
||||
deletable: Callable[[dict], bool] | None = None,
|
||||
) -> dict:
|
||||
"""Return ``{'orphan_ids', 'checked', 'suspicious'}``.
|
||||
|
||||
A row is an orphan when it has a non-empty ``file_path`` but ``resolve`` can
|
||||
find no file for it. ``suspicious`` is True when every checked row is
|
||||
missing and there are at least ``min_for_safety`` of them — the mount-down
|
||||
signature; the caller should refuse to delete in that case.
|
||||
|
||||
``deletable`` (optional) protects rows from removal WITHOUT weakening the
|
||||
safety gate: a protected row still counts toward ``checked`` and the
|
||||
all-missing signal (so e.g. a few unverified orphans can't be swept during a
|
||||
mount outage just because protected rows were filtered out first), but it
|
||||
never appears in ``orphan_ids``. Default: every missing row is deletable.
|
||||
"""
|
||||
orphan_ids = []
|
||||
checked = 0
|
||||
missing = 0
|
||||
for row in rows:
|
||||
if not str((row.get('file_path') or '')).strip():
|
||||
continue
|
||||
checked += 1
|
||||
if resolve(row) is None:
|
||||
missing += 1
|
||||
if deletable is None or deletable(row):
|
||||
orphan_ids.append(row.get('id'))
|
||||
suspicious = checked >= min_for_safety and missing == checked
|
||||
return {'orphan_ids': orphan_ids, 'checked': checked, 'suspicious': suspicious}
|
||||
|
|
@ -171,6 +171,21 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
|
|||
logger.info(f"[Post-Processing] Task {task_id} already completed by stream processor, skipping verification")
|
||||
return
|
||||
|
||||
# RACE GUARD: the monitor sets status -> 'post_processing' immediately
|
||||
# before submitting this worker. If the status is now anything else, the
|
||||
# browser-poll post-processor already took ownership of this task — e.g.
|
||||
# it quarantined the file and requeued the next-best candidate (status
|
||||
# -> 'searching', source identity cleared). Bail WITHOUT marking failed
|
||||
# or notifying batch completion: otherwise we clobber that in-flight
|
||||
# retry with a bogus "missing file or source information" failure while a
|
||||
# parallel attempt is importing the song.
|
||||
if task['status'] != 'post_processing':
|
||||
logger.info(
|
||||
f"[Post-Processing] Task {task_id} no longer in 'post_processing' "
|
||||
f"(now '{task['status']}') — another path took over, skipping"
|
||||
)
|
||||
return
|
||||
|
||||
# Extract file information for verification
|
||||
track_info = task.get('track_info', {})
|
||||
task_filename = task.get('filename') or track_info.get('filename')
|
||||
|
|
@ -438,7 +453,19 @@ def run_post_processing_worker(task_id: str, batch_id: str, deps: PostProcessDep
|
|||
logger.error(f"[Post-Processing] Task {task_id} was completed by stream processor - not marking as failed")
|
||||
return
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f'File not found on disk after {_file_search_max_retries} search attempts. Expected: {os.path.basename(task_filename)}'
|
||||
# slskd reported the transfer complete, but the finder never located
|
||||
# the file under the configured download folder. Name the folder we
|
||||
# searched and the two real causes — "still being written" (timing)
|
||||
# or "SoulSync's download path doesn't match slskd's" (the classic
|
||||
# standalone config mismatch) — so the user can self-diagnose instead
|
||||
# of getting an opaque "not found". (Discord: Shdjfgatdif.)
|
||||
_searched_name = os.path.basename((task_filename or '').replace('\\', '/')) or task_filename
|
||||
download_tasks[task_id]['error_message'] = (
|
||||
f"slskd reported '{_searched_name}' downloaded, but it never appeared "
|
||||
f"under the download folder ({download_dir}) after {_file_search_max_retries} "
|
||||
f"checks. Either it's still being written, or SoulSync's download path "
|
||||
f"doesn't match slskd's download directory — they must point at the same folder."
|
||||
)
|
||||
deps.on_download_completed(batch_id, task_id, False)
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -94,6 +94,10 @@ class StatusDeps:
|
|||
run_async: Optional[Callable] = None
|
||||
on_download_completed: Optional[Callable[[str, str, bool], None]] = None
|
||||
get_persistent_download_history: Optional[Callable[[int], list[dict]]] = None
|
||||
# Returns ALL library_history rows with verification_status in
|
||||
# ('unverified', 'force_imported') — no recency limit, so historical
|
||||
# entries are never buried by the general history tail cap.
|
||||
get_unverified_download_history: Optional[Callable[[], list[dict]]] = None
|
||||
|
||||
|
||||
# Streaming sources the engine fallback applies to. Soulseek goes through
|
||||
|
|
@ -796,6 +800,13 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
'progress': progress,
|
||||
'error': task.get('error_message'),
|
||||
'verification_status': task.get('verification_status'),
|
||||
# library_history row id (set at import) so the Unverified review
|
||||
# queue can act on a still-live completed task before it becomes
|
||||
# a persistent-history row.
|
||||
'history_id': task.get('history_id'),
|
||||
# Real probed audio quality (mutagen-read from the actual file),
|
||||
# surfaced so the Downloads page can show what was downloaded.
|
||||
'quality': task.get('quality') or '',
|
||||
'retry_info': task.get('retry_info'),
|
||||
'retry_trigger': task.get('retry_trigger'),
|
||||
'batch_id': batch_id,
|
||||
|
|
@ -812,6 +823,32 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
'is_persistent_history': False,
|
||||
})
|
||||
|
||||
# --- Unverified history (unconditional, no limit) ---
|
||||
# Always load every library_history row that still needs human confirmation
|
||||
# (verification_status IN ('unverified', 'force_imported')). This is NOT
|
||||
# gated on len(items) < limit so that historical entries from past batches
|
||||
# are visible even during a large active batch that would otherwise exhaust
|
||||
# the limit before the history tail is read. Dedup against live tasks by
|
||||
# identity so a track currently in post-processing isn't shown twice.
|
||||
if deps.get_unverified_download_history is not None:
|
||||
try:
|
||||
unverified_entries = deps.get_unverified_download_history() or []
|
||||
except Exception as exc:
|
||||
logger.debug("[Downloads] unverified history lookup failed: %s", exc)
|
||||
unverified_entries = []
|
||||
|
||||
for entry in unverified_entries:
|
||||
item = _build_history_download_item(entry)
|
||||
identity = _download_identity(item.get('title'), item.get('artist'), item.get('album'))
|
||||
if identity in live_identities:
|
||||
continue
|
||||
items.append(item)
|
||||
live_identities.add(identity)
|
||||
|
||||
# --- General recent-history tail (capped, recency-ordered) ---
|
||||
# Fills in the completed/verified tail so the full Downloads list looks
|
||||
# populated. Gated on len(items) < limit so a busy batch doesn't trigger
|
||||
# an extra DB round-trip when we're already at capacity.
|
||||
if deps.get_persistent_download_history is not None and len(items) < limit:
|
||||
history_limit = min(limit - len(items), _PERSISTENT_HISTORY_TAIL_LIMIT)
|
||||
try:
|
||||
|
|
@ -832,7 +869,14 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
live_identities.add(identity)
|
||||
appended_history += 1
|
||||
|
||||
# Sort: active first (by priority), then by timestamp desc within each group
|
||||
# Sort: active first (by priority), then by timestamp desc within each group.
|
||||
# NOTE: the array order is presentation-only — the Downloads page filters
|
||||
# client-side per tab. What matters is that EVERY live task is present: an
|
||||
# earlier `items[:limit]` truncation (active-first) starved completed/failed/
|
||||
# unverified rows off the end during a busy batch, so those tabs stayed empty
|
||||
# until the batch drained. `limit` now bounds only the persistent-history
|
||||
# tail (handled above); live in-memory tasks are always returned in full
|
||||
# (they're already bounded by the 5-min cleanup automation).
|
||||
items.sort(key=lambda x: (x['priority'], -x['timestamp']))
|
||||
|
||||
# Build batch summaries for the batch context panel
|
||||
|
|
@ -860,7 +904,7 @@ def build_unified_downloads_response(limit: int, deps: StatusDeps) -> dict:
|
|||
|
||||
return {
|
||||
'success': True,
|
||||
'downloads': items[:limit],
|
||||
'downloads': items,
|
||||
'total': len(items),
|
||||
'batches': batch_summaries,
|
||||
'timestamp': time.time(),
|
||||
|
|
|
|||
|
|
@ -54,6 +54,37 @@ def _cand_user_file(candidate):
|
|||
return getattr(candidate, 'username', None), getattr(candidate, 'filename', None)
|
||||
|
||||
|
||||
def _candidate_ordering():
|
||||
"""Return ``(quality_first, targets)`` for the active search mode + toggle.
|
||||
|
||||
The candidate walk is ordered by the user's profile quality rank
|
||||
(best→worst) instead of confidence-first when EITHER:
|
||||
- best-quality search mode is active (always quality-first), OR
|
||||
- priority mode and the ``rank_candidates_by_quality`` toggle is on
|
||||
(opt-in; default off keeps the byte-for-byte confidence-first walk).
|
||||
|
||||
Quality-first ordering also makes the version-mismatch force-import pick
|
||||
the highest-quality candidate, because that fallback accepts the
|
||||
first-tried (= best-ordered) quarantined entry.
|
||||
|
||||
Fails closed to confidence-first ordering on any error so a profile/DB
|
||||
hiccup never blocks a download. See
|
||||
docs/superpowers/specs/2026-06-14-best-quality-search-mode-design.md.
|
||||
"""
|
||||
try:
|
||||
from core.quality.selection import (
|
||||
load_search_mode,
|
||||
load_profile_targets,
|
||||
load_rank_candidates_by_quality,
|
||||
)
|
||||
if load_search_mode() == 'best_quality' or load_rank_candidates_by_quality():
|
||||
targets, _ = load_profile_targets()
|
||||
return True, targets
|
||||
except Exception as exc:
|
||||
logger.debug("[Modal Worker] quality ordering unavailable: %s", exc)
|
||||
return False, None
|
||||
|
||||
|
||||
def _try_cached_candidates(task_id, batch_id, track, deps):
|
||||
"""Quarantine-retry fast path: attempt the already-found candidates before
|
||||
re-searching anything.
|
||||
|
|
@ -91,7 +122,10 @@ def _try_cached_candidates(task_id, batch_id, track, deps):
|
|||
f"[Modal Worker] Quarantine retry: trying {len(remaining)} cached "
|
||||
f"candidate(s) before re-searching (task {task_id})"
|
||||
)
|
||||
return deps.attempt_download_with_candidates(task_id, remaining, track, batch_id)
|
||||
_qf, _qt = _candidate_ordering()
|
||||
return deps.attempt_download_with_candidates(
|
||||
task_id, remaining, track, batch_id, quality_first=_qf, quality_targets=_qt,
|
||||
)
|
||||
|
||||
|
||||
def _private_album_bundle_staging_miss_reason(batch_id: Optional[str], deps: Any) -> Optional[str]:
|
||||
|
|
@ -369,6 +403,11 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
logger.info(f"[Modal Worker] Generated {len(search_queries)} smart search queries for '{track.name}': {search_queries}")
|
||||
logger.info(f"[Modal Worker] About to start search loop for task {task_id} (track: '{track.name}')")
|
||||
|
||||
# Best-quality search mode: the orchestrator already pooled candidates
|
||||
# across every source for each query, so order the candidate walk by the
|
||||
# user's profile quality rank (best→worst). Computed once per task.
|
||||
_best_quality, _quality_targets = _candidate_ordering()
|
||||
|
||||
# 2. Sequential Query Search (matches GUI's start_search_worker_parallel logic)
|
||||
search_diagnostics = [] # Track what happened per query for detailed error messages
|
||||
all_raw_results = [] # Collect raw results across queries for candidate review modal
|
||||
|
|
@ -495,7 +534,10 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
download_tasks[task_id]['cached_candidates'] = candidates
|
||||
|
||||
# Try to download with these candidates
|
||||
success = deps.attempt_download_with_candidates(task_id, candidates, track, batch_id)
|
||||
success = deps.attempt_download_with_candidates(
|
||||
task_id, candidates, track, batch_id,
|
||||
quality_first=_best_quality, quality_targets=_quality_targets,
|
||||
)
|
||||
if success:
|
||||
# Download initiated successfully - let the download monitoring system handle completion
|
||||
if batch_id:
|
||||
|
|
@ -529,7 +571,10 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
# === HYBRID FALLBACK: If primary source failed, try remaining sources directly ===
|
||||
# The orchestrator's hybrid search stops at the first source with results, even if
|
||||
# those results all fail quality filtering. Try remaining sources individually.
|
||||
if getattr(deps.download_orchestrator, 'mode', '') == 'hybrid':
|
||||
#
|
||||
# Best-quality mode already searched EVERY source per query (the pool), so this
|
||||
# block would only re-search the same sources — skip it there.
|
||||
if not _best_quality and getattr(deps.download_orchestrator, 'mode', '') == 'hybrid':
|
||||
try:
|
||||
orch = deps.download_orchestrator
|
||||
hybrid_order = getattr(orch, 'hybrid_order', None) or []
|
||||
|
|
|
|||
|
|
@ -13,9 +13,48 @@ Pure + import-safe: parsing only, no network.
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Optional, Tuple
|
||||
from typing import Any, List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
def linked_track_id(track: Any) -> str:
|
||||
"""The source track id stamped on a search result, read from
|
||||
``_source_metadata['track_id']`` — the field every ID-downloadable source
|
||||
(Tidal, Qobuz) records. Empty string when absent. ``TrackResult`` has no
|
||||
top-level ``id``, so callers must NOT use ``getattr(t, 'id')`` (that always
|
||||
missed and left the pasted-link bubble a silent no-op — #932)."""
|
||||
meta = getattr(track, '_source_metadata', None)
|
||||
if not isinstance(meta, dict):
|
||||
return ''
|
||||
return str(meta.get('track_id') or '')
|
||||
|
||||
|
||||
def bubble_linked_track_first(tracks: List[Any], link_track_id: str) -> List[Any]:
|
||||
"""Float the result whose source id matches a pasted link to the top so the
|
||||
user sees the EXACT track they linked, not a fuzzy text-search lookalike
|
||||
(#813/#932). Stable + a graceful no-op when no result carries the id."""
|
||||
if not link_track_id or not tracks:
|
||||
return tracks
|
||||
target = str(link_track_id)
|
||||
return sorted(tracks, key=lambda t: linked_track_id(t) != target)
|
||||
|
||||
|
||||
def inject_linked_track_first(
|
||||
tracks: List[Any], linked_result: Any, link_track_id: str
|
||||
) -> List[Any]:
|
||||
"""Put the EXACT linked track first.
|
||||
|
||||
When ``linked_result`` is the track fetched directly by id, prepend it and
|
||||
drop any search duplicate of it — so an obscure track a text search never
|
||||
surfaced is still present and downloadable (#932). When it's None (the source
|
||||
can't fetch one), fall back to bubbling a matching search result. Pure."""
|
||||
if not link_track_id:
|
||||
return tracks
|
||||
target = str(link_track_id)
|
||||
if linked_result is not None:
|
||||
return [linked_result] + [t for t in tracks if linked_track_id(t) != target]
|
||||
return bubble_linked_track_first(tracks, target)
|
||||
|
||||
# host substring → download source id. Only ID-downloadable streaming sources.
|
||||
_HOSTS = (
|
||||
('tidal.com', 'tidal'),
|
||||
|
|
|
|||
|
|
@ -93,6 +93,56 @@ def _backfill_album_context(
|
|||
album_context['image_url'] = first['url']
|
||||
|
||||
|
||||
# Placeholder album ids used when no real source album id is known — never queryable.
|
||||
_SENTINEL_ALBUM_IDS = {'explicit_album', 'from_sync_modal', ''}
|
||||
|
||||
|
||||
def backfill_album_context_from_source(
|
||||
album_context: Dict[str, Any],
|
||||
primary_source: Optional[str],
|
||||
get_album_for_source_fn: Any,
|
||||
) -> bool:
|
||||
"""Hydrate a lean album context from the user's PRIMARY metadata source (#915).
|
||||
|
||||
Post-processing's only album backfill (:func:`hydrate_download_metadata`) goes through
|
||||
``spotify_client.get_track_details`` — Spotify-only. An iTunes/Deezer-primary user's
|
||||
download therefore kept a lean context (no ``release_date``), so the path dropped the
|
||||
``$year`` and the date defaulted to ``YYYY-01-01`` — until they ran a Reorganize, which
|
||||
reads the full album from the PRIMARY source. This closes that gap by doing the same:
|
||||
fetch the full album from the primary source and backfill, so a download's pathing/tags
|
||||
match what a later reorganize would produce.
|
||||
|
||||
``get_album_for_source_fn(source, album_id)`` is injected (the real one is
|
||||
``core.metadata.album_tracks.get_album_for_source``) so this stays pure + testable.
|
||||
No-op when: the context is already complete; the primary source is spotify (the existing
|
||||
track-details path covers it); or no real source album id is present. Returns True when
|
||||
it filled anything. Never raises — a backfill failure must not break a download.
|
||||
"""
|
||||
if not isinstance(album_context, dict) or not _album_is_lean(album_context):
|
||||
return False
|
||||
if not primary_source or primary_source == 'spotify':
|
||||
return False
|
||||
album_id = album_context.get('id')
|
||||
if not album_id or str(album_id) in _SENTINEL_ALBUM_IDS:
|
||||
return False
|
||||
try:
|
||||
album = get_album_for_source_fn(primary_source, str(album_id))
|
||||
except Exception as e: # noqa: BLE001 — defensive: never let backfill break a download
|
||||
logger.warning("[Context] primary-source (%s) album backfill failed: %s", primary_source, e)
|
||||
return False
|
||||
if not isinstance(album, dict):
|
||||
return False
|
||||
before = album_context.get('release_date')
|
||||
_backfill_album_context(album_context, {'album': album})
|
||||
if album_context.get('release_date') and album_context.get('release_date') != before:
|
||||
logger.info(
|
||||
"[Context] Hydrated lean album context from primary source %s "
|
||||
"(release_date=%r, total_tracks=%r)",
|
||||
primary_source, album_context.get('release_date'), album_context.get('total_tracks'),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def hydrate_download_metadata(
|
||||
track: Any,
|
||||
track_info: Any,
|
||||
|
|
@ -172,4 +222,5 @@ def hydrate_download_metadata(
|
|||
__all__ = [
|
||||
'ResolvedTrackMetadata',
|
||||
'hydrate_download_metadata',
|
||||
'backfill_album_context_from_source',
|
||||
]
|
||||
|
|
|
|||
364
core/exports/export_sources.py
Normal file
364
core/exports/export_sources.py
Normal file
|
|
@ -0,0 +1,364 @@
|
|||
"""Wire the real cheapest-first sources for the export MBID waterfall (#903).
|
||||
|
||||
``mbid_resolver`` is the pure waterfall; this module supplies the real I/O behind each
|
||||
source and assembles the ``resolve_fn`` the export job uses:
|
||||
|
||||
1. **cache** — ``recording_mbid_cache`` (persistent (artist,title)->mbid).
|
||||
2. **DB** — a text-matched library track's ``tracks.musicbrainz_recording_id``.
|
||||
3. **file** — ``MUSICBRAINZ_RECORDING_ID`` tag of that track's file (when the DB row had
|
||||
no recording id but the file was tagged on import).
|
||||
4. **MusicBrainz** — live ``match_recording(track, artist)`` (rate-limited tail).
|
||||
|
||||
Every source is wrapped so any failure (missing table, unreadable file, MB timeout) returns
|
||||
None — the waterfall just falls through, the export never breaks. ``build_resolve_fn`` also
|
||||
writes a fresh non-cache hit back to the cache so the next export of the same song is free.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
from core.exports.mbid_resolver import (
|
||||
SRC_CACHE,
|
||||
SRC_DB,
|
||||
SRC_FILE,
|
||||
SRC_MUSICBRAINZ,
|
||||
normalize_key,
|
||||
resolve_recording_mbid,
|
||||
)
|
||||
|
||||
logger = get_logger("exports.export_sources")
|
||||
|
||||
|
||||
def _db_match(artist: str, title: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Text-match a library track by (artist, title); return (recording_mbid, file_path).
|
||||
Either may be None. Fail-safe — any DB error returns (None, None)."""
|
||||
if not title:
|
||||
return (None, None)
|
||||
try:
|
||||
from database.music_database import get_database
|
||||
db = get_database()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"SELECT t.musicbrainz_recording_id, t.file_path "
|
||||
"FROM tracks t JOIN artists a ON t.artist_id = a.id "
|
||||
"WHERE LOWER(t.title) = LOWER(?) AND LOWER(a.name) = LOWER(?) "
|
||||
"LIMIT 1",
|
||||
(title, artist),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return (None, None)
|
||||
mbid = row[0] if not hasattr(row, "keys") else row["musicbrainz_recording_id"]
|
||||
fpath = row[1] if not hasattr(row, "keys") else row["file_path"]
|
||||
return ((mbid or None), (fpath or None))
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug(f"export db_match failed for '{artist} - {title}': {exc}")
|
||||
return (None, None)
|
||||
|
||||
|
||||
def db_recording_mbid(artist: str, title: str) -> Optional[str]:
|
||||
"""Recording MBID stored on a matched library track (``musicbrainz_recording_id``)."""
|
||||
return _db_match(artist, title)[0]
|
||||
|
||||
|
||||
# Service → the tracks-table column carrying that service's track ID (set by enrichment).
|
||||
# Trusted constants — never user input — so safe to interpolate into the SELECT below.
|
||||
_SERVICE_ID_COLUMNS = {"spotify": "spotify_track_id", "deezer": "deezer_id"}
|
||||
|
||||
|
||||
def db_service_track_id(artist: str, title: str, service: str) -> Optional[str]:
|
||||
"""The service track ID (``spotify_track_id`` / ``deezer_id``) stored on a matched
|
||||
library track — what lets a mirrored playlist be exported BACK to Spotify/Deezer
|
||||
without re-searching, since enrichment already pinned it (#945). Text-matches by
|
||||
(artist, title), same as the MBID resolver. Fail-safe: any miss/error returns None."""
|
||||
column = _SERVICE_ID_COLUMNS.get((service or "").lower())
|
||||
if not column or not title:
|
||||
return None
|
||||
try:
|
||||
from database.music_database import get_database
|
||||
db = get_database()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
f"SELECT t.{column} FROM tracks t JOIN artists a ON t.artist_id = a.id "
|
||||
"WHERE LOWER(t.title) = LOWER(?) AND LOWER(a.name) = LOWER(?) LIMIT 1",
|
||||
(title, artist),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
val = row[0] if not hasattr(row, "keys") else row[column]
|
||||
return val or None
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug(f"export service-id lookup failed for '{artist} - {title}' ({service}): {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def build_service_resolve_fn(service: str) -> Callable[[str, str], Tuple[Optional[str], Optional[str]]]:
|
||||
"""resolve_fn for service-playlist export: ``(artist, title) -> (service_track_id, 'library')``.
|
||||
Plugs into ``resolve_playlist_tracks(..., id_key='service_track_id')`` exactly like the
|
||||
MBID resolver plugs in for ListenBrainz."""
|
||||
def resolve_fn(artist: str, title: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
tid = db_service_track_id(artist, title, service)
|
||||
return (tid, "library" if tid else None)
|
||||
return resolve_fn
|
||||
|
||||
|
||||
def service_id_from_extra_data(track: Any, service: str) -> Optional[str]:
|
||||
"""The target-service track ID the DISCOVERY step already resolved for this mirrored
|
||||
track, read from its ``extra_data`` blob (#945 — Boulder: "all 50 are discovered to
|
||||
Deezer already, it's not using any of that"). This is free (no API call) and reliable
|
||||
(it's the same id used to mirror the track).
|
||||
|
||||
Only trusted when the track was discovered ON the export's target service — a
|
||||
Deezer-discovered track carries a Deezer id under ``matched_data['id']``, and its
|
||||
``provider`` is the service name. A ``wing_it_fallback`` provider (the low-confidence
|
||||
guess path) deliberately does NOT match here, so those fall through to the library/
|
||||
none path rather than risk a wrong track in the exported playlist."""
|
||||
raw = track.get("extra_data") if isinstance(track, dict) else None
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(raw) if isinstance(raw, str) else raw
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(data, dict) or not data.get("discovered"):
|
||||
return None
|
||||
if str(data.get("provider") or "").lower() != str(service or "").lower():
|
||||
return None
|
||||
matched = data.get("matched_data")
|
||||
tid = matched.get("id") if isinstance(matched, dict) else None
|
||||
return str(tid) if tid else None
|
||||
|
||||
|
||||
def _track_field(track: Dict[str, Any], *names: str) -> str:
|
||||
for n in names:
|
||||
v = track.get(n)
|
||||
if v:
|
||||
return str(v)
|
||||
return ""
|
||||
|
||||
|
||||
def resolve_service_track_ids(
|
||||
tracks: List[Dict[str, Any]],
|
||||
service: str,
|
||||
*,
|
||||
db_fn: Optional[Callable[[str, str, str], Optional[str]]] = None,
|
||||
search_id_fn: Optional[Callable[[str, str], Optional[str]]] = None,
|
||||
on_progress: Optional[Callable[[int, int, Dict[str, Any]], None]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Resolve a mirrored playlist's tracks to target-service track IDs for export.
|
||||
|
||||
Waterfall per track: the discovery cache (``extra_data`` — free + already confidently
|
||||
matched) → the library track's stored service id → (only when ``search_id_fn`` is
|
||||
given, i.e. the opt-in backfill toggle) a confident live-search match. A track that
|
||||
clears none of these is reported unmatched (caller skips it — never a guessed/wrong
|
||||
id). Returns ``{"resolved": [{artist, title, album, service_track_id}], "stats":
|
||||
{...}}`` with ``from_cache`` / ``from_library`` / ``from_search`` / ``unmatched``
|
||||
tallies for the status display.
|
||||
"""
|
||||
db_fn = db_fn or db_service_track_id
|
||||
total = len(tracks or [])
|
||||
resolved: List[Dict[str, Any]] = []
|
||||
stats: Dict[str, Any] = {
|
||||
"total": total, "resolved": 0, "unmatched": 0,
|
||||
"from_cache": 0, "from_library": 0, "from_search": 0,
|
||||
}
|
||||
for i, t in enumerate(tracks or []):
|
||||
if not isinstance(t, dict):
|
||||
t = {}
|
||||
artist = _track_field(t, "artist", "artist_name", "creator")
|
||||
title = _track_field(t, "title", "track_name", "name")
|
||||
album = _track_field(t, "album", "album_name", "release_name")
|
||||
|
||||
tid = service_id_from_extra_data(t, service)
|
||||
if tid:
|
||||
stats["from_cache"] += 1
|
||||
else:
|
||||
tid = db_fn(artist, title, service)
|
||||
if tid:
|
||||
stats["from_library"] += 1
|
||||
elif search_id_fn is not None:
|
||||
tid = search_id_fn(artist, title)
|
||||
if tid:
|
||||
stats["from_search"] += 1
|
||||
|
||||
resolved.append({"artist": artist, "title": title, "album": album,
|
||||
"service_track_id": tid or None})
|
||||
stats["resolved" if tid else "unmatched"] += 1
|
||||
if on_progress is not None:
|
||||
try:
|
||||
on_progress(i + 1, total, stats)
|
||||
except Exception: # noqa: S110 — a progress error must never fail the export
|
||||
pass
|
||||
|
||||
return {"resolved": resolved, "stats": stats}
|
||||
|
||||
|
||||
# Confidence floor for backfill, on the score_track scale (~1.5 = exact title + exact
|
||||
# artist, thanks to the 1.5x artist boost). A cover/karaoke (x0.05) or a wrong-artist hit
|
||||
# (no boost, caps ~1.0) can't clear this — so backfill never adds a guessed/wrong version.
|
||||
BACKFILL_MIN_SCORE = 1.2
|
||||
|
||||
|
||||
def search_service_track_id(
|
||||
artist: str,
|
||||
title: str,
|
||||
*,
|
||||
search_fn: Callable[[str], List[Any]],
|
||||
min_score: float = BACKFILL_MIN_SCORE,
|
||||
) -> Optional[str]:
|
||||
"""Confident live-search match for export backfill (#945): search the target service
|
||||
for (artist, title), rerank by relevance, and return the top match's id ONLY if it
|
||||
clears the confidence floor. Below the floor → None: the track is left out of the
|
||||
export rather than risk a wrong/cover/karaoke version (the whole point of backfill is
|
||||
coverage WITHOUT the wrong-track risk). ``search_fn(query) -> List[Track]`` is injected
|
||||
so this is unit-testable without a live service."""
|
||||
if not title:
|
||||
return None
|
||||
from core.metadata.relevance import build_combined_search_query, filter_and_rerank
|
||||
query = build_combined_search_query(title, artist)
|
||||
try:
|
||||
candidates = list(search_fn(query) or [])
|
||||
except Exception as exc:
|
||||
logger.debug(f"export backfill search failed for '{artist} - {title}': {exc}")
|
||||
return None
|
||||
if not candidates:
|
||||
return None
|
||||
ranked = filter_and_rerank(
|
||||
candidates, expected_title=title, expected_artist=artist, min_score=min_score,
|
||||
)
|
||||
if not ranked:
|
||||
return None
|
||||
tid = getattr(ranked[0], "id", None)
|
||||
return str(tid) if tid else None
|
||||
|
||||
|
||||
def file_recording_mbid(artist: str, title: str) -> Optional[str]:
|
||||
"""Recording MBID read from the matched track's file tag (set on import post-processing)."""
|
||||
_mbid, fpath = _db_match(artist, title)
|
||||
if not fpath:
|
||||
return None
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(fpath)
|
||||
if audio is None or not getattr(audio, "tags", None):
|
||||
return None
|
||||
tags = audio.tags
|
||||
# ID3 UFID (MusicBrainz), Vorbis/MP4 musicbrainz_trackid, etc.
|
||||
for key in ("UFID:http://musicbrainz.org", "musicbrainz_trackid",
|
||||
"MUSICBRAINZ_TRACKID", "----:com.apple.iTunes:MusicBrainz Track Id"):
|
||||
try:
|
||||
val = tags.get(key)
|
||||
except Exception:
|
||||
val = None
|
||||
if not val:
|
||||
continue
|
||||
if hasattr(val, "data"): # ID3 UFID frame
|
||||
val = val.data.decode("utf-8", "ignore")
|
||||
if isinstance(val, (list, tuple)):
|
||||
val = val[0] if val else ""
|
||||
if isinstance(val, bytes):
|
||||
val = val.decode("utf-8", "ignore")
|
||||
val = str(val).strip()
|
||||
if val:
|
||||
return val
|
||||
except Exception as exc:
|
||||
logger.debug(f"export file_recording_mbid failed for {fpath}: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
_mb_service = None
|
||||
_mb_service_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_mb_service():
|
||||
"""Shared MusicBrainzService (client + cache + DB), created lazily so importing this
|
||||
module never triggers a DB/network connection on paths that don't export."""
|
||||
global _mb_service
|
||||
if _mb_service is None:
|
||||
with _mb_service_lock:
|
||||
if _mb_service is None:
|
||||
from core.musicbrainz_service import MusicBrainzService
|
||||
from database.music_database import get_database
|
||||
_mb_service = MusicBrainzService(get_database())
|
||||
return _mb_service
|
||||
|
||||
|
||||
def musicbrainz_recording_mbid(artist: str, title: str) -> Optional[str]:
|
||||
"""Live MusicBrainz ``match_recording`` — the rate-limited tail."""
|
||||
if not title:
|
||||
return None
|
||||
try:
|
||||
svc = _get_mb_service()
|
||||
if not svc:
|
||||
return None
|
||||
result = svc.match_recording(title, artist)
|
||||
if result and result.get("mbid"):
|
||||
return result["mbid"]
|
||||
except Exception as exc:
|
||||
logger.debug(f"export musicbrainz_recording_mbid failed for '{artist} - {title}': {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def build_resolve_fn(
|
||||
*,
|
||||
db_fn: Callable[[str, str], Optional[str]] = db_recording_mbid,
|
||||
file_fn: Callable[[str, str], Optional[str]] = file_recording_mbid,
|
||||
mb_fn: Callable[[str, str], Optional[str]] = musicbrainz_recording_mbid,
|
||||
cache_lookup: Optional[Callable[[str], Optional[str]]] = None,
|
||||
cache_record: Optional[Callable[[str, str], bool]] = None,
|
||||
) -> Callable[[str, str], Tuple[Optional[str], Optional[str]]]:
|
||||
"""Assemble the export ``resolve_fn(artist, title) -> (mbid, source_label)``.
|
||||
|
||||
Runs cache -> DB -> file -> MusicBrainz, and writes a fresh (non-cache) hit back to the
|
||||
persistent cache. All sources are injectable so the wiring is unit-testable; defaults
|
||||
use the real cache module.
|
||||
"""
|
||||
if cache_lookup is None or cache_record is None:
|
||||
from core.exports import recording_mbid_cache as _cache
|
||||
cache_lookup = cache_lookup or _cache.lookup
|
||||
cache_record = cache_record or _cache.record
|
||||
|
||||
def resolve_fn(artist: str, title: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
sources = [
|
||||
(SRC_CACHE, lambda a, t: cache_lookup(normalize_key(a, t))),
|
||||
(SRC_DB, db_fn),
|
||||
(SRC_FILE, file_fn),
|
||||
(SRC_MUSICBRAINZ, mb_fn),
|
||||
]
|
||||
mbid, label = resolve_recording_mbid(artist, title, sources)
|
||||
if mbid and label and label != SRC_CACHE:
|
||||
try:
|
||||
cache_record(normalize_key(artist, title), mbid)
|
||||
except Exception: # noqa: S110 — cache write is best-effort
|
||||
pass
|
||||
return (mbid, label)
|
||||
|
||||
return resolve_fn
|
||||
|
||||
|
||||
__all__ = [
|
||||
"build_resolve_fn",
|
||||
"db_recording_mbid",
|
||||
"file_recording_mbid",
|
||||
"musicbrainz_recording_mbid",
|
||||
]
|
||||
89
core/exports/jspf_export.py
Normal file
89
core/exports/jspf_export.py
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
"""Build a JSPF playlist (ListenBrainz-compatible) from resolved SoulSync tracks.
|
||||
|
||||
ListenBrainz's ``POST /1/playlist/create`` requires JSPF where **every track carries a
|
||||
``identifier`` of ``https://musicbrainz.org/recording/<recording-mbid>``** — text-only
|
||||
entries (title/creator alone) are rejected. So a track can only be exported once we've
|
||||
resolved its MusicBrainz *recording* MBID (see ``mbid_resolver``); tracks without one are
|
||||
dropped here and surfaced to the user as "unmatched".
|
||||
|
||||
Pure + I/O-free: callers pass already-resolved track dicts, this returns the JSPF dict
|
||||
(and a small coverage summary). The same JSPF is used for both the downloadable ``.jspf``
|
||||
file and the direct create-playlist POST, so there's one source of truth for the shape.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
MB_RECORDING_PREFIX = "https://musicbrainz.org/recording/"
|
||||
|
||||
# A MusicBrainz MBID is a canonical UUID. Validate to avoid emitting garbage identifiers
|
||||
# that LB would reject (or, worse, that silently point nowhere).
|
||||
_UUID_RE = re.compile(
|
||||
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def is_valid_recording_mbid(mbid: Any) -> bool:
|
||||
"""True when ``mbid`` is a well-formed MusicBrainz UUID."""
|
||||
return bool(mbid) and isinstance(mbid, str) and bool(_UUID_RE.match(mbid.strip()))
|
||||
|
||||
|
||||
def _track_entry(track: Dict[str, Any]) -> Dict[str, Any] | None:
|
||||
"""Build one JSPF track entry, or None if the track has no valid recording MBID."""
|
||||
mbid = (track.get("recording_mbid") or "").strip() if isinstance(track.get("recording_mbid"), str) else ""
|
||||
if not is_valid_recording_mbid(mbid):
|
||||
return None
|
||||
entry: Dict[str, Any] = {"identifier": f"{MB_RECORDING_PREFIX}{mbid}"}
|
||||
# Optional, human-friendly fields — LB ignores them on create but they make the
|
||||
# downloaded .jspf readable and round-trippable.
|
||||
if track.get("title"):
|
||||
entry["title"] = str(track["title"])
|
||||
if track.get("artist"):
|
||||
entry["creator"] = str(track["artist"])
|
||||
if track.get("album"):
|
||||
entry["album"] = str(track["album"])
|
||||
return entry
|
||||
|
||||
|
||||
def build_jspf(
|
||||
title: str,
|
||||
tracks: List[Dict[str, Any]],
|
||||
*,
|
||||
creator: str = "",
|
||||
) -> Tuple[Dict[str, Any], Dict[str, int]]:
|
||||
"""Build a ListenBrainz-compatible JSPF dict from resolved tracks.
|
||||
|
||||
``tracks`` is an ordered list of dicts with ``recording_mbid`` (required to be
|
||||
included), plus optional ``title`` / ``artist`` / ``album``. Tracks without a valid
|
||||
recording MBID are skipped (LB rejects them).
|
||||
|
||||
Returns ``(jspf, summary)`` where ``jspf`` is ``{"playlist": {...}}`` and ``summary``
|
||||
is ``{"total", "included", "skipped"}`` for the coverage display.
|
||||
"""
|
||||
jspf_tracks: List[Dict[str, Any]] = []
|
||||
for t in tracks or []:
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
entry = _track_entry(t)
|
||||
if entry is not None:
|
||||
jspf_tracks.append(entry)
|
||||
|
||||
playlist: Dict[str, Any] = {
|
||||
"title": (title or "SoulSync Export").strip() or "SoulSync Export",
|
||||
"track": jspf_tracks,
|
||||
}
|
||||
if creator:
|
||||
playlist["creator"] = str(creator)
|
||||
|
||||
total = sum(1 for t in (tracks or []) if isinstance(t, dict))
|
||||
summary = {
|
||||
"total": total,
|
||||
"included": len(jspf_tracks),
|
||||
"skipped": total - len(jspf_tracks),
|
||||
}
|
||||
return {"playlist": playlist}, summary
|
||||
|
||||
|
||||
__all__ = ["build_jspf", "is_valid_recording_mbid", "MB_RECORDING_PREFIX"]
|
||||
88
core/exports/mbid_resolver.py
Normal file
88
core/exports/mbid_resolver.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
"""Resolve a playlist track's MusicBrainz *recording* MBID, cheapest source first.
|
||||
|
||||
A ListenBrainz playlist export needs each track's recording MBID (``jspf_export``). A
|
||||
SoulSync track can supply it from several places, in increasing cost:
|
||||
|
||||
1. **resolution cache** — a prior (artist,title)->mbid result (persistent; reused across
|
||||
playlists and runs, so the same song never costs twice).
|
||||
2. **library DB** — ``tracks.musicbrainz_recording_id`` (set by the MusicBrainz
|
||||
enrichment worker).
|
||||
3. **file tags** — ``MUSICBRAINZ_RECORDING_ID`` written into the audio file on import
|
||||
post-processing (catches tracks enriched at import but not via the worker).
|
||||
4. **MusicBrainz lookup** — a live ``match_recording(artist, title)`` (rate-limited
|
||||
~1 req/s; the slow tail — only hit when 1–3 miss).
|
||||
|
||||
This module is the **pure waterfall**: the caller passes ordered ``(label, fn)`` sources,
|
||||
each ``fn(artist, title) -> mbid | None``, and ``resolve_recording_mbid`` returns the
|
||||
first valid hit plus its label (for the live status / stats). The actual I/O (DB query,
|
||||
mutagen read, MB request, cache read/write) lives in the export job that wires the real
|
||||
sources — so this stays trivially unit-testable and short-circuits correctly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Callable, List, Optional, Tuple
|
||||
|
||||
# Source labels (also used in the live-status breakdown).
|
||||
SRC_CACHE = "cache"
|
||||
SRC_DB = "db"
|
||||
SRC_FILE = "file"
|
||||
SRC_MUSICBRAINZ = "musicbrainz"
|
||||
SRC_NONE = None
|
||||
|
||||
_UUID_RE = re.compile(
|
||||
r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE
|
||||
)
|
||||
|
||||
Source = Tuple[str, Callable[[str, str], Optional[str]]]
|
||||
|
||||
|
||||
def _valid(mbid: Any) -> Optional[str]:
|
||||
"""Return the trimmed MBID if it's a well-formed UUID, else None."""
|
||||
if not isinstance(mbid, str):
|
||||
return None
|
||||
m = mbid.strip()
|
||||
return m if _UUID_RE.match(m) else None
|
||||
|
||||
|
||||
def normalize_key(artist: Any, title: Any) -> str:
|
||||
"""Stable cache key for an (artist, title) pair — lower, punctuation-stripped,
|
||||
whitespace-collapsed — so trivial variations share a cache entry."""
|
||||
def _n(v: Any) -> str:
|
||||
s = re.sub(r"[^\w\s]", "", str(v or "").lower())
|
||||
return re.sub(r"\s+", " ", s).strip()
|
||||
return f"{_n(artist)}␟{_n(title)}"
|
||||
|
||||
|
||||
def resolve_recording_mbid(
|
||||
artist: str,
|
||||
title: str,
|
||||
sources: List[Source],
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Walk ``sources`` in order; return ``(mbid, label)`` of the first that yields a
|
||||
valid recording MBID, or ``(None, None)`` when every source misses.
|
||||
|
||||
Each source is ``(label, fn)`` and ``fn(artist, title)`` returns an MBID or None. A
|
||||
source that raises is treated as a miss (never aborts the waterfall) — so one flaky
|
||||
lookup (e.g. a MusicBrainz timeout) can't fail the whole export. Short-circuits: a
|
||||
later/expensive source isn't called once an earlier one hits.
|
||||
"""
|
||||
for label, fn in sources or []:
|
||||
try:
|
||||
mbid = _valid(fn(artist, title))
|
||||
except Exception:
|
||||
mbid = None
|
||||
if mbid:
|
||||
return (mbid, label)
|
||||
return (None, None)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"resolve_recording_mbid",
|
||||
"normalize_key",
|
||||
"SRC_CACHE",
|
||||
"SRC_DB",
|
||||
"SRC_FILE",
|
||||
"SRC_MUSICBRAINZ",
|
||||
]
|
||||
98
core/exports/playlist_export.py
Normal file
98
core/exports/playlist_export.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
"""Orchestrate resolving a playlist's tracks to recording MBIDs for export (#903).
|
||||
|
||||
This is the testable heart of the export job: walk the playlist's tracks, resolve each to a
|
||||
MusicBrainz recording MBID via an injected ``resolve_fn`` (which the job wires to the
|
||||
cache -> DB -> file -> MusicBrainz waterfall), dedup repeated songs within the run so they
|
||||
only cost one resolution, build the ordered "pseudo-playlist" of resolved tracks, and tally
|
||||
live stats (resolved / unmatched / per-source / deduped) for the on-card status display.
|
||||
|
||||
Pure: all I/O (DB, file reads, MusicBrainz, cache) is behind ``resolve_fn`` and the optional
|
||||
``on_progress`` callback, so the dedup + accounting logic is unit-testable without any
|
||||
network or database. The returned ``resolved`` list feeds straight into ``jspf_export``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from core.exports.mbid_resolver import normalize_key
|
||||
|
||||
# resolve_fn(artist, title) -> (recording_mbid|None, source_label|None)
|
||||
ResolveFn = Callable[[str, str], Tuple[Optional[str], Optional[str]]]
|
||||
ProgressFn = Callable[[int, int, Dict[str, Any]], None]
|
||||
|
||||
|
||||
def _field(track: Dict[str, Any], *names: str) -> str:
|
||||
"""First non-empty value among ``names`` (handles both playlist + LB-cache shapes)."""
|
||||
for n in names:
|
||||
v = track.get(n)
|
||||
if v:
|
||||
return str(v)
|
||||
return ""
|
||||
|
||||
|
||||
def resolve_playlist_tracks(
|
||||
tracks: List[Dict[str, Any]],
|
||||
resolve_fn: ResolveFn,
|
||||
*,
|
||||
on_progress: Optional[ProgressFn] = None,
|
||||
id_key: str = "recording_mbid",
|
||||
) -> Dict[str, Any]:
|
||||
"""Resolve every track to an ID and build the export pseudo-playlist.
|
||||
|
||||
``resolve_fn(artist, title) -> (id, source)`` returns whatever ID the target needs —
|
||||
a MusicBrainz recording MBID for ListenBrainz/JSPF (the default), or a Spotify/Deezer
|
||||
track ID for service export. ``id_key`` names the field that ID lands under in each
|
||||
resolved entry (defaults to ``recording_mbid`` so existing LB/JSPF callers are
|
||||
untouched). The dedup + stats + ordering logic is identical regardless of ID type.
|
||||
|
||||
``tracks`` items may use ``artist``/``artist_name`` and ``title``/``track_name`` and
|
||||
``album``/``album_name`` (both the mirrored-playlist and LB-cache shapes are accepted).
|
||||
|
||||
Returns ``{"resolved": [...], "stats": {...}}`` where each resolved entry is
|
||||
``{artist, title, album, <id_key>}`` (the ID is None when unmatched), in original
|
||||
order, and stats carries ``total, resolved, unmatched, deduped, by_source``.
|
||||
"""
|
||||
total = len(tracks or [])
|
||||
memo: Dict[str, Tuple[Optional[str], Optional[str]]] = {}
|
||||
resolved: List[Dict[str, Any]] = []
|
||||
stats: Dict[str, Any] = {
|
||||
"total": total, "resolved": 0, "unmatched": 0, "deduped": 0, "by_source": {},
|
||||
}
|
||||
|
||||
for i, t in enumerate(tracks or []):
|
||||
if not isinstance(t, dict):
|
||||
t = {}
|
||||
artist = _field(t, "artist", "artist_name", "creator")
|
||||
title = _field(t, "title", "track_name", "name")
|
||||
album = _field(t, "album", "album_name", "release_name")
|
||||
key = normalize_key(artist, title)
|
||||
|
||||
if key in memo:
|
||||
mbid, source = memo[key]
|
||||
stats["deduped"] += 1
|
||||
fresh = False
|
||||
else:
|
||||
mbid, source = resolve_fn(artist, title)
|
||||
memo[key] = (mbid, source)
|
||||
fresh = True
|
||||
|
||||
resolved.append({"artist": artist, "title": title, "album": album, id_key: mbid})
|
||||
|
||||
if mbid:
|
||||
stats["resolved"] += 1
|
||||
if fresh and source:
|
||||
stats["by_source"][source] = stats["by_source"].get(source, 0) + 1
|
||||
else:
|
||||
stats["unmatched"] += 1
|
||||
|
||||
if on_progress is not None:
|
||||
try:
|
||||
on_progress(i + 1, total, stats)
|
||||
except Exception: # noqa: S110 — a progress-display error must never fail the export
|
||||
pass
|
||||
|
||||
return {"resolved": resolved, "stats": stats}
|
||||
|
||||
|
||||
__all__ = ["resolve_playlist_tracks"]
|
||||
126
core/exports/recording_mbid_cache.py
Normal file
126
core/exports/recording_mbid_cache.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
"""Persistent (artist,title) -> MusicBrainz recording-MBID cache for playlist export.
|
||||
|
||||
The export waterfall (``core.exports.mbid_resolver``) ends in a live MusicBrainz lookup
|
||||
that's rate-limited to ~1 req/s — the slow tail of exporting a big playlist. Remembering a
|
||||
resolved recording MBID ONCE means the same song never costs a second lookup, across every
|
||||
future export and every playlist it appears in.
|
||||
|
||||
Mirrors ``core.metadata.album_mbid_cache`` exactly: a tiny SQLite table, lazy DB accessor,
|
||||
every function wrapped so any DB error degrades to a cache miss / no-op. If this module
|
||||
breaks, exports still work — they just re-resolve via the live waterfall like a cold cache.
|
||||
Key is the normalized ``track_key`` from ``mbid_resolver.normalize_key(artist, title)``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("exports.recording_mbid_cache")
|
||||
|
||||
_db_factory_lock = threading.Lock()
|
||||
_db_factory = None
|
||||
|
||||
|
||||
def _get_database():
|
||||
"""Resolve the MusicDatabase singleton lazily; None on any failure (treated as miss)."""
|
||||
global _db_factory
|
||||
with _db_factory_lock:
|
||||
if _db_factory is None:
|
||||
try:
|
||||
from database.music_database import get_database
|
||||
_db_factory = get_database
|
||||
except Exception as exc:
|
||||
logger.warning(f"Recording-MBID cache: could not load database module: {exc}")
|
||||
return None
|
||||
try:
|
||||
return _db_factory()
|
||||
except Exception as exc:
|
||||
logger.warning(f"Recording-MBID cache: database accessor failed: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def lookup(track_key: str) -> Optional[str]:
|
||||
"""Read a cached recording MBID for ``track_key``; None on miss or any DB error."""
|
||||
if not track_key:
|
||||
return None
|
||||
db = _get_database()
|
||||
if db is None:
|
||||
return None
|
||||
conn = None
|
||||
try:
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT recording_mbid FROM mb_recording_cache WHERE track_key = ? LIMIT 1",
|
||||
(track_key,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
return (row[0] if not hasattr(row, "keys") else row["recording_mbid"]) or None
|
||||
except Exception as exc:
|
||||
logger.debug(f"Recording-MBID cache lookup failed: {exc}")
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110 — finally cleanup
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def record(track_key: str, recording_mbid: str) -> bool:
|
||||
"""Persist ``track_key`` -> ``recording_mbid`` (idempotent). False on any failure."""
|
||||
if not track_key or not recording_mbid:
|
||||
return False
|
||||
db = _get_database()
|
||||
if db is None:
|
||||
return False
|
||||
conn = None
|
||||
try:
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO mb_recording_cache "
|
||||
"(track_key, recording_mbid, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)",
|
||||
(track_key, recording_mbid),
|
||||
)
|
||||
conn.commit()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug(f"Recording-MBID cache record failed: {exc}")
|
||||
return False
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110 — finally cleanup
|
||||
pass
|
||||
|
||||
|
||||
def clear_all() -> bool:
|
||||
"""Wipe the cache (tests / forced re-resolve)."""
|
||||
db = _get_database()
|
||||
if db is None:
|
||||
return False
|
||||
conn = None
|
||||
try:
|
||||
conn = db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM mb_recording_cache")
|
||||
conn.commit()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning(f"Recording-MBID cache clear failed: {exc}")
|
||||
return False
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # noqa: S110 — finally cleanup
|
||||
pass
|
||||
|
||||
|
||||
__all__ = ["lookup", "record", "clear_all"]
|
||||
|
|
@ -36,9 +36,34 @@ import requests as http_requests
|
|||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.quality.source_map import quality_from_tidal_tier, quality_tier_for_source
|
||||
|
||||
logger = get_logger("hifi_client")
|
||||
|
||||
# A media playlist whose total runtime is below this fraction of the track's
|
||||
# real duration is a preview (some Monochrome instances only have 30s Tidal
|
||||
# DOWNLOAD access — a 220s track comes back as ~30s of segments + ENDLIST).
|
||||
_PREVIEW_DURATION_RATIO = 0.85
|
||||
_EXTINF_RE = re.compile(r'#EXTINF:\s*([0-9.]+)')
|
||||
|
||||
|
||||
def hls_total_seconds(playlist_text: str) -> float:
|
||||
"""Sum the ``#EXTINF`` segment durations in an HLS media playlist."""
|
||||
return sum(float(x) for x in _EXTINF_RE.findall(playlist_text or ''))
|
||||
|
||||
|
||||
def is_preview_playlist(playlist_s: float, track_s: float,
|
||||
ratio: float = _PREVIEW_DURATION_RATIO) -> bool:
|
||||
"""True when the playlist runtime is far shorter than the track's real
|
||||
duration (a preview). Returns False when either duration is unknown, so a
|
||||
missing reference never false-positives — the post-download audio guard is
|
||||
the safety net.
|
||||
"""
|
||||
if not playlist_s or not track_s or track_s <= 0:
|
||||
return False
|
||||
return playlist_s < track_s * ratio
|
||||
|
||||
|
||||
# HLS quality presets mapping to /trackManifests/ format parameters
|
||||
HLS_QUALITY_MAP = {
|
||||
'hires': {
|
||||
|
|
@ -140,6 +165,81 @@ def compute_new_default_pushes(all_defaults, offered, legacy_baseline, existing)
|
|||
return to_add, new_offered
|
||||
|
||||
|
||||
_EXTINF_RE = re.compile(r'#EXTINF:\s*([0-9]+(?:\.[0-9]+)?)')
|
||||
|
||||
|
||||
def sum_hls_segment_seconds(playlist_text: str) -> float:
|
||||
"""Total audio seconds an HLS media playlist actually provides — the sum of its
|
||||
``#EXTINF`` segment durations. This is the authoritative "how much audio is really
|
||||
here" signal: a PREVIEW manifest serves only ~30s of segments even though the track
|
||||
is full-length, so summing EXTINF catches it before we waste the download. Returns
|
||||
0.0 when the playlist has no EXTINF lines (master playlists, legacy manifests) — the
|
||||
caller treats 0 as 'unknown', never as 'preview'."""
|
||||
total = 0.0
|
||||
for m in _EXTINF_RE.finditer(playlist_text or ''):
|
||||
try:
|
||||
total += float(m.group(1))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return total
|
||||
|
||||
|
||||
def is_short_audio(actual_seconds: float, expected_seconds: float, threshold: float = 0.8) -> bool:
|
||||
"""True when ``actual`` is meaningfully shorter than ``expected`` — i.e. a preview
|
||||
clip or a truncated/corrupt download. Conservative: returns False whenever either
|
||||
value is missing/zero (unknown ⇒ never reject), and only trips below ``threshold``
|
||||
of the expected length (previews are ~15% of full, so the margin is huge)."""
|
||||
try:
|
||||
a, e = float(actual_seconds or 0), float(expected_seconds or 0)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if a <= 0 or e <= 0:
|
||||
return False
|
||||
return a < e * threshold
|
||||
|
||||
|
||||
def is_fake_lossless_bitrate(size_bytes, claimed_seconds, sample_rate, bits_per_sample,
|
||||
channels, min_ratio: float = 0.30) -> bool:
|
||||
"""True when a 'lossless' file's data is FAR too small for its claimed length — the
|
||||
fingerprint of a ~30s preview whose STREAMINFO/container was faked to the full
|
||||
duration (so every length header reads 'full' and only the bitrate gives it away).
|
||||
Real FLAC is ~40-75% of raw PCM; a preview padded to full length implies single-digit
|
||||
%. Conservative: 0 / bad inputs return False (never reject on unknowns)."""
|
||||
try:
|
||||
sz, secs = float(size_bytes or 0), float(claimed_seconds or 0)
|
||||
sr, bits, ch = int(sample_rate or 0), int(bits_per_sample or 0), int(channels or 0)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if sz <= 0 or secs <= 0 or sr <= 0 or bits <= 0 or ch <= 0:
|
||||
return False
|
||||
return (sz * 8 / secs) < (sr * bits * ch) * min_ratio
|
||||
|
||||
|
||||
def parse_ffmpeg_time(stderr_text) -> float:
|
||||
"""The last ``time=HH:MM:SS.xx`` ffmpeg prints while decoding — the REAL decoded
|
||||
length (immune to a faked container/STREAMINFO duration). 0.0 if not found."""
|
||||
last = 0.0
|
||||
for m in re.finditer(r'time=(\d+):(\d+):(\d+(?:\.\d+)?)', stderr_text or ''):
|
||||
last = int(m.group(1)) * 3600 + int(m.group(2)) * 60 + float(m.group(3))
|
||||
return last
|
||||
|
||||
|
||||
def is_preview_download(real_seconds, reference_seconds, *, is_lossless, size_bytes,
|
||||
sample_rate, bits_per_sample, channels):
|
||||
"""Is a finished file a preview/truncated fake? Two independent signals, so it fires
|
||||
even when the fakery declares full length at every layer:
|
||||
1. DECODED length far below the reference (the ground truth, when a decoder ran);
|
||||
2. for lossless, an impossibly-low implied bitrate (no decoder needed).
|
||||
Returns ``(is_fake, reason)``."""
|
||||
if real_seconds and is_short_audio(real_seconds, reference_seconds):
|
||||
return True, "decoded %.0fs of %.0fs" % (real_seconds, reference_seconds)
|
||||
if is_lossless and is_fake_lossless_bitrate(size_bytes, reference_seconds, sample_rate,
|
||||
bits_per_sample, channels):
|
||||
kbps = (float(size_bytes) * 8 / reference_seconds / 1000) if reference_seconds else 0
|
||||
return True, "%.0fkbps lossless over %.0fs (far too low — a ~30s preview)" % (kbps, reference_seconds)
|
||||
return False, ""
|
||||
|
||||
|
||||
# Run the new-default push at most once per process.
|
||||
_pushed_new_defaults = False
|
||||
|
||||
|
|
@ -157,7 +257,10 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
|
||||
self._instances = []
|
||||
self._instance_lock = threading.Lock()
|
||||
|
|
@ -622,7 +725,8 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
|
||||
return init_uri, segment_uris
|
||||
|
||||
def _get_hls_manifest(self, track_id: int, quality: str = 'lossless') -> Optional[Dict]:
|
||||
def _get_hls_manifest(self, track_id: int, quality: str = 'lossless',
|
||||
expected_duration_s: float = 0) -> Optional[Dict]:
|
||||
q_info = HLS_QUALITY_MAP.get(quality, HLS_QUALITY_MAP['lossless'])
|
||||
formats = q_info['formats']
|
||||
|
||||
|
|
@ -662,6 +766,7 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
logger.warning(f"Failed to parse HLS playlist for track {track_id}: {e}")
|
||||
return None
|
||||
|
||||
media_text = playlist_text # the playlist that actually carries the EXTINF segments
|
||||
if '#EXT-X-STREAM-INF' in playlist_text and segment_uris:
|
||||
playlist_uri = segment_uris[0]
|
||||
try:
|
||||
|
|
@ -669,11 +774,26 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
variant_resp = self.session.get(playlist_uri, allow_redirects=True, timeout=30)
|
||||
variant_resp.raise_for_status()
|
||||
variant_text = variant_resp.text
|
||||
media_text = variant_text
|
||||
init_uri, segment_uris = self._parse_hls_playlist(variant_text, playlist_uri)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch variant playlist for track {track_id}: {e}")
|
||||
return None
|
||||
|
||||
# Preview detection — some instances only have 30s Tidal DOWNLOAD
|
||||
# access, returning a playlist far shorter than the real track. Decline
|
||||
# it (and rotate the instance) so the orchestrator falls through to a
|
||||
# real source instead of fetching a 30s file that gets quarantined.
|
||||
playlist_s = hls_total_seconds(media_text)
|
||||
if is_preview_playlist(playlist_s, expected_duration_s):
|
||||
logger.warning(
|
||||
f"HiFi manifest for track {track_id} ({quality}) is a "
|
||||
f"{playlist_s:.0f}s preview of a {expected_duration_s:.0f}s track — "
|
||||
f"declining this instance"
|
||||
)
|
||||
self._rotate_instance(self._current_instance)
|
||||
return None
|
||||
|
||||
if init_uri:
|
||||
logger.info(f"HiFi HLS manifest for track {track_id}: "
|
||||
f"init segment + {len(segment_uris)} segments ({quality})")
|
||||
|
|
@ -687,6 +807,9 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
'extension': q_info['extension'],
|
||||
'codec': q_info['codec'],
|
||||
'quality': quality,
|
||||
# Real audio length the manifest provides (sum of EXTINF) — used to reject
|
||||
# preview manifests before downloading. 0.0 = unknown (don't reject).
|
||||
'manifest_duration': sum_hls_segment_seconds(media_text),
|
||||
}
|
||||
|
||||
def _get_legacy_track_manifest(self, track_id: int, quality: str = 'lossless') -> Optional[Dict]:
|
||||
|
|
@ -711,15 +834,57 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
'quality': quality,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _probe_audio_seconds(path) -> float:
|
||||
"""Real decoded audio length of a finished file, via mutagen (already a dep).
|
||||
0.0 on any failure — the caller treats 0 as 'unknown' and never rejects on it."""
|
||||
try:
|
||||
from mutagen import File as _MutagenFile
|
||||
mf = _MutagenFile(str(path))
|
||||
info = getattr(mf, 'info', None) if mf is not None else None
|
||||
if info is not None:
|
||||
return float(getattr(info, 'length', 0) or 0)
|
||||
except Exception as _probe_err: # noqa: BLE001
|
||||
logger.debug("mutagen audio-length probe failed for %s: %s", path, _probe_err)
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _find_ffmpeg():
|
||||
ff = shutil.which('ffmpeg')
|
||||
if ff:
|
||||
return ff
|
||||
cand = Path(__file__).parent.parent / 'tools' / ('ffmpeg.exe' if os.name == 'nt' else 'ffmpeg')
|
||||
return str(cand) if cand.exists() else None
|
||||
|
||||
def _probe_real_seconds(self, path) -> float:
|
||||
"""REAL decoded audio length via ffmpeg — decodes the actual frames, so it sees
|
||||
through a faked STREAMINFO/container duration (a 30s preview claiming full
|
||||
length decodes to 30s). 0.0 if ffmpeg is unavailable or on error."""
|
||||
ff = self._find_ffmpeg()
|
||||
if not ff:
|
||||
return 0.0
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[ff, '-hide_banner', '-nostdin', '-i', str(path), '-map', '0:a:0', '-f', 'null', '-'],
|
||||
capture_output=True, text=True, timeout=180)
|
||||
return parse_ffmpeg_time(proc.stderr)
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _flac_props(path):
|
||||
"""(sample_rate, bits_per_sample, channels) for the bitrate sanity check, or None."""
|
||||
try:
|
||||
from mutagen.flac import FLAC
|
||||
si = FLAC(str(path)).info
|
||||
return (si.sample_rate, si.bits_per_sample, si.channels)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _demux_flac(self, input_path: Path, output_path: Path) -> None:
|
||||
ffmpeg = shutil.which('ffmpeg')
|
||||
ffmpeg = self._find_ffmpeg()
|
||||
if not ffmpeg:
|
||||
tools_dir = Path(__file__).parent.parent / 'tools'
|
||||
ffmpeg_candidate = tools_dir / ('ffmpeg.exe' if os.name == 'nt' else 'ffmpeg')
|
||||
if ffmpeg_candidate.exists():
|
||||
ffmpeg = str(ffmpeg_candidate)
|
||||
else:
|
||||
raise RuntimeError('ffmpeg is required to demux FLAC from MP4. Install ffmpeg and retry.')
|
||||
raise RuntimeError('ffmpeg is required to demux FLAC from MP4. Install ffmpeg and retry.')
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
|
|
@ -751,13 +916,18 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
loop = asyncio.get_event_loop()
|
||||
tracks = await loop.run_in_executor(None, lambda: self.search_raw(query))
|
||||
|
||||
quality_key = config_manager.get('hifi_download.quality', 'lossless')
|
||||
quality_key = quality_tier_for_source('hifi', default='lossless')
|
||||
q_info = HLS_QUALITY_MAP.get(quality_key, HLS_QUALITY_MAP['lossless'])
|
||||
|
||||
# HiFi is Tidal-backed; stamp the configured tier so the global
|
||||
# ranker sees real sample_rate/bit_depth, not just 'flac'.
|
||||
tier_quality = quality_from_tidal_tier(quality_key)
|
||||
|
||||
results = []
|
||||
for t in tracks:
|
||||
try:
|
||||
tr = self._to_track_result(t, q_info)
|
||||
tr.set_quality(tier_quality)
|
||||
results.append(tr)
|
||||
except Exception as e:
|
||||
logger.debug(f"Skipping track result conversion: {e}")
|
||||
|
|
@ -828,7 +998,7 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
)
|
||||
|
||||
def _download_sync(self, download_id: str, track_id: int, display_name: str) -> Optional[str]:
|
||||
quality_key = config_manager.get('hifi_download.quality', 'lossless')
|
||||
quality_key = quality_tier_for_source('hifi', default='lossless')
|
||||
chain = ['hires', 'lossless', 'high', 'low']
|
||||
start = chain.index(quality_key) if quality_key in chain else 1
|
||||
allow_fallback = config_manager.get('hifi_download.allow_fallback', True)
|
||||
|
|
@ -836,12 +1006,26 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
|
||||
MIN_AUDIO_SIZE = 100 * 1024
|
||||
|
||||
# Expected track length, drives every preview/truncation guard here:
|
||||
# * _get_hls_manifest's pre-download is_preview_playlist check
|
||||
# * the pre-download is_short_audio manifest check
|
||||
# * the post-download is_preview_download faked-header decode check
|
||||
# Best-effort: a 0 here just disables the duration checks, never rejects.
|
||||
expected_s = 0.0
|
||||
try:
|
||||
info = self.get_track_info(track_id) or {}
|
||||
expected_s = float(info.get('duration_s') or 0)
|
||||
except Exception:
|
||||
expected_s = 0.0
|
||||
expected_duration_s = expected_s # alias for _get_hls_manifest's param name
|
||||
|
||||
for q_key in chain:
|
||||
if self.shutdown_check and self.shutdown_check():
|
||||
logger.info("Shutdown detected, aborting HiFi download")
|
||||
return None
|
||||
|
||||
manifest_info = self._get_hls_manifest(track_id, quality=q_key)
|
||||
manifest_info = self._get_hls_manifest(track_id, quality=q_key,
|
||||
expected_duration_s=expected_duration_s)
|
||||
if (
|
||||
not manifest_info
|
||||
or (
|
||||
|
|
@ -852,6 +1036,18 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
logger.warning(f"No HLS manifest at quality {q_key}, trying next")
|
||||
continue
|
||||
|
||||
# Preview guard #1 (pre-download): a preview manifest serves only ~30s of
|
||||
# segments for a full-length track. A preview means THIS SOURCE only has a
|
||||
# preview of the track — lower quality tiers are the SAME preview — so abort
|
||||
# HiFi entirely and let the orchestrator fall through to the next SOURCE
|
||||
# (soulseek/youtube/…), rather than landing a lower-tier preview.
|
||||
manifest_s = float(manifest_info.get('manifest_duration') or 0)
|
||||
if is_short_audio(manifest_s, expected_s):
|
||||
logger.warning(
|
||||
"HiFi has only a PREVIEW of '%s' (manifest %.0fs of %.0fs at %s) — "
|
||||
"failing HiFi so the next source is tried", display_name, manifest_s, expected_s, q_key)
|
||||
return None
|
||||
|
||||
extension = manifest_info['extension']
|
||||
safe_name = re.sub(r'[<>:"/\\|?*]', '_', display_name)
|
||||
out_filename = f"{safe_name}.{extension}"
|
||||
|
|
@ -931,6 +1127,31 @@ class HiFiClient(DownloadSourcePlugin):
|
|||
out_path.unlink(missing_ok=True)
|
||||
continue
|
||||
|
||||
# Preview guard #2 (post-download): the real catch. HiFi previews fake the
|
||||
# FULL length in every header — manifest EXTINF, m4a moov, FLAC
|
||||
# total_samples — so only the DECODED audio (or, for lossless, the
|
||||
# bitrate) reveals the ~30s truth. Reference = the largest length any
|
||||
# header claims (so the file's own faked claim becomes the bar its real
|
||||
# audio must clear); is_preview_download decodes + bitrate-checks.
|
||||
ref_s = max(expected_s, self._probe_audio_seconds(out_path))
|
||||
real_s = self._probe_real_seconds(out_path)
|
||||
props = self._flac_props(out_path) if is_flac else None
|
||||
fake, why = is_preview_download(
|
||||
real_s, ref_s, is_lossless=is_flac, size_bytes=final_size,
|
||||
sample_rate=(props[0] if props else 0),
|
||||
bits_per_sample=(props[1] if props else 0),
|
||||
channels=(props[2] if props else 0))
|
||||
if fake:
|
||||
# A preview at this tier means the SOURCE only has a preview — every
|
||||
# lower tier is the same 30s clip (and the lossy ones dodge the
|
||||
# bitrate check). Abort HiFi so the orchestrator tries the next
|
||||
# SOURCE, instead of cascading down into an accepted lower-tier preview.
|
||||
logger.warning(
|
||||
"HiFi has only a PREVIEW of '%s' (%s at %s) — failing HiFi so the "
|
||||
"next source is tried", display_name, why, q_key)
|
||||
out_path.unlink(missing_ok=True)
|
||||
return None
|
||||
|
||||
logger.info(f"HiFi download complete ({q_key}): {out_path} "
|
||||
f"({final_size / (1024*1024):.1f} MB)")
|
||||
return str(out_path)
|
||||
|
|
|
|||
94
core/imports/album_position.py
Normal file
94
core/imports/album_position.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
"""Resolve a track's position WITHIN its album's track list.
|
||||
|
||||
The bug this fixes: a track auto-downloaded from the playlist pipeline / wishlist /
|
||||
watchlist is identified as belonging to an album, but the per-track position is
|
||||
unknown — Deezer's search/track and MusicBrainz's recording lookups don't carry a
|
||||
track position (only their album endpoint does). ``detect_album_info_web`` then
|
||||
leaves ``track_number = None``, the import pipeline falls through to the default-1
|
||||
floor, and the file lands as ``01/1`` even though the album is known
|
||||
(``core/imports/context.py``). Verified live: e.g. Deezer says "Obelisk" is track
|
||||
9 of *The Grand Mirage*, but it was tagged 1/1.
|
||||
|
||||
This is the pure matcher: given the album's track list (fetched by the caller via
|
||||
``core.metadata.album_tracks.get_album_tracks_for_source`` — so this stays
|
||||
source-agnostic and I/O-free) plus the track's own identifiers, return its real
|
||||
``(track_number, disc_number)``. Match priority is by reliability:
|
||||
|
||||
1. **ISRC** — an exact recording identity; trusted immediately.
|
||||
2. **source track id** — exact within this album.
|
||||
3. **normalized title** — last resort.
|
||||
|
||||
Returns ``(None, None)`` on no confident match, so the caller keeps its existing
|
||||
behaviour (never worse than today).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, List, Optional, Tuple
|
||||
|
||||
|
||||
def _norm_title(value: Any) -> str:
|
||||
"""Lower, strip punctuation, collapse whitespace — for tolerant title match."""
|
||||
s = re.sub(r"[^\w\s]", "", str(value or "").lower())
|
||||
return re.sub(r"\s+", " ", s).strip()
|
||||
|
||||
|
||||
def _pos_int(value: Any) -> Optional[int]:
|
||||
try:
|
||||
n = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return n if n >= 1 else None
|
||||
|
||||
|
||||
def resolve_track_position_in_album(
|
||||
album_tracks: List[dict],
|
||||
*,
|
||||
title: str = "",
|
||||
track_id: str = "",
|
||||
isrc: str = "",
|
||||
) -> Tuple[Optional[int], Optional[int]]:
|
||||
"""Return ``(track_number, disc_number)`` for this track within ``album_tracks``,
|
||||
or ``(None, None)`` when no confident match is found.
|
||||
|
||||
``album_tracks`` is the list under ``get_album_tracks_for_source(...)['tracks']``
|
||||
— each entry has ``track_number`` / ``disc_number`` / ``id`` / ``name`` / ``isrc``.
|
||||
Entries without a valid positive ``track_number`` are skipped. Pure: no I/O.
|
||||
"""
|
||||
if not album_tracks:
|
||||
return (None, None)
|
||||
|
||||
want_isrc = str(isrc or "").strip().upper()
|
||||
want_id = str(track_id or "").strip()
|
||||
want_title = _norm_title(title)
|
||||
|
||||
by_id: Optional[Tuple[int, int]] = None
|
||||
by_title: Optional[Tuple[int, int]] = None
|
||||
|
||||
for t in album_tracks:
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
tn = _pos_int(t.get("track_number"))
|
||||
if tn is None:
|
||||
continue
|
||||
dn = _pos_int(t.get("disc_number")) or 1
|
||||
|
||||
# 1) ISRC — exact recording. Win immediately.
|
||||
if want_isrc and str(t.get("isrc") or "").strip().upper() == want_isrc:
|
||||
return (tn, dn)
|
||||
# 2) source track id — exact within the album.
|
||||
if by_id is None and want_id and str(t.get("id") or "").strip() == want_id:
|
||||
by_id = (tn, dn)
|
||||
# 3) normalized title — last resort.
|
||||
if by_title is None and want_title and _norm_title(t.get("name")) == want_title:
|
||||
by_title = (tn, dn)
|
||||
|
||||
if by_id is not None:
|
||||
return by_id
|
||||
if by_title is not None:
|
||||
return by_title
|
||||
return (None, None)
|
||||
|
||||
|
||||
__all__ = ["resolve_track_position_in_album"]
|
||||
|
|
@ -433,16 +433,24 @@ def detect_album_info_web(context, artist_context=None):
|
|||
track_name.strip().lower(),
|
||||
artist_name.strip().lower(),
|
||||
}:
|
||||
_tn = track_info.get("track_number")
|
||||
_dn = track_info.get("disc_number")
|
||||
# The album is identified but discovery often doesn't carry the per-track
|
||||
# POSITION — Deezer's search/track and MusicBrainz's recording lookups omit
|
||||
# it (only their album endpoint has it). Without a position the pipeline
|
||||
# falls through to the default-1 floor and files an album track as 01/1
|
||||
# (e.g. Deezer says "Obelisk" is track 9 of The Grand Mirage). Resolve the
|
||||
# REAL position from the album's own track list when we have its id.
|
||||
# Fail-safe: leaves the numbers untouched on any miss, so behaviour is
|
||||
# never worse than the old preserve-None-and-fall-through.
|
||||
if _tn is None:
|
||||
_tn, _dn = _resolve_album_position_from_source(context, artist_context, _dn)
|
||||
return build_import_album_info(
|
||||
context,
|
||||
album_info={
|
||||
"album_name": album_name,
|
||||
# Preserve missing numbers as None so the import pipeline
|
||||
# can fall through to ``extract_track_number_from_filename``
|
||||
# at ``core/imports/pipeline.py:652`` instead of locking
|
||||
# to track/disc 01 for every wishlist re-attempt.
|
||||
"track_number": track_info.get("track_number"),
|
||||
"disc_number": track_info.get("disc_number"),
|
||||
"track_number": _tn,
|
||||
"disc_number": _dn,
|
||||
"album_image_url": album_ctx.get("image_url", ""),
|
||||
"confidence": 0.5,
|
||||
},
|
||||
|
|
@ -454,6 +462,48 @@ def detect_album_info_web(context, artist_context=None):
|
|||
return _resolve_single_to_parent_album(context, artist_context)
|
||||
|
||||
|
||||
def _resolve_album_position_from_source(context, artist_context, current_disc):
|
||||
"""Look up a track's real ``(track_number, disc_number)`` from its album's track
|
||||
list, for the case where the album is known but discovery didn't carry a
|
||||
position (Deezer/MusicBrainz search omit it).
|
||||
|
||||
Uses the SAME album id discovery already resolved (``get_import_source_ids`` →
|
||||
``album_id``), so it re-homes the track onto its own album with no re-search and
|
||||
no edition guessing. Matches by ISRC → source track id → title via the pure
|
||||
``core.imports.album_position`` seam. Returns ``(None, current_disc)`` on any
|
||||
miss/error so the caller falls back exactly as before — never worse than today.
|
||||
"""
|
||||
try:
|
||||
source = get_import_source(context)
|
||||
ids = get_import_source_ids(context)
|
||||
album_id = str(ids.get("album_id") or "")
|
||||
if not source or not album_id:
|
||||
return None, current_disc
|
||||
|
||||
from core.metadata.album_tracks import get_album_tracks_for_source
|
||||
payload = get_album_tracks_for_source(source, album_id) or {}
|
||||
tracks = payload.get("tracks") or []
|
||||
if not tracks:
|
||||
return None, current_disc
|
||||
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
title = (track_info.get("name") or original_search.get("title") or "").strip()
|
||||
isrc = str(track_info.get("isrc") or original_search.get("isrc") or "")
|
||||
|
||||
from core.imports.album_position import resolve_track_position_in_album
|
||||
tn, dn = resolve_track_position_in_album(
|
||||
tracks, title=title, track_id=str(ids.get("track_id") or ""), isrc=isrc)
|
||||
if tn is not None:
|
||||
logger.info("album-position: resolved '%s' to track %s/disc %s from album %s (%s)",
|
||||
title, tn, dn, album_id, source)
|
||||
return tn, (dn if dn is not None else current_disc)
|
||||
return None, current_disc
|
||||
except Exception as e:
|
||||
logger.debug("album-position resolution failed: %s", e)
|
||||
return None, current_disc
|
||||
|
||||
|
||||
def _resolve_single_to_parent_album(context, artist_context):
|
||||
"""A single-matched track -> a promoted album_info for its parent album, or
|
||||
None. GATED by ``metadata_enhancement.single_to_album`` (default OFF — it's a
|
||||
|
|
|
|||
|
|
@ -52,6 +52,14 @@ _DEFAULT_LENGTH_TOLERANCE_S = 3.0
|
|||
_LENGTH_TOLERANCE_LONG_TRACK_S = 5.0
|
||||
_LONG_TRACK_THRESHOLD_S = 600.0 # 10 minutes
|
||||
|
||||
# A file that runs LONGER than the expected metadata is the opposite of a truncated
|
||||
# download — it's almost always a different master/version (a remaster with a longer
|
||||
# outro, an extended fade, an album cut vs the radio edit). The duration check exists to
|
||||
# catch TRUNCATION (short files) and wildly-wrong matches, so on the auto default we allow
|
||||
# more drift in the longer direction and keep the tight bound for short files. A wrong-song
|
||||
# match still trips this — it's usually off by far more than 15s. (#937)
|
||||
_LONGER_VERSION_TOLERANCE_S = 15.0
|
||||
|
||||
# Upper bound for the user-configurable override. Anything past 60s
|
||||
# means the check is effectively off — cap defends against accidental
|
||||
# nonsense like 9999 making logs misleading. Users who genuinely want
|
||||
|
|
@ -242,18 +250,32 @@ def check_audio_integrity(
|
|||
if expected_length_s > _LONG_TRACK_THRESHOLD_S
|
||||
else _DEFAULT_LENGTH_TOLERANCE_S
|
||||
)
|
||||
user_pinned_tolerance = False
|
||||
else:
|
||||
user_pinned_tolerance = True
|
||||
checks["length_tolerance_s"] = length_tolerance_s
|
||||
|
||||
drift_s = abs(actual_length_s - expected_length_s)
|
||||
# Positive drift = the file runs LONGER than expected (not truncation). On the auto
|
||||
# default, give the longer direction more room so legit longer masters/versions aren't
|
||||
# quarantined (#937); a user-pinned tolerance is honoured symmetrically.
|
||||
signed_drift_s = actual_length_s - expected_length_s
|
||||
drift_s = abs(signed_drift_s)
|
||||
checks["length_drift_s"] = drift_s
|
||||
effective_tolerance_s = length_tolerance_s
|
||||
if signed_drift_s > 0 and not user_pinned_tolerance:
|
||||
effective_tolerance_s = max(length_tolerance_s, _LONGER_VERSION_TOLERANCE_S)
|
||||
checks["effective_tolerance_s"] = effective_tolerance_s
|
||||
|
||||
if drift_s > length_tolerance_s:
|
||||
if drift_s > effective_tolerance_s:
|
||||
runs_long = signed_drift_s > 0
|
||||
return IntegrityResult(
|
||||
ok=False,
|
||||
reason=f"Duration mismatch: file is {actual_length_s:.1f}s, "
|
||||
f"expected {expected_length_s:.1f}s "
|
||||
f"(drift {drift_s:.1f}s > tolerance {length_tolerance_s:.1f}s) — "
|
||||
"likely truncated download or wrong file matched",
|
||||
f"(drift {drift_s:.1f}s > tolerance {effective_tolerance_s:.1f}s) — "
|
||||
+ ("runs longer than expected — likely a different version/master or wrong file"
|
||||
if runs_long
|
||||
else "likely truncated download or wrong file matched"),
|
||||
checks=checks,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
|
@ -102,6 +103,41 @@ def cleanup_slskd_dedup_siblings(source_path) -> List[str]:
|
|||
return deleted
|
||||
|
||||
|
||||
def _atomic_cross_device_move(src: Path, dst: Path) -> None:
|
||||
"""Move ``src`` to ``dst`` across filesystems WITHOUT ever exposing a partial file at
|
||||
the final path.
|
||||
|
||||
Copies into a hidden temp sibling of ``dst`` (same filesystem), fsyncs, then does an
|
||||
atomic ``os.replace`` into place, then deletes ``src``. A media-server file watcher
|
||||
(Jellyfin/Plex real-time monitoring) therefore only ever indexes the COMPLETE file —
|
||||
an incremental in-place copy was what Jellyfin could catch mid-write and cache with
|
||||
null/incomplete metadata (tracks landing with no disc). Cleans up the temp on failure.
|
||||
"""
|
||||
src, dst = Path(src), Path(dst)
|
||||
tmp = dst.parent / f".{dst.name}.ssync-tmp"
|
||||
try:
|
||||
with open(src, "rb") as f_src, open(tmp, "wb") as f_dst:
|
||||
shutil.copyfileobj(f_src, f_dst)
|
||||
f_dst.flush()
|
||||
os.fsync(f_dst.fileno())
|
||||
try:
|
||||
shutil.copystat(str(src), str(tmp)) # preserve mtime/permissions (copy2-like)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(str(tmp), str(dst)) # atomic within dst's filesystem
|
||||
except Exception:
|
||||
try:
|
||||
if tmp.exists():
|
||||
tmp.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
try:
|
||||
src.unlink()
|
||||
except OSError:
|
||||
logger.info(f"Could not delete source after cross-device move (may be owned by another process): {src}")
|
||||
|
||||
|
||||
def safe_move_file(src, dst):
|
||||
"""Move a file safely across filesystems."""
|
||||
src = Path(src)
|
||||
|
|
@ -129,7 +165,13 @@ def safe_move_file(src, dst):
|
|||
break
|
||||
|
||||
try:
|
||||
shutil.move(str(src), str(dst))
|
||||
# Same-filesystem move: an atomic rename that also overwrites dst. A media-server
|
||||
# watcher (Jellyfin/Plex real-time monitoring) therefore never sees a partial file
|
||||
# at the final name. Cross-filesystem raises EXDEV (some network mounts raise
|
||||
# EPERM/EACCES) and we copy atomically below instead of letting the move write the
|
||||
# destination incrementally — the partial-file-at-final-name is what caused tracks
|
||||
# to land in Jellyfin with null/incomplete metadata (no disc).
|
||||
os.replace(str(src), str(dst))
|
||||
return
|
||||
except FileNotFoundError:
|
||||
if dst.exists():
|
||||
|
|
@ -137,8 +179,6 @@ def safe_move_file(src, dst):
|
|||
return
|
||||
raise
|
||||
except (OSError, PermissionError) as e:
|
||||
error_msg = str(e).lower()
|
||||
|
||||
if dst.exists() and dst.stat().st_size > 0:
|
||||
logger.warning(f"Move raised {type(e).__name__} but destination exists, treating as success: {e}")
|
||||
try:
|
||||
|
|
@ -147,23 +187,21 @@ def safe_move_file(src, dst):
|
|||
logger.info(f"Could not delete source file (may be owned by another process): {src}")
|
||||
return
|
||||
|
||||
if "cross-device" in error_msg or "operation not permitted" in error_msg or "permission denied" in error_msg:
|
||||
logger.warning(f"Cross-device move detected, using fallback copy method: {e}")
|
||||
error_msg = str(e).lower()
|
||||
cross_device = (
|
||||
getattr(e, "errno", None) in (errno.EXDEV, errno.EPERM, errno.EACCES)
|
||||
or "cross-device" in error_msg
|
||||
or "operation not permitted" in error_msg
|
||||
or "permission denied" in error_msg
|
||||
)
|
||||
if cross_device:
|
||||
logger.warning(f"Cross-device move, using atomic copy+rename: {e}")
|
||||
try:
|
||||
with open(src, "rb") as f_src:
|
||||
with open(dst, "wb") as f_dst:
|
||||
shutil.copyfileobj(f_src, f_dst)
|
||||
f_dst.flush()
|
||||
os.fsync(f_dst.fileno())
|
||||
|
||||
try:
|
||||
src.unlink()
|
||||
except PermissionError:
|
||||
logger.info(f"Could not delete source file (may be owned by another process): {src}")
|
||||
logger.info(f"Successfully moved file using fallback method: {src} -> {dst}")
|
||||
_atomic_cross_device_move(src, dst)
|
||||
logger.info(f"Successfully moved file atomically across filesystems: {src} -> {dst}")
|
||||
return
|
||||
except Exception as fallback_error:
|
||||
logger.error(f"Fallback copy also failed: {fallback_error}")
|
||||
logger.error(f"Atomic cross-device move failed: {fallback_error}")
|
||||
raise
|
||||
raise
|
||||
|
||||
|
|
@ -192,6 +230,9 @@ def get_audio_quality_string(file_path):
|
|||
if ext == ".flac":
|
||||
from mutagen.flac import FLAC
|
||||
audio = FLAC(file_path)
|
||||
sr = getattr(audio.info, "sample_rate", 0) or 0
|
||||
if sr:
|
||||
return f"FLAC {audio.info.bits_per_sample}bit/{sr / 1000:g}kHz"
|
||||
return f"FLAC {audio.info.bits_per_sample}bit"
|
||||
|
||||
if ext == ".mp3":
|
||||
|
|
@ -225,6 +266,122 @@ def get_audio_quality_string(file_path):
|
|||
return ""
|
||||
|
||||
|
||||
def probe_audio_quality(file_path: str):
|
||||
"""Read the actual file and return an AudioQuality with real measured values.
|
||||
|
||||
Uses mutagen to extract sample_rate, bit_depth, and bitrate from the
|
||||
downloaded file — these are ground-truth values, not estimates.
|
||||
Returns None when the file cannot be read.
|
||||
"""
|
||||
from core.quality.model import AudioQuality
|
||||
try:
|
||||
ext = os.path.splitext(file_path)[1].lower().lstrip('.')
|
||||
|
||||
if ext == 'flac':
|
||||
from mutagen.flac import FLAC
|
||||
audio = FLAC(file_path)
|
||||
return AudioQuality(
|
||||
format='flac',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
bit_depth=audio.info.bits_per_sample,
|
||||
)
|
||||
|
||||
if ext == 'mp3':
|
||||
from mutagen.mp3 import MP3
|
||||
audio = MP3(file_path)
|
||||
return AudioQuality(
|
||||
format='mp3',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext in ('m4a', 'aac', 'mp4'):
|
||||
from mutagen.mp4 import MP4
|
||||
audio = MP4(file_path)
|
||||
# .m4a can carry AAC (lossy) OR ALAC (lossless) — only the real
|
||||
# codec tells them apart, which is why extension-based classification
|
||||
# defaults to 'aac' and we correct it here from the probed file.
|
||||
codec = (getattr(audio.info, 'codec', '') or '').lower()
|
||||
if 'alac' in codec:
|
||||
return AudioQuality(
|
||||
format='alac',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
bit_depth=getattr(audio.info, 'bits_per_sample', None) or None,
|
||||
)
|
||||
return AudioQuality(
|
||||
format='aac',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext == 'ogg':
|
||||
from mutagen.oggvorbis import OggVorbis
|
||||
audio = OggVorbis(file_path)
|
||||
return AudioQuality(
|
||||
format='ogg',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext == 'opus':
|
||||
from mutagen.oggopus import OggOpus
|
||||
audio = OggOpus(file_path)
|
||||
return AudioQuality(
|
||||
format='opus',
|
||||
bitrate=audio.info.bitrate // 1000,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
)
|
||||
|
||||
if ext in ('wav', 'aiff', 'aif'):
|
||||
# AIFF must use mutagen.aiff.AIFF — WAVE() can't parse it and would
|
||||
# raise, making the file fail open and silently bypass the quality
|
||||
# filter. Both are uncompressed PCM, so they share the 'wav' tier.
|
||||
if ext == 'wav':
|
||||
from mutagen.wave import WAVE
|
||||
audio = WAVE(file_path)
|
||||
else:
|
||||
from mutagen.aiff import AIFF
|
||||
audio = AIFF(file_path)
|
||||
return AudioQuality(
|
||||
format='wav',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=audio.info.sample_rate,
|
||||
bit_depth=getattr(audio.info, 'bits_per_sample', None),
|
||||
)
|
||||
|
||||
if ext == 'wma':
|
||||
from mutagen.asf import ASF
|
||||
audio = ASF(file_path)
|
||||
return AudioQuality(
|
||||
format='wma',
|
||||
bitrate=audio.info.bitrate // 1000 if audio.info.bitrate else None,
|
||||
sample_rate=getattr(audio.info, 'sample_rate', None),
|
||||
)
|
||||
|
||||
if ext in ('dsf', 'dff'):
|
||||
# DSD (DSD Stream File / DSDIFF) — 1-bit hi-res lossless (#939). mutagen
|
||||
# reads .dsf (rate/bitrate/bit_depth); .dff has no mutagen reader, so it
|
||||
# still classifies as the lossless 'dsf' tier just without measured detail.
|
||||
sr = bd = br = None
|
||||
if ext == 'dsf':
|
||||
try:
|
||||
from mutagen.dsf import DSF
|
||||
info = DSF(file_path).info
|
||||
sr = getattr(info, 'sample_rate', None)
|
||||
bd = getattr(info, 'bits_per_sample', None)
|
||||
br = info.bitrate // 1000 if getattr(info, 'bitrate', None) else None
|
||||
except Exception: # noqa: S110 — unreadable DSF still classifies lossless, just without measured detail
|
||||
pass
|
||||
return AudioQuality(format='dsf', bitrate=br, sample_rate=sr, bit_depth=bd)
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug("probe_audio_quality failed for %s: %s", file_path, e)
|
||||
return None
|
||||
|
||||
|
||||
def get_quality_tier_from_extension(file_path):
|
||||
"""Classify a file extension into a quality tier."""
|
||||
if not file_path:
|
||||
|
|
@ -367,14 +524,29 @@ def downsample_hires_flac(final_path, context):
|
|||
return None
|
||||
|
||||
|
||||
def m4a_codec(path):
|
||||
"""Codec of an .m4a/.mp4 container ('alac', 'aac', …) or None — lets the
|
||||
lossless check tell ALAC (lossless) from AAC (lossy), since both are .m4a."""
|
||||
try:
|
||||
from mutagen.mp4 import MP4
|
||||
return (getattr(MP4(path).info, 'codec', '') or '').lower() or None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def create_lossy_copy(final_path):
|
||||
"""Convert a FLAC file to a lossy copy using the configured codec."""
|
||||
from mutagen.flac import FLAC
|
||||
"""Convert a lossless file (FLAC / ALAC / WAV / AIFF / DSD) to a lossy copy
|
||||
using the configured codec. Non-lossless inputs are skipped (#941)."""
|
||||
from core.quality.lossless import (
|
||||
is_lossless_audio_path,
|
||||
lossy_output_would_overwrite_source,
|
||||
)
|
||||
|
||||
if not config_manager.get("lossy_copy.enabled", False):
|
||||
return None
|
||||
|
||||
if os.path.splitext(final_path)[1].lower() != ".flac":
|
||||
# Was FLAC-only; now any lossless source. .m4a is probed (ALAC vs AAC).
|
||||
if not is_lossless_audio_path(final_path, probe_codec=m4a_codec):
|
||||
return None
|
||||
|
||||
codec = config_manager.get("lossy_copy.codec", "mp3").lower()
|
||||
|
|
@ -403,6 +575,16 @@ def create_lossy_copy(final_path):
|
|||
out_basename = out_basename.replace(original_quality, quality_label)
|
||||
out_path = os.path.join(os.path.dirname(out_path), out_basename)
|
||||
|
||||
# Safety invariant: never write the lossy copy over its own source (an .m4a
|
||||
# ALAC source + AAC target lands on the same .m4a path). ffmpeg runs with -y,
|
||||
# so this guard MUST precede it — the later delete-original guard is too late.
|
||||
if lossy_output_would_overwrite_source(final_path, out_path):
|
||||
logger.info(
|
||||
f"[Lossy Copy] Skipping — {codec.upper()} output would overwrite the "
|
||||
f"source: {os.path.basename(final_path)}"
|
||||
)
|
||||
return None
|
||||
|
||||
ffmpeg_bin = shutil.which("ffmpeg")
|
||||
if not ffmpeg_bin:
|
||||
local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg")
|
||||
|
|
|
|||
|
|
@ -105,30 +105,72 @@ def move_to_quarantine(file_path: str, context: dict, reason: str, automation_en
|
|||
|
||||
|
||||
def check_flac_bit_depth(file_path: str, context: dict) -> Optional[str]:
|
||||
"""Return a rejection message if a FLAC file violates the configured bit depth."""
|
||||
if not context.get("_audio_quality", "").startswith("FLAC"):
|
||||
"""Legacy wrapper — delegates to check_quality_target.
|
||||
|
||||
Kept for callers that still pass trigger='bit_depth'; the new guard
|
||||
covers bit_depth as part of the full quality target check.
|
||||
"""
|
||||
return check_quality_target(file_path, context)
|
||||
|
||||
|
||||
def check_quality_target(file_path: str, context: dict) -> Optional[str]:
|
||||
"""Return a rejection message when the downloaded file does not satisfy
|
||||
the user's quality priority list.
|
||||
|
||||
Probes the actual file with mutagen (ground-truth sample_rate,
|
||||
bit_depth, bitrate) and checks it against the profile's
|
||||
``ranked_targets``. Falls back gracefully when fallback_enabled=True.
|
||||
|
||||
Works for all formats and all download sources — no Soulseek-specific
|
||||
logic here.
|
||||
"""
|
||||
from core.imports.file_ops import probe_audio_quality
|
||||
from core.quality.selection import targets_from_profile, quality_meets_profile
|
||||
|
||||
# Master toggle (Settings → Import). When OFF, the quality check is skipped
|
||||
# entirely and files import regardless of quality — the user opted out of
|
||||
# quality-filtering on import. Default ON preserves existing behaviour. The
|
||||
# library Quality Upgrade scanner still flags below-profile files either way.
|
||||
if _get_config_manager().get("import.quality_filter_enabled", True) is False:
|
||||
logger.debug(
|
||||
"[QualityGuard] import.quality_filter_enabled=False — skipping quality "
|
||||
"filter for %s", os.path.basename(file_path),
|
||||
)
|
||||
return None
|
||||
|
||||
quality_profile = MusicDatabase().get_quality_profile()
|
||||
flac_config = quality_profile.get("qualities", {}).get("flac", {})
|
||||
flac_pref = flac_config.get("bit_depth", "any")
|
||||
if flac_pref == "any":
|
||||
aq = probe_audio_quality(file_path)
|
||||
if aq is None:
|
||||
logger.debug("[QualityGuard] Could not probe %s — skipping check", os.path.basename(file_path))
|
||||
return None
|
||||
|
||||
actual_bits = context["_audio_quality"].replace("FLAC ", "").replace("bit", "")
|
||||
if actual_bits == flac_pref:
|
||||
profile = MusicDatabase().get_quality_profile()
|
||||
targets, fallback_enabled = targets_from_profile(profile)
|
||||
|
||||
if not targets:
|
||||
return None
|
||||
|
||||
flac_fallback = flac_config.get("bit_depth_fallback", True)
|
||||
downsample_enabled = _get_config_manager().get("lossy_copy.downsample_hires", False)
|
||||
|
||||
matched = quality_meets_profile(aq, targets)
|
||||
|
||||
track_info = context.get("track_info", {})
|
||||
track_name = track_info.get("name", os.path.basename(file_path))
|
||||
actual_label = aq.label()
|
||||
|
||||
if flac_fallback or downsample_enabled:
|
||||
if downsample_enabled:
|
||||
logger.info("[FLAC Downsample] Accepted %s-bit FLAC (will be downsampled to %s-bit): %s", actual_bits, flac_pref, track_name)
|
||||
else:
|
||||
logger.warning("[FLAC Fallback] Accepted %s-bit FLAC (preferred %s-bit): %s", actual_bits, flac_pref, track_name)
|
||||
if matched:
|
||||
logger.info("[QualityGuard] %s meets profile: %s", track_name, actual_label)
|
||||
return None
|
||||
|
||||
return f"FLAC bit depth mismatch: file is {actual_bits}-bit, preference is {flac_pref}-bit"
|
||||
# No target matched
|
||||
best_label = targets[0].label if targets else "?"
|
||||
if fallback_enabled or downsample_enabled:
|
||||
logger.warning(
|
||||
"[QualityGuard] %s did not match any target (got %s, wanted %s) — accepting via fallback",
|
||||
track_name, actual_label, best_label,
|
||||
)
|
||||
return None
|
||||
|
||||
return (
|
||||
f"Quality mismatch: file is {actual_label}, "
|
||||
f"does not satisfy any configured target (best wanted: {best_label})"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -34,9 +34,11 @@ from core.imports.context import (
|
|||
)
|
||||
from core.imports.file_integrity import check_audio_integrity, expected_duration_for_check, resolve_duration_tolerance
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
from core.imports.guards import check_flac_bit_depth, move_to_quarantine
|
||||
from core.imports.guards import check_flac_bit_depth, check_quality_target, move_to_quarantine
|
||||
from core.imports.silence import detect_broken_audio
|
||||
from core.imports.quarantine import (
|
||||
approve_quarantine_entry,
|
||||
delete_quarantine_entry,
|
||||
entry_id_from_quarantined_filename,
|
||||
list_quarantine_entries,
|
||||
)
|
||||
|
|
@ -160,7 +162,9 @@ def import_rejection_reason(context: dict) -> str | None:
|
|||
f"{context.get('_acoustid_failure_msg', 'fingerprint mismatch')}"
|
||||
)
|
||||
if context.get('_bitdepth_rejected'):
|
||||
return "rejected by bit-depth filter"
|
||||
return "rejected by quality filter"
|
||||
if context.get('_silence_rejected'):
|
||||
return "rejected by audio guard (incomplete or silent audio)"
|
||||
if context.get('_race_guard_failed'):
|
||||
return "source file disappeared before import completed"
|
||||
return None
|
||||
|
|
@ -321,11 +325,14 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.error(f"File quarantined due to integrity failure: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting broken file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as del_error:
|
||||
logger.error(f"Could not delete broken file either: {del_error}")
|
||||
# Quarantine MOVE failed (e.g. cross-device / permission on a NAS).
|
||||
# Do NOT delete — destroying a download we couldn't even quarantine is
|
||||
# data loss and forces a re-download. Leave it in place so it can be
|
||||
# retried; the task is still marked failed below either way (#kettui).
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file in place "
|
||||
f"for retry (not deleting): {file_path}"
|
||||
)
|
||||
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
|
|
@ -353,6 +360,118 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
f"drift={integrity.checks.get('length_drift_s', 'n/a')})"
|
||||
)
|
||||
|
||||
# Audio-completeness guard — runs right where the length is verified,
|
||||
# BEFORE the AcoustID/quality gates, so a truncated file (container
|
||||
# claims the full length but only ~30s actually decodes, e.g. HiFi/
|
||||
# Monochrome HLS assembly) or a mostly-silent file is caught regardless
|
||||
# of its quality verdict and gets the right reason. Same quarantine +
|
||||
# next-candidate retry pattern as the integrity check.
|
||||
#
|
||||
# Opt-in (default OFF): this is the one check that fully DECODES the file
|
||||
# with ffmpeg, so it is the most CPU-expensive step in post-processing.
|
||||
# Most preview/truncation cases are already caught at the source
|
||||
# (HiFi/Qobuz have their own guards), so it stays off unless the user
|
||||
# turns it on under Settings → Post-processing.
|
||||
_audio_guard_enabled = config_manager.get('post_processing.audio_completeness_check', False)
|
||||
_skip_audio = (not _audio_guard_enabled) or _should_skip_quarantine_check(context, 'silence')
|
||||
audio_reason = None if _skip_audio else detect_broken_audio(file_path)
|
||||
if audio_reason:
|
||||
logger.error(f"[AudioGuard] Rejected {_basename}: {audio_reason}")
|
||||
context['_silence_rejected'] = True
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path, context, audio_reason, automation_engine,
|
||||
trigger='silence',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.warning("File quarantined — incomplete/silent audio: %s", quarantine_path)
|
||||
except Exception as quarantine_error:
|
||||
# Don't delete a file we couldn't quarantine — leave it for retry
|
||||
# instead of forcing a re-download (data loss). See integrity block.
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file in place "
|
||||
f"for retry (not deleting): {file_path}"
|
||||
)
|
||||
|
||||
with matched_context_lock:
|
||||
matched_downloads_context.pop(context_key, None)
|
||||
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
# Try the next-best candidate before giving up — same pattern as
|
||||
# the integrity / AcoustID / quality failures.
|
||||
if _requeue_quarantined_task_for_retry(task_id, batch_id, 'silence'):
|
||||
logger.info(
|
||||
"Incomplete/silent audio for task %s — retrying next-best candidate: %s",
|
||||
task_id, audio_reason,
|
||||
)
|
||||
return
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Audio guard: {audio_reason}"
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
# QUALITY GATE — runs BEFORE AcoustID so (1) a wrong-quality file is
|
||||
# rejected without paying for an AcoustID fingerprint, and (2) the real
|
||||
# audio quality is recorded on the context (→ quarantine sidecar) for
|
||||
# EVERY trigger, so it's known when reviewing/approving any quarantined
|
||||
# file. force_import still never fires on a quality mismatch.
|
||||
context['_audio_quality'] = get_audio_quality_string(file_path)
|
||||
if context['_audio_quality']:
|
||||
logger.info(f"Audio quality detected: {context['_audio_quality']}")
|
||||
|
||||
_skip_quality = _should_skip_quarantine_check(context, 'bit_depth') or \
|
||||
_should_skip_quarantine_check(context, 'quality')
|
||||
rejection_reason = None if _skip_quality else check_quality_target(file_path, context)
|
||||
if _skip_quality:
|
||||
logger.info(f"[QualityGuard] Skipped (user approval) for {_basename}")
|
||||
if rejection_reason:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
rejection_reason,
|
||||
automation_engine,
|
||||
trigger='quality',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.info(f"File quarantined due to quality mismatch: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
# Don't delete a file we couldn't quarantine — leave it for retry
|
||||
# instead of forcing a re-download (data loss). See integrity block.
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file in place "
|
||||
f"for retry (not deleting): {file_path}"
|
||||
)
|
||||
|
||||
context['_bitdepth_rejected'] = True
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
with matched_context_lock:
|
||||
matched_downloads_context.pop(context_key, None)
|
||||
|
||||
# Try the next-best candidate before giving up — same pattern
|
||||
# as AcoustID and integrity failures.
|
||||
if _requeue_quarantined_task_for_retry(task_id, batch_id, 'quality'):
|
||||
logger.info(
|
||||
"Quality mismatch for task %s — retrying next-best candidate: %s",
|
||||
task_id, rejection_reason,
|
||||
)
|
||||
return
|
||||
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Quality filter: {rejection_reason}"
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
_skip_acoustid = _should_skip_quarantine_check(context, 'acoustid')
|
||||
if _skip_acoustid:
|
||||
logger.info(f"[AcoustID] Skipped (user approval) for {_basename}")
|
||||
|
|
@ -390,24 +509,44 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}")
|
||||
context['_acoustid_result'] = verification_result.value
|
||||
|
||||
if verification_result == VerificationResult.FAIL:
|
||||
# Fail-closed mode: when the user requires a hard AcoustID
|
||||
# PASS, a SKIP (ran but couldn't confirm — no fingerprint
|
||||
# match / cross-script metadata) is treated like a FAIL:
|
||||
# quarantine + try the next candidate, instead of importing
|
||||
# an unverified file. ERROR (rate-limit / infra) is NOT
|
||||
# blocked — that would stall the whole pipeline during an
|
||||
# outage; those still import with their existing flag.
|
||||
require_verified = config_manager.get('acoustid.require_verified', False)
|
||||
_skip_as_fail = (
|
||||
require_verified
|
||||
and verification_result == VerificationResult.SKIP
|
||||
)
|
||||
if _skip_as_fail:
|
||||
verification_msg = (
|
||||
f"AcoustID could not confirm the track and 'require verified' "
|
||||
f"is on — rejecting unverified file ({verification_msg})"
|
||||
)
|
||||
logger.warning("[AcoustID] Require-verified: SKIP treated as FAIL — %s", verification_msg)
|
||||
|
||||
if verification_result == VerificationResult.FAIL or _skip_as_fail:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
verification_msg,
|
||||
automation_engine,
|
||||
trigger='acoustid',
|
||||
trigger='acoustid_unverified' if _skip_as_fail else 'acoustid',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.error(f"File quarantined due to verification failure: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}")
|
||||
logger.error(f"Quarantine failed, deleting wrong file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as del_error:
|
||||
logger.error(f"Could not delete wrong file either: {del_error}")
|
||||
# Don't delete a file we couldn't quarantine — leave it for
|
||||
# retry instead of forcing a re-download (data loss). The
|
||||
# task is still marked failed / requeued below. See integrity.
|
||||
logger.error(
|
||||
f"Quarantine failed ({quarantine_error}) — leaving file "
|
||||
f"in place for retry (not deleting): {file_path}"
|
||||
)
|
||||
|
||||
context['_acoustid_quarantined'] = True
|
||||
context['_acoustid_failure_msg'] = verification_msg
|
||||
|
|
@ -577,48 +716,6 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
album_info.get('album_name', 'None'),
|
||||
)
|
||||
|
||||
context['_audio_quality'] = get_audio_quality_string(file_path)
|
||||
if context['_audio_quality']:
|
||||
logger.info(f"Audio quality detected: {context['_audio_quality']}")
|
||||
|
||||
_skip_bit_depth = _should_skip_quarantine_check(context, 'bit_depth')
|
||||
rejection_reason = None if _skip_bit_depth else check_flac_bit_depth(file_path, context)
|
||||
if _skip_bit_depth:
|
||||
logger.info(f"[BitDepth] Skipped (user approval) for {_basename}")
|
||||
if rejection_reason:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
rejection_reason,
|
||||
automation_engine,
|
||||
trigger='bit_depth',
|
||||
)
|
||||
_mark_task_quarantined(context, quarantine_path)
|
||||
logger.info(f"File quarantined due to bit depth filter: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as e:
|
||||
logger.debug("delete quarantine fallback: %s", e)
|
||||
|
||||
context['_bitdepth_rejected'] = True
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}"
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
file_ext = os.path.splitext(file_path)[1]
|
||||
clean_track_name = get_import_clean_title(
|
||||
context,
|
||||
|
|
@ -655,6 +752,16 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
album_info['clean_track_name'] = clean_track_name
|
||||
logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata")
|
||||
|
||||
# Sync the disc number the SAME way (and via the SAME resolver) the embedded
|
||||
# tag will use — otherwise the "Disc N" folder is built from album_info's
|
||||
# original disc while the tag takes the per-track disc, so a disc-2/3 track
|
||||
# lands in the Disc 1 folder and every disc's tracks collapse together (Sokhi).
|
||||
from core.imports.track_number import resolve_disc_for_track
|
||||
_resolved_disc = resolve_disc_for_track(get_import_original_search(context), album_info)
|
||||
if album_info.get('disc_number') != _resolved_disc:
|
||||
logger.info(f"[FIX] Updated album_info disc_number to {_resolved_disc} for consistent metadata")
|
||||
album_info['disc_number'] = _resolved_disc
|
||||
|
||||
final_path, _ = build_final_path_for_track(context, artist_context, album_info, file_ext)
|
||||
logger.info(f"Resolved path: '{final_path}'")
|
||||
context['_final_processed_path'] = final_path
|
||||
|
|
@ -977,6 +1084,22 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
if original_batch_id:
|
||||
context['batch_id'] = original_batch_id
|
||||
|
||||
# Quality / audio-guard quarantine. Unlike acoustid/integrity (whose
|
||||
# retry+fail is driven by THIS wrapper below), the inner pipeline fully
|
||||
# owns the quality and audio-guard outcome: it quarantines the file and
|
||||
# then either re-queues the next-best candidate or marks the task failed
|
||||
# and notifies. The wrapper must NOT continue to the "assume success"
|
||||
# fall-through — that marked the quarantined file Completed, so the same
|
||||
# track showed in BOTH Completed and Quarantine (e.g. an MP3-VBR rejected
|
||||
# by a FLAC-only profile). It must also NOT mark it failed here, which
|
||||
# would clobber a successful next-candidate retry. Just return.
|
||||
if context.get('_bitdepth_rejected') or context.get('_silence_rejected'):
|
||||
logger.info(
|
||||
f"Task {task_id} quarantined by the quality/audio guard — inner "
|
||||
f"pipeline already handled retry/fail; wrapper not marking completed"
|
||||
)
|
||||
return
|
||||
|
||||
if context.get('_race_guard_failed'):
|
||||
logger.info(f"Race guard: source file gone for task {task_id} — marking as failed")
|
||||
with tasks_lock:
|
||||
|
|
@ -990,6 +1113,33 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
return
|
||||
|
||||
if context.get('_acoustid_quarantined'):
|
||||
# Race-condition guard: if the user approved an alternative quarantine
|
||||
# entry for this track while this download was already in flight, the
|
||||
# pipeline will have created a new quarantine entry for the just-finished
|
||||
# file. Delete that entry and exit — the user's choice already won.
|
||||
_approved_alt = False
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
_ct = download_tasks.get(task_id)
|
||||
if isinstance(_ct, dict) and _ct.get('_quarantine_approved_alternative'):
|
||||
_approved_alt = True
|
||||
if _approved_alt:
|
||||
_eid = context.get('_quarantine_entry_id')
|
||||
if _eid:
|
||||
try:
|
||||
from core.imports.guards import _get_config_manager
|
||||
_dl_dir = _get_config_manager().get('soulseek.download_path', './downloads')
|
||||
_qdir = os.path.join(docker_resolve_path(_dl_dir), 'ss_quarantine')
|
||||
delete_quarantine_entry(_qdir, _eid)
|
||||
logger.info(
|
||||
f"[Quarantine] Discarded late entry {_eid} — task {task_id} "
|
||||
f"was cancelled by prior quarantine approval"
|
||||
)
|
||||
except Exception as _dqe:
|
||||
logger.debug(f"[Quarantine] Could not clean up late entry {_eid}: {_dqe}")
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed')
|
||||
# Before failing outright, try the next-best candidate. The wrong
|
||||
# file was just quarantined; re-running the worker (with the bad
|
||||
|
|
@ -1106,6 +1256,10 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
_mark_task_completed(task_id, context.get('track_info'))
|
||||
if context.get('_verification_status'):
|
||||
download_tasks[task_id]['verification_status'] = context['_verification_status']
|
||||
if context.get('_history_id'):
|
||||
download_tasks[task_id]['history_id'] = context['_history_id']
|
||||
if context.get('_audio_quality'):
|
||||
download_tasks[task_id]['quality'] = context['_audio_quality']
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
|
@ -1120,6 +1274,10 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
download_tasks[task_id]['metadata_enhanced'] = True
|
||||
if context.get('_verification_status'):
|
||||
download_tasks[task_id]['verification_status'] = context['_verification_status']
|
||||
if context.get('_history_id'):
|
||||
download_tasks[task_id]['history_id'] = context['_history_id']
|
||||
if context.get('_audio_quality'):
|
||||
download_tasks[task_id]['quality'] = context['_audio_quality']
|
||||
redownload_ctx = download_tasks[task_id].get('_redownload_context')
|
||||
|
||||
with matched_context_lock:
|
||||
|
|
|
|||
|
|
@ -168,16 +168,15 @@ def quarantine_group_key(
|
|||
fetch — NOT the downloaded file's own tags. That matters: the file's
|
||||
metadata is frequently *wrong* (that's why it failed acoustid /
|
||||
integrity), whereas the target is fixed and identical across every
|
||||
alternative for one song (they're all Soulseek uploads of the *same*
|
||||
source track), so grouping by it is an exact relationship, not a fuzzy
|
||||
metadata guess.
|
||||
alternative for one song.
|
||||
|
||||
Prefers a stable target-track id from the sidecar `context.track_info`
|
||||
when present — isrc, then source id, then uri — since those are exact
|
||||
and constant across siblings. Falls back to the normalized
|
||||
artist|track name only for legacy/thin sidecars that carry no context.
|
||||
Keys are kind-prefixed so an id-based key never collides with a
|
||||
name-based one.
|
||||
Uses ISRC when available (truly universal across sources and batches).
|
||||
Falls back to normalized artist|track name, which is stable across
|
||||
different batches and sources.
|
||||
|
||||
Source-specific IDs (Spotify track id, Qobuz id, uri) are intentionally
|
||||
NOT used: the same song imported from different playlists or sources gets
|
||||
different source IDs, so id-based keys break cross-batch sibling matching.
|
||||
|
||||
Returns ``None`` when nothing identifies the target (no usable id and
|
||||
both name fields empty). Callers treat a ``None`` key as "its own
|
||||
|
|
@ -191,12 +190,6 @@ def quarantine_group_key(
|
|||
isrc = str(ti.get("isrc") or "").strip().lower()
|
||||
if isrc:
|
||||
return f"isrc:{isrc}"
|
||||
tid = str(ti.get("id") or "").strip()
|
||||
if tid:
|
||||
return f"id:{tid}"
|
||||
uri = str(ti.get("uri") or "").strip()
|
||||
if uri:
|
||||
return f"uri:{uri}"
|
||||
artist = " ".join(str(expected_artist or "").split()).lower()
|
||||
track = " ".join(str(expected_track or "").split()).lower()
|
||||
if not artist and not track:
|
||||
|
|
@ -302,6 +295,10 @@ def list_quarantine_entries(quarantine_dir: str) -> List[Dict[str, Any]]:
|
|||
"source_username": source_username,
|
||||
"source_filename": source_filename,
|
||||
"thumb_url": _extract_context_thumb(ctx),
|
||||
# Real probed audio quality (recorded on the context before the
|
||||
# quality/AcoustID gates) so the review UI shows what the file
|
||||
# actually is when deciding to approve/delete.
|
||||
"quality": ctx.get("_audio_quality", "") if isinstance(ctx, dict) else "",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,10 +3,12 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import as_completed
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Dict
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from core.imports.album import build_album_import_context, build_album_import_match_payload, resolve_album_artist_context
|
||||
from core.imports.context import get_import_context_artist, get_import_track_info, normalize_import_context
|
||||
|
|
@ -16,6 +18,7 @@ from core.imports.staging import (
|
|||
AUDIO_EXTENSIONS,
|
||||
get_import_suggestions_cache,
|
||||
get_primary_source as _get_primary_source,
|
||||
get_primary_source_label as _get_primary_source_label,
|
||||
get_staging_path as _get_staging_path,
|
||||
read_staging_file_metadata as _read_staging_file_metadata,
|
||||
refresh_import_suggestions_cache as _refresh_import_suggestions_cache,
|
||||
|
|
@ -48,6 +51,7 @@ class ImportRouteRuntime:
|
|||
read_staging_file_metadata: Callable[[str, str], Dict[str, Any]] = _read_staging_file_metadata
|
||||
read_tags: Callable[[str], Any] = _default_read_tags
|
||||
get_primary_source: Callable[[], str] = _get_primary_source
|
||||
get_primary_source_label: Callable[[], str] = _get_primary_source_label
|
||||
search_import_albums: Callable[..., list] = _search_import_albums
|
||||
search_import_tracks: Callable[..., list] = _search_import_tracks
|
||||
build_album_import_match_payload: Callable[..., Dict[str, Any]] = build_album_import_match_payload
|
||||
|
|
@ -69,36 +73,219 @@ class ImportRouteRuntime:
|
|||
logger: Any = module_logger
|
||||
|
||||
|
||||
# ── Shared staging scan ──────────────────────────────────────────────────────
|
||||
# Opening the Import page fires staging files/groups/hints together; each used to
|
||||
# os.walk the whole staging folder AND mutagen-read every file independently — 3×
|
||||
# the directory walk + 3× the tag I/O on every page open (the import-page scan
|
||||
# storm + memory spike, issue #935). They all need the same per-file tag data, so
|
||||
# scan ONCE and let all three derive their views in-memory. A short TTL + a lock
|
||||
# means the three near-simultaneous page-open requests (and any concurrent caller)
|
||||
# share a single scan instead of each kicking off a full re-read.
|
||||
_STAGING_SCAN_LOCK = threading.Lock()
|
||||
_STAGING_SCAN_TTL = 6.0 # seconds — covers the page-open burst; re-scans after
|
||||
_staging_scan_cache: Dict[str, Any] = {"path": None, "ts": 0.0, "records": None}
|
||||
# Bumped by invalidate_staging_scan_cache() so a background scan that finishes after an
|
||||
# import doesn't re-commit stale (pre-import) records (see the generation guard above).
|
||||
_staging_scan_generation: Dict[str, int] = {"value": 0}
|
||||
|
||||
# Background-scan plumbing: a large staging folder (whole-library migration, #947) makes
|
||||
# the synchronous scan exceed gunicorn's 120s request timeout. The runner moves the SAME
|
||||
# scan off the request thread; the endpoints report progress instead of blocking.
|
||||
_staging_scan_status: Dict[str, Any] = {
|
||||
"status": "idle", "scanned": 0, "total": 0, "path": None, "error": None,
|
||||
}
|
||||
_staging_scan_status_lock = threading.Lock()
|
||||
|
||||
|
||||
def _staging_cache_hit(staging_path: str) -> Optional[list]:
|
||||
"""The cached records for ``staging_path`` if still fresh, else None (no scan triggered)."""
|
||||
c = _staging_scan_cache
|
||||
if (c["records"] is not None and c["path"] == staging_path
|
||||
and (time.time() - c["ts"]) < _STAGING_SCAN_TTL):
|
||||
return c["records"]
|
||||
return None
|
||||
|
||||
|
||||
def ensure_background_staging_scan(runtime: ImportRouteRuntime, staging_path: str) -> None:
|
||||
"""Start a background scan for ``staging_path`` unless the cache is warm or a scan for
|
||||
this path is already running. Idempotent — safe to call on every request."""
|
||||
if _staging_cache_hit(staging_path) is not None:
|
||||
return
|
||||
with _staging_scan_status_lock:
|
||||
if (_staging_scan_status["status"] == "scanning"
|
||||
and _staging_scan_status["path"] == staging_path):
|
||||
return
|
||||
_staging_scan_status.update({"status": "scanning", "scanned": 0, "total": 0,
|
||||
"path": staging_path, "error": None})
|
||||
|
||||
def _run() -> None:
|
||||
try:
|
||||
_scan_staging_records(runtime, staging_path, progress=_staging_scan_status)
|
||||
with _staging_scan_status_lock:
|
||||
if _staging_scan_status["path"] == staging_path:
|
||||
_staging_scan_status["status"] = "done"
|
||||
except Exception as exc: # noqa: BLE001 — surface any scan error to the poller
|
||||
with _staging_scan_status_lock:
|
||||
_staging_scan_status.update({"status": "error", "error": str(exc)})
|
||||
|
||||
threading.Thread(target=_run, name="staging-scan", daemon=True).start()
|
||||
|
||||
|
||||
def get_staging_records_or_status(runtime: ImportRouteRuntime, staging_path: str,
|
||||
*, grace_seconds: float = 3.0) -> tuple[str, Any]:
|
||||
"""Non-blocking staging access for the page endpoints. Returns ``("ready", records)``
|
||||
when the cache is warm or the scan completes within ``grace_seconds`` (so small/normal
|
||||
folders still answer in a single request), otherwise ``("scanning", status_dict)`` after
|
||||
making sure a background scan is running."""
|
||||
records = _staging_cache_hit(staging_path)
|
||||
if records is not None:
|
||||
return ("ready", records)
|
||||
ensure_background_staging_scan(runtime, staging_path)
|
||||
deadline = time.time() + max(0.0, grace_seconds)
|
||||
while True:
|
||||
records = _staging_cache_hit(staging_path)
|
||||
if records is not None:
|
||||
return ("ready", records)
|
||||
with _staging_scan_status_lock:
|
||||
status = dict(_staging_scan_status)
|
||||
if status.get("status") == "error":
|
||||
return ("error", status)
|
||||
if time.time() >= deadline:
|
||||
return ("scanning", status)
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def _records_or_scanning_payload(runtime: ImportRouteRuntime, staging_path: str):
|
||||
"""Shared helper for the page endpoints: returns ``(records, None)`` when the scan is
|
||||
ready, or ``(None, payload)`` when a background scan is still running — the caller
|
||||
returns that payload so the page polls + shows progress instead of blocking/timing out.
|
||||
|
||||
A scan error is re-raised so the endpoint's own try/except logs + returns it exactly as
|
||||
when the scan ran inline (preserves the existing error contract)."""
|
||||
state, val = get_staging_records_or_status(runtime, staging_path)
|
||||
if state == "error":
|
||||
raise RuntimeError(val.get("error") or "staging scan failed")
|
||||
if state == "scanning":
|
||||
return None, {"success": True, "scanning": True,
|
||||
"progress": {"scanned": val.get("scanned", 0),
|
||||
"total": val.get("total", 0)}}
|
||||
return val, None
|
||||
|
||||
|
||||
def staging_scan_status(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
||||
"""Lightweight, instant scan-progress poll for the page (no grace-wait, no file I/O) —
|
||||
``ready`` true once the cache is warm and the files/groups/hints calls will answer fast."""
|
||||
try:
|
||||
staging_path = runtime.get_staging_path()
|
||||
except Exception as exc:
|
||||
return {"success": False, "error": str(exc)}, 500
|
||||
with _staging_scan_status_lock:
|
||||
st = dict(_staging_scan_status)
|
||||
return {
|
||||
"success": True,
|
||||
"ready": _staging_cache_hit(staging_path) is not None,
|
||||
"status": st.get("status", "idle"),
|
||||
"scanned": st.get("scanned", 0),
|
||||
"total": st.get("total", 0),
|
||||
"error": st.get("error"),
|
||||
}, 200
|
||||
|
||||
|
||||
def _scan_staging_records(runtime: ImportRouteRuntime, staging_path: str,
|
||||
*, progress: Optional[Dict[str, Any]] = None) -> list[Dict[str, Any]]:
|
||||
"""Walk staging + read each audio file's tags ONCE, returning per-file records
|
||||
that staging files/groups/hints all derive from. Briefly cached + locked so the
|
||||
page-open trio shares a single scan rather than each re-walking and re-reading.
|
||||
|
||||
``progress`` (optional, default None = unchanged behaviour) is a dict the scan
|
||||
updates live with ``total`` (audio-file count, from a fast first pass) and ``scanned``
|
||||
(tag-reads done so far) so a background runner can report progress. A generation guard
|
||||
keeps a scan that finishes AFTER an import (which bumped ``_staging_scan_generation``)
|
||||
from committing stale records to the cache."""
|
||||
now = time.time()
|
||||
cached = _staging_scan_cache
|
||||
if (cached["records"] is not None and cached["path"] == staging_path
|
||||
and (now - cached["ts"]) < _STAGING_SCAN_TTL):
|
||||
return cached["records"]
|
||||
|
||||
with _STAGING_SCAN_LOCK:
|
||||
# Double-check: another request may have filled the cache while we waited.
|
||||
now = time.time()
|
||||
if (cached["records"] is not None and cached["path"] == staging_path
|
||||
and (now - cached["ts"]) < _STAGING_SCAN_TTL):
|
||||
return cached["records"]
|
||||
|
||||
start_generation = _staging_scan_generation["value"]
|
||||
|
||||
# Pass 1 (fast): collect the audio-file list — no tag I/O — so we know the total.
|
||||
audio_files: list[tuple[str, str, Optional[str]]] = []
|
||||
if os.path.isdir(staging_path):
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
rel_dir = os.path.relpath(root, staging_path)
|
||||
top_folder = rel_dir.split(os.sep)[0] if rel_dir != "." else None
|
||||
for fname in filenames:
|
||||
if os.path.splitext(fname)[1].lower() in AUDIO_EXTENSIONS:
|
||||
audio_files.append((root, fname, top_folder))
|
||||
if progress is not None:
|
||||
progress["total"] = len(audio_files)
|
||||
progress["scanned"] = 0
|
||||
|
||||
# Pass 2 (slow): read each file's tags, updating progress as we go.
|
||||
records: list[Dict[str, Any]] = []
|
||||
for root, fname, top_folder in audio_files:
|
||||
full_path = os.path.join(root, fname)
|
||||
rel_path = os.path.relpath(full_path, staging_path)
|
||||
meta = runtime.read_staging_file_metadata(full_path, rel_path)
|
||||
records.append({
|
||||
"filename": fname, "rel_path": rel_path, "full_path": full_path,
|
||||
"extension": os.path.splitext(fname)[1].lower(),
|
||||
"title": meta["title"], "album": meta["album"],
|
||||
"artist": meta["artist"], "albumartist": meta["albumartist"],
|
||||
"track_number": meta["track_number"], "disc_number": meta["disc_number"],
|
||||
"top_folder": top_folder,
|
||||
})
|
||||
if progress is not None:
|
||||
progress["scanned"] += 1
|
||||
|
||||
# Generation guard: if an import invalidated the cache mid-scan, these records are
|
||||
# stale — return them to this caller but do NOT commit them as the shared cache.
|
||||
if _staging_scan_generation["value"] == start_generation:
|
||||
_staging_scan_cache.update({"path": staging_path, "ts": time.time(), "records": records})
|
||||
return records
|
||||
|
||||
|
||||
def invalidate_staging_scan_cache() -> None:
|
||||
"""Drop the cached staging scan (call after an import moves/removes files so the
|
||||
next files/groups/hints request reflects the new state immediately). Also bumps the
|
||||
scan generation so an in-flight background scan won't re-commit pre-import records."""
|
||||
_staging_scan_generation["value"] += 1
|
||||
_staging_scan_cache.update({"path": None, "ts": 0.0, "records": None})
|
||||
|
||||
|
||||
def staging_files(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
||||
"""Scan the staging folder and return audio files with tag metadata."""
|
||||
try:
|
||||
staging_path = runtime.get_staging_path()
|
||||
os.makedirs(staging_path, exist_ok=True)
|
||||
|
||||
files = []
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
for fname in filenames:
|
||||
ext = os.path.splitext(fname)[1].lower()
|
||||
if ext not in AUDIO_EXTENSIONS:
|
||||
continue
|
||||
full_path = os.path.join(root, fname)
|
||||
rel_path = os.path.relpath(full_path, staging_path)
|
||||
records, scanning = _records_or_scanning_payload(runtime, staging_path)
|
||||
if scanning is not None:
|
||||
return scanning, 200
|
||||
|
||||
meta = runtime.read_staging_file_metadata(full_path, rel_path)
|
||||
|
||||
files.append(
|
||||
{
|
||||
"filename": fname,
|
||||
"rel_path": rel_path,
|
||||
"full_path": full_path,
|
||||
"title": meta["title"],
|
||||
"artist": meta["albumartist"] or meta["artist"] or "Unknown Artist",
|
||||
"album": meta["album"],
|
||||
"track_number": meta["track_number"],
|
||||
"disc_number": meta["disc_number"],
|
||||
"extension": ext,
|
||||
}
|
||||
)
|
||||
files = [
|
||||
{
|
||||
"filename": r["filename"],
|
||||
"rel_path": r["rel_path"],
|
||||
"full_path": r["full_path"],
|
||||
"title": r["title"],
|
||||
"artist": r["albumartist"] or r["artist"] or "Unknown Artist",
|
||||
"album": r["album"],
|
||||
"track_number": r["track_number"],
|
||||
"disc_number": r["disc_number"],
|
||||
"extension": r["extension"],
|
||||
}
|
||||
for r in records
|
||||
]
|
||||
|
||||
files.sort(key=lambda f: f["filename"].lower())
|
||||
return {"success": True, "files": files, "staging_path": staging_path}, 200
|
||||
|
|
@ -114,32 +301,28 @@ def staging_groups(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
|||
if not os.path.isdir(staging_path):
|
||||
return {"success": True, "groups": []}, 200
|
||||
|
||||
records, scanning = _records_or_scanning_payload(runtime, staging_path)
|
||||
if scanning is not None:
|
||||
return scanning, 200
|
||||
|
||||
album_groups = {}
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
for fname in filenames:
|
||||
ext = os.path.splitext(fname)[1].lower()
|
||||
if ext not in AUDIO_EXTENSIONS:
|
||||
continue
|
||||
full_path = os.path.join(root, fname)
|
||||
rel_path = os.path.relpath(full_path, staging_path)
|
||||
for r in records:
|
||||
album = r["album"]
|
||||
artist = r["albumartist"] or r["artist"]
|
||||
if not album or not artist:
|
||||
continue
|
||||
|
||||
meta = runtime.read_staging_file_metadata(full_path, rel_path)
|
||||
album = meta["album"]
|
||||
artist = meta["albumartist"] or meta["artist"]
|
||||
if not album or not artist:
|
||||
continue
|
||||
|
||||
key = (album.lower().strip(), artist.lower().strip())
|
||||
if key not in album_groups:
|
||||
album_groups[key] = {"album": album.strip(), "artist": artist.strip(), "files": []}
|
||||
album_groups[key]["files"].append(
|
||||
{
|
||||
"filename": fname,
|
||||
"full_path": full_path,
|
||||
"title": meta["title"],
|
||||
"track_number": meta["track_number"],
|
||||
}
|
||||
)
|
||||
key = (album.lower().strip(), artist.lower().strip())
|
||||
if key not in album_groups:
|
||||
album_groups[key] = {"album": album.strip(), "artist": artist.strip(), "files": []}
|
||||
album_groups[key]["files"].append(
|
||||
{
|
||||
"filename": r["filename"],
|
||||
"full_path": r["full_path"],
|
||||
"title": r["title"],
|
||||
"track_number": r["track_number"],
|
||||
}
|
||||
)
|
||||
|
||||
groups = []
|
||||
for group in album_groups.values():
|
||||
|
|
@ -169,30 +352,21 @@ def staging_hints(runtime: ImportRouteRuntime) -> tuple[Dict[str, Any], int]:
|
|||
if not os.path.isdir(staging_path):
|
||||
return {"success": True, "hints": []}, 200
|
||||
|
||||
records, scanning = _records_or_scanning_payload(runtime, staging_path)
|
||||
if scanning is not None:
|
||||
return scanning, 200
|
||||
|
||||
tag_albums = {}
|
||||
folder_hints = {}
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS]
|
||||
if not audio_files:
|
||||
continue
|
||||
for r in records:
|
||||
if r["top_folder"]:
|
||||
folder_hints[r["top_folder"]] = folder_hints.get(r["top_folder"], 0) + 1
|
||||
|
||||
rel_dir = os.path.relpath(root, staging_path)
|
||||
if rel_dir != ".":
|
||||
top_folder = rel_dir.split(os.sep)[0]
|
||||
folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files)
|
||||
|
||||
for fname in audio_files:
|
||||
full_path = os.path.join(root, fname)
|
||||
try:
|
||||
tags = runtime.read_tags(full_path)
|
||||
if tags:
|
||||
album = (tags.get("album") or [None])[0]
|
||||
artist = (tags.get("artist") or (tags.get("albumartist") or [None]))[0]
|
||||
if album:
|
||||
key = (album.strip(), (artist or "").strip())
|
||||
tag_albums[key] = tag_albums.get(key, 0) + 1
|
||||
except Exception as exc:
|
||||
runtime.logger.debug("tag read failed: %s", exc)
|
||||
album = r["album"]
|
||||
artist = r["artist"] or r["albumartist"]
|
||||
if album:
|
||||
key = (album.strip(), (artist or "").strip())
|
||||
tag_albums[key] = tag_albums.get(key, 0) + 1
|
||||
|
||||
queries = []
|
||||
seen_queries_lower = set()
|
||||
|
|
@ -222,7 +396,7 @@ def staging_suggestions() -> tuple[Dict[str, Any], int]:
|
|||
"success": True,
|
||||
"suggestions": cache["suggestions"],
|
||||
"ready": cache["built"],
|
||||
"primary_source": _get_primary_source(),
|
||||
"primary_source": _get_primary_source_label(),
|
||||
}, 200
|
||||
|
||||
|
||||
|
|
@ -239,7 +413,10 @@ def search_albums(runtime: ImportRouteRuntime, query: str, limit: int = 12) -> t
|
|||
runtime.hydrabase_worker.enqueue(query, "albums")
|
||||
|
||||
albums = runtime.search_import_albums(query, limit=limit)
|
||||
return {"success": True, "albums": albums, "primary_source": primary_source}, 200
|
||||
# The label names the user's CONFIGURED source (Spotify Free reads as
|
||||
# 'spotify', not the deezer fallback the functional source downgrades to).
|
||||
return {"success": True, "albums": albums,
|
||||
"primary_source": runtime.get_primary_source_label()}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error searching albums for import: %s", exc)
|
||||
return {"success": False, "error": str(exc)}, 500
|
||||
|
|
@ -366,6 +543,11 @@ def album_process(runtime: ImportRouteRuntime, data: Dict[str, Any]) -> tuple[Di
|
|||
)
|
||||
runtime.refresh_import_suggestions_cache()
|
||||
|
||||
# Files just left staging — drop the shared scan so the next files/groups/hints
|
||||
# reflects reality immediately instead of waiting out the cache TTL.
|
||||
if processed > 0:
|
||||
invalidate_staging_scan_cache()
|
||||
|
||||
return {"success": True, "processed": processed, "total": len(matches), "errors": errors}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error processing album import: %s", exc)
|
||||
|
|
@ -385,7 +567,8 @@ def search_tracks(runtime: ImportRouteRuntime, query: str, limit: int = 10) -> t
|
|||
runtime.hydrabase_worker.enqueue(query, "tracks")
|
||||
|
||||
tracks = runtime.search_import_tracks(query, limit=limit)
|
||||
return {"success": True, "tracks": tracks, "primary_source": primary_source}, 200
|
||||
return {"success": True, "tracks": tracks,
|
||||
"primary_source": runtime.get_primary_source_label()}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error searching tracks for import: %s", exc)
|
||||
return {"success": False, "error": str(exc)}, 500
|
||||
|
|
@ -500,6 +683,10 @@ def singles_process(runtime: ImportRouteRuntime, files: list[Dict[str, Any]]) ->
|
|||
)
|
||||
runtime.refresh_import_suggestions_cache()
|
||||
|
||||
# Files just left staging — drop the shared scan so the list updates immediately.
|
||||
if processed > 0:
|
||||
invalidate_staging_scan_cache()
|
||||
|
||||
return {"success": True, "processed": processed, "total": len(files), "errors": errors}, 200
|
||||
except Exception as exc:
|
||||
runtime.logger.error("Error processing singles import: %s", exc)
|
||||
|
|
|
|||
|
|
@ -252,7 +252,7 @@ def record_library_history_download(context: Dict[str, Any]) -> None:
|
|||
origin, origin_context = derive_download_origin(context)
|
||||
|
||||
db = get_database()
|
||||
db.add_library_history_entry(
|
||||
_history_id = db.add_library_history_entry(
|
||||
event_type="download",
|
||||
title=title,
|
||||
artist_name=artist_name,
|
||||
|
|
@ -270,6 +270,10 @@ def record_library_history_download(context: Dict[str, Any]) -> None:
|
|||
origin_context=origin_context,
|
||||
verification_status=context.get("_verification_status"),
|
||||
)
|
||||
# Stash the row id so the live download task can link to its
|
||||
# library_history row (the Unverified review queue needs it).
|
||||
if isinstance(_history_id, int) and _history_id > 0:
|
||||
context["_history_id"] = _history_id
|
||||
except Exception as e:
|
||||
logger.debug("library history record failed: %s", e)
|
||||
|
||||
|
|
|
|||
289
core/imports/silence.py
Normal file
289
core/imports/silence.py
Normal file
|
|
@ -0,0 +1,289 @@
|
|||
"""Audio-completeness guard — detect files whose container duration looks
|
||||
right but whose REAL audio is far shorter, or mostly silence.
|
||||
|
||||
Motivating bug: HiFi/Monochrome HLS assembly can yield a file whose container
|
||||
claims the full track length (e.g. 3:08) while only ~30s of audio actually
|
||||
decodes — the rest is missing. The duration-agreement and quality guards both
|
||||
pass (mutagen reads the container's 3:08 and the format/bitrate are fine), so
|
||||
nothing catches it until you listen. ffmpeg's ``time=`` even reports 0 with no
|
||||
error on such a file, so the robust signal is to DECODE the audio and compare
|
||||
the real duration (sample count / sample rate, via ``astats``) against the
|
||||
container duration. A separate ``silencedetect`` pass also flags genuine
|
||||
silence-padding.
|
||||
|
||||
The parsers here are pure and unit-tested; the ffmpeg invocations are
|
||||
integration glue that fails open (returns None) when ffmpeg or mutagen can't
|
||||
run, so a tooling problem never blocks a legitimate import.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("imports.silence")
|
||||
|
||||
# Real decoded audio must cover at least this fraction of the container
|
||||
# duration. A legit file decodes to ~100% (encoder padding aside); a truncated
|
||||
# file decodes to a small fraction (the Blossom file: 30s of a 188s container
|
||||
# = 16%). 0.85 leaves generous headroom against false positives.
|
||||
DEFAULT_MIN_DURATION_RATIO = 0.85
|
||||
|
||||
_SAMPLES_RE = re.compile(r"Number of samples:\s*([0-9]+)")
|
||||
|
||||
# Defaults: treat audio below -50 dB lasting >= 2s as silence, and reject when
|
||||
# more than half the track is silent. A normal song — even with quiet intros/
|
||||
# outros — sits far below 0.5; a 30s-real + padded-silence file sits near 0.85.
|
||||
DEFAULT_NOISE_DB = -50
|
||||
DEFAULT_MIN_SILENCE_S = 2.0
|
||||
DEFAULT_THRESHOLD = 0.5
|
||||
|
||||
_SILENCE_DURATION_RE = re.compile(r"silence_duration:\s*([0-9]+(?:\.[0-9]+)?)")
|
||||
|
||||
|
||||
def silence_ratio_from_output(ffmpeg_stderr: str, total_duration_s: float) -> float:
|
||||
"""Fraction of *total_duration_s* covered by detected silence.
|
||||
|
||||
Sums every ``silence_duration: X`` reported by ffmpeg ``silencedetect``
|
||||
and divides by the track length. Capped at 1.0; returns 0.0 when the
|
||||
duration is unknown/zero or no silence was reported.
|
||||
"""
|
||||
if not total_duration_s or total_duration_s <= 0:
|
||||
return 0.0
|
||||
total_silence = sum(float(m) for m in _SILENCE_DURATION_RE.findall(ffmpeg_stderr or ""))
|
||||
if total_silence <= 0:
|
||||
return 0.0
|
||||
return min(total_silence / total_duration_s, 1.0)
|
||||
|
||||
|
||||
def is_mostly_silent_reason(
|
||||
ffmpeg_stderr: str,
|
||||
total_duration_s: float,
|
||||
*,
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
) -> Optional[str]:
|
||||
"""Return a rejection reason when the silent fraction meets *threshold*."""
|
||||
ratio = silence_ratio_from_output(ffmpeg_stderr, total_duration_s)
|
||||
if ratio >= threshold:
|
||||
pct = round(ratio * 100)
|
||||
audible_s = round(total_duration_s * (1 - ratio))
|
||||
return (
|
||||
f"Audio is mostly silent: {pct}% silence (only ~{audible_s}s audible of "
|
||||
f"{round(total_duration_s)}s) — likely a truncated/preview file padded "
|
||||
f"to full length"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _ffmpeg_available() -> bool:
|
||||
try:
|
||||
subprocess.run(
|
||||
["ffmpeg", "-version"],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=10, check=True,
|
||||
)
|
||||
return True
|
||||
except (FileNotFoundError, subprocess.SubprocessError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def _probe_duration_s(file_path: str) -> Optional[float]:
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(file_path)
|
||||
if audio and audio.info and getattr(audio.info, "length", None):
|
||||
return float(audio.info.length)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("silence guard duration probe failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
|
||||
def detect_mostly_silent(
|
||||
file_path: str,
|
||||
*,
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
noise_db: int = DEFAULT_NOISE_DB,
|
||||
min_silence_s: float = DEFAULT_MIN_SILENCE_S,
|
||||
) -> Optional[str]:
|
||||
"""Run ffmpeg ``silencedetect`` over *file_path* and return a rejection
|
||||
reason when the file is mostly silence, else None.
|
||||
|
||||
Fails open: returns None when ffmpeg/mutagen are unavailable or error, so
|
||||
a tooling problem never quarantines a legitimate file.
|
||||
"""
|
||||
if not _ffmpeg_available():
|
||||
logger.debug("silence guard skipped — ffmpeg not available")
|
||||
return None
|
||||
|
||||
total_duration_s = _probe_duration_s(file_path)
|
||||
if not total_duration_s:
|
||||
return None
|
||||
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
"ffmpeg", "-hide_banner", "-nostats", "-i", file_path,
|
||||
"-af", f"silencedetect=noise={noise_db}dB:d={min_silence_s}",
|
||||
"-f", "null", "-",
|
||||
],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
|
||||
timeout=120,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
logger.debug("silence guard ffmpeg run failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace") if proc.stderr else ""
|
||||
return is_mostly_silent_reason(stderr, total_duration_s, threshold=threshold)
|
||||
|
||||
|
||||
# ── Truncation: real decoded duration vs container duration ────────────────
|
||||
|
||||
def measured_duration_from_astats(astats_stderr: str, sample_rate: int) -> Optional[float]:
|
||||
"""Real decoded audio duration in seconds from ffmpeg ``astats`` output.
|
||||
|
||||
``astats`` reports the per-channel ``Number of samples``; dividing by the
|
||||
sample rate gives the true decoded length. Returns None when the sample
|
||||
count or sample rate is unavailable.
|
||||
"""
|
||||
if not sample_rate or sample_rate <= 0:
|
||||
return None
|
||||
m = _SAMPLES_RE.search(astats_stderr or "")
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)) / float(sample_rate)
|
||||
|
||||
|
||||
def is_dsd_path(file_path: str) -> bool:
|
||||
"""True for DSD audio (.dsf / .dff). The decoded-samples truncation check is
|
||||
invalid for DSD: ffmpeg decodes DSD to PCM at a different rate than the DSD
|
||||
container's 2.8 MHz, so samples ÷ container-sample-rate massively under-counts
|
||||
and would falsely report the file as truncated (#939)."""
|
||||
return os.path.splitext(str(file_path or ''))[1].lower() in ('.dsf', '.dff')
|
||||
|
||||
|
||||
def incomplete_audio_reason(
|
||||
measured_s: Optional[float],
|
||||
container_s: Optional[float],
|
||||
*,
|
||||
min_ratio: float = DEFAULT_MIN_DURATION_RATIO,
|
||||
) -> Optional[str]:
|
||||
"""Return a rejection reason when the real decoded duration falls short of
|
||||
the container duration (a truncated file whose metadata over-states length).
|
||||
"""
|
||||
if not measured_s or not container_s or container_s <= 0:
|
||||
return None
|
||||
if measured_s >= container_s * min_ratio:
|
||||
return None
|
||||
pct = round(measured_s / container_s * 100)
|
||||
return (
|
||||
f"Incomplete audio: only ~{round(measured_s)}s actually decodes of a "
|
||||
f"{round(container_s)}s file ({pct}%) — truncated/broken download "
|
||||
f"(container duration over-states the real audio)"
|
||||
)
|
||||
|
||||
|
||||
def _measured_audio_duration_s(file_path: str, sample_rate: int) -> Optional[float]:
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["ffmpeg", "-hide_banner", "-nostats", "-i", file_path,
|
||||
"-af", "astats=metadata=1", "-f", "null", "-"],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=120,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
logger.debug("astats run failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace") if proc.stderr else ""
|
||||
return measured_duration_from_astats(stderr, sample_rate)
|
||||
|
||||
|
||||
def detect_incomplete_audio(
|
||||
file_path: str,
|
||||
*,
|
||||
min_ratio: float = DEFAULT_MIN_DURATION_RATIO,
|
||||
) -> Optional[str]:
|
||||
"""Decode the file and reject when the real audio is far shorter than the
|
||||
container claims. Fails open when ffmpeg/mutagen are unavailable.
|
||||
"""
|
||||
if not _ffmpeg_available():
|
||||
return None
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(file_path)
|
||||
if not (audio and audio.info):
|
||||
return None
|
||||
container_s = float(getattr(audio.info, "length", 0) or 0)
|
||||
sample_rate = int(getattr(audio.info, "sample_rate", 0) or 0)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("container probe failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
measured_s = _measured_audio_duration_s(file_path, sample_rate)
|
||||
return incomplete_audio_reason(measured_s, container_s, min_ratio=min_ratio)
|
||||
|
||||
|
||||
def detect_broken_audio(
|
||||
file_path: str,
|
||||
*,
|
||||
min_ratio: float = DEFAULT_MIN_DURATION_RATIO,
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
noise_db: int = DEFAULT_NOISE_DB,
|
||||
min_silence_s: float = DEFAULT_MIN_SILENCE_S,
|
||||
) -> Optional[str]:
|
||||
"""Combined post-download audio guard: reject a file that is truncated
|
||||
(real audio far shorter than the container) or mostly silence. Returns the
|
||||
first failure reason, or None when the audio looks complete.
|
||||
|
||||
Runs a SINGLE ffmpeg decode pass with both the ``astats`` (truncation) and
|
||||
``silencedetect`` (silence) filters chained — one decode of the file feeds
|
||||
both checks instead of two full decodes. Halves the CPU cost versus running
|
||||
``detect_incomplete_audio`` and ``detect_mostly_silent`` back to back.
|
||||
|
||||
Fails open: returns None when ffmpeg/mutagen are unavailable or error, so a
|
||||
tooling problem never quarantines a legitimate file.
|
||||
"""
|
||||
if not _ffmpeg_available():
|
||||
logger.debug("audio guard skipped — ffmpeg not available")
|
||||
return None
|
||||
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(file_path)
|
||||
if not (audio and audio.info):
|
||||
return None
|
||||
container_s = float(getattr(audio.info, "length", 0) or 0)
|
||||
sample_rate = int(getattr(audio.info, "sample_rate", 0) or 0)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("container probe failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
"ffmpeg", "-hide_banner", "-nostats", "-i", file_path,
|
||||
"-af", f"astats=metadata=1,silencedetect=noise={noise_db}dB:d={min_silence_s}",
|
||||
"-f", "null", "-",
|
||||
],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=120,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
logger.debug("audio guard ffmpeg run failed for %s: %s", file_path, exc)
|
||||
return None
|
||||
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace") if proc.stderr else ""
|
||||
|
||||
# Truncation check first (real audio far shorter than the container) — but
|
||||
# NOT for DSD: the astats sample-count ÷ DSD-rate math is invalid there and
|
||||
# would always false-positive (#939). Silence detection below still applies.
|
||||
if not is_dsd_path(file_path):
|
||||
measured_s = measured_duration_from_astats(stderr, sample_rate)
|
||||
reason = incomplete_audio_reason(measured_s, container_s, min_ratio=min_ratio)
|
||||
if reason:
|
||||
return reason
|
||||
|
||||
# Then silence-padding (mostly-silent file).
|
||||
return is_mostly_silent_reason(stderr, container_s, threshold=threshold)
|
||||
|
|
@ -56,6 +56,12 @@ def get_primary_source() -> str:
|
|||
return _get_primary_source()
|
||||
|
||||
|
||||
def get_primary_source_label() -> str:
|
||||
from core.metadata_service import get_primary_source_label as _get_primary_source_label
|
||||
|
||||
return _get_primary_source_label()
|
||||
|
||||
|
||||
def get_source_priority(preferred_source: str):
|
||||
from core.metadata_service import get_source_priority as _get_source_priority
|
||||
|
||||
|
|
|
|||
|
|
@ -159,3 +159,53 @@ def resolve_track_number(
|
|||
# value the pre-fix resolver would have used. A correctly-named file
|
||||
# with a stale/wrong embedded tag is therefore never regressed.
|
||||
return _coerce_positive(embedded_track_number)
|
||||
|
||||
|
||||
def normalize_disc_number(value) -> int:
|
||||
"""Coerce a disc value to a positive int, defaulting to 1.
|
||||
|
||||
Every track in a multi-disc album MUST carry a disc number, or Jellyfin/Plex
|
||||
leave the disc-less ones floating ungrouped above the disc sections (Sokhi's
|
||||
"tracks 3/9/15 at the top"). Upstream sources can hand back 0, None, '', or a
|
||||
non-numeric string for some tracks — especially when a track resolved to a
|
||||
different edition than its siblings — and the tag-writer only wrote the disc
|
||||
tag when it was truthy, so those tracks lost it entirely on the clear-then-
|
||||
rewrite. Flooring to >=1 here means a track is never written disc-less.
|
||||
"""
|
||||
try:
|
||||
n = int(value) # int, float, or clean int-string
|
||||
except (TypeError, ValueError):
|
||||
try:
|
||||
n = int(float(str(value).strip())) # tolerate "2.0"
|
||||
except (TypeError, ValueError):
|
||||
return 1
|
||||
return n if n >= 1 else 1
|
||||
|
||||
|
||||
def resolve_disc_for_track(original_search, album_info) -> int:
|
||||
"""The disc number for a track — resolved IDENTICALLY for the 'Disc N' folder
|
||||
(import pipeline) and the embedded tag (metadata.source), so the two can never
|
||||
disagree.
|
||||
|
||||
Sokhi: the pipeline synced the resolved track_number into album_info (so the
|
||||
folder matched the tag) but never did the same for disc — the folder used
|
||||
album_info's original disc (often 1) while the tag took the per-track disc
|
||||
(e.g. 2/3 from a MusicBrainz multi-medium release). Result: a disc-2/3 track
|
||||
landed in the Disc 1 folder, collapsing every disc's tracks into one folder.
|
||||
|
||||
Returns the first VALID positive disc — the per-track search's, else the album
|
||||
context's — else 1. A falsy/unknown (0/None/'') per-track disc falls through to
|
||||
the album rather than flooring early. Single source of truth so both call sites
|
||||
stay in lockstep."""
|
||||
for src in ((original_search or {}), (album_info or {})):
|
||||
raw = src.get("disc_number")
|
||||
try:
|
||||
n = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
try:
|
||||
n = int(float(str(raw).strip()))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if n >= 1:
|
||||
return n
|
||||
return 1
|
||||
|
|
|
|||
|
|
@ -103,9 +103,12 @@ def select_version_mismatch_fallback(
|
|||
# don't guess which the user wants.
|
||||
return None
|
||||
|
||||
# First tried = oldest = highest-confidence (the retry walks candidates
|
||||
# best-first). The id is a "<date>_<time>_<name>" timestamp prefix, so the
|
||||
# lexicographically smallest id is the earliest attempt.
|
||||
# First tried = oldest = best (the retry walks candidates best-first; what
|
||||
# "best" means follows the active ordering — confidence-first by default, or
|
||||
# ranked-target quality when best_quality mode / the rank_candidates_by_quality
|
||||
# toggle is on, so this naturally accepts the highest-quality candidate then).
|
||||
# The id is a "<date>_<time>_<name>" timestamp prefix, so the lexicographically
|
||||
# smallest id is the earliest attempt.
|
||||
return min((e for _, e in candidates), key=lambda e: e["id"])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -739,9 +739,27 @@ class iTunesClient:
|
|||
cache = get_metadata_cache()
|
||||
cached = cache.get_entity('itunes', 'album', f"{album_id}_tracks")
|
||||
if cached and cached.get('items'):
|
||||
return cached
|
||||
# #918 follow-up: a tracks entry cached BEFORE the limit=200 fix is truncated
|
||||
# to 50 and survives in the persistent cache (30-day TTL), so every window that
|
||||
# loads this album from cache still shows 50 — not just the one path that was
|
||||
# re-fetched fresh. Self-heal: entries written by the fixed fetch carry
|
||||
# `_complete`; a legacy entry without it is re-validated against the album's
|
||||
# known trackCount and re-fetched if it's short. (trackCount comes from the
|
||||
# collection metadata and is unaffected by the tracks-limit bug.)
|
||||
if cached.get('_complete'):
|
||||
return cached
|
||||
album_meta = cache.get_entity('itunes', 'album', str(album_id))
|
||||
expected = (album_meta or {}).get('trackCount')
|
||||
if not (isinstance(expected, int) and expected > len(cached['items'])):
|
||||
return cached
|
||||
logger.info(
|
||||
"iTunes album %s tracks cache looks truncated (%d cached < %d trackCount) — refetching",
|
||||
album_id, len(cached['items']), expected,
|
||||
)
|
||||
|
||||
results = self._lookup(id=album_id, entity='song')
|
||||
# #918: the iTunes Lookup API returns only 50 related entities unless `limit` is
|
||||
# passed (max 200), so albums >50 tracks were truncated in the download window.
|
||||
results = self._lookup(id=album_id, entity='song', limit=200)
|
||||
|
||||
if not results:
|
||||
return None
|
||||
|
|
@ -763,7 +781,7 @@ class iTunesClient:
|
|||
try:
|
||||
fb_results = self.session.get(
|
||||
self.LOOKUP_URL,
|
||||
params={'id': album_id, 'entity': 'song', 'country': fallback},
|
||||
params={'id': album_id, 'entity': 'song', 'country': fallback, 'limit': 200}, # #918
|
||||
timeout=15
|
||||
)
|
||||
if fb_results.status_code == 200:
|
||||
|
|
@ -849,7 +867,11 @@ class iTunesClient:
|
|||
'items': tracks,
|
||||
'total': len(tracks),
|
||||
'limit': len(tracks),
|
||||
'next': None
|
||||
'next': None,
|
||||
# Marks this entry as fetched with the limit=200 query (#918) so the
|
||||
# stale-cache self-heal above trusts it and never re-fetches in a loop —
|
||||
# important for region-restricted albums where len(tracks) < trackCount.
|
||||
'_complete': True,
|
||||
}
|
||||
|
||||
# Cache the album tracks listing
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from datetime import datetime
|
|||
import json
|
||||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
from core.library.bulk_paginate import paginate_all_items
|
||||
|
||||
# Shared dataclasses live in the neutral media_server package — every
|
||||
# server client used to define a near-identical XTrackInfo /
|
||||
|
|
@ -105,6 +106,7 @@ class JellyfinTrack:
|
|||
self.title = jellyfin_data.get('Name', 'Unknown Track')
|
||||
self.duration = jellyfin_data.get('RunTimeTicks', 0) // 10000 # Convert from ticks to milliseconds
|
||||
self.trackNumber = jellyfin_data.get('IndexNumber')
|
||||
self.discNumber = jellyfin_data.get('ParentIndexNumber') # multi-disc: disc number
|
||||
self.year = jellyfin_data.get('ProductionYear')
|
||||
self.userRating = jellyfin_data.get('UserData', {}).get('Rating')
|
||||
self.addedAt = self._parse_date(jellyfin_data.get('DateCreated'))
|
||||
|
|
@ -511,12 +513,8 @@ class JellyfinClient(MediaServerClient):
|
|||
try:
|
||||
# SIMPLIFIED APPROACH: Fetch all tracks, then all albums separately (robust and fast)
|
||||
logger.info("Fetching all tracks in bulk...")
|
||||
all_tracks = []
|
||||
start_index = 0
|
||||
limit = 10000
|
||||
consecutive_failures = 0
|
||||
|
||||
while True:
|
||||
|
||||
def _fetch_tracks_page(start_index, limit):
|
||||
params = {
|
||||
'ParentId': self.music_library_id,
|
||||
'IncludeItemTypes': 'Audio',
|
||||
|
|
@ -527,41 +525,19 @@ class JellyfinClient(MediaServerClient):
|
|||
'StartIndex': start_index,
|
||||
'Limit': limit
|
||||
}
|
||||
|
||||
response = self._make_request(f'/Users/{self.user_id}/Items', params)
|
||||
|
||||
if not response:
|
||||
consecutive_failures += 1
|
||||
# Wait before retrying — the server may still be processing the timed-out request
|
||||
time.sleep(5)
|
||||
if limit > 1000:
|
||||
limit = limit // 2
|
||||
consecutive_failures = 0 # Reset — give the smaller batch a fair chance
|
||||
logger.warning(f"Track fetch failed - reducing batch size to {limit}")
|
||||
continue
|
||||
elif consecutive_failures >= 2:
|
||||
logger.warning("Multiple track fetch failures at minimum batch size - stopping")
|
||||
break
|
||||
else:
|
||||
logger.warning("Track fetch failed at minimum batch size - retrying once")
|
||||
continue
|
||||
return response.get('Items', []) if response else None # None = failed page
|
||||
|
||||
# Page in modest chunks so progress is reported every page — a single
|
||||
# huge silent request used to trip the 300s no-progress watchdog on
|
||||
# slow servers even though it was alive (see bulk_paginate docstring).
|
||||
all_tracks = paginate_all_items(
|
||||
_fetch_tracks_page,
|
||||
report_progress=self._progress_callback,
|
||||
label="tracks",
|
||||
on_retry_wait=lambda: time.sleep(5),
|
||||
)
|
||||
|
||||
consecutive_failures = 0
|
||||
batch_tracks = response.get('Items', [])
|
||||
if not batch_tracks:
|
||||
break
|
||||
|
||||
all_tracks.extend(batch_tracks)
|
||||
|
||||
if len(batch_tracks) < limit:
|
||||
break
|
||||
|
||||
start_index += limit
|
||||
progress_msg = f"Fetched {len(all_tracks)} tracks so far..."
|
||||
logger.info(f" {progress_msg} (batch size: {limit})")
|
||||
if self._progress_callback:
|
||||
self._progress_callback(progress_msg)
|
||||
|
||||
# Group tracks by album ID for instant lookup
|
||||
self._track_cache = {}
|
||||
for track_data in all_tracks:
|
||||
|
|
@ -577,12 +553,8 @@ class JellyfinClient(MediaServerClient):
|
|||
|
||||
# STEP 2: Fetch all albums in bulk (same proven pattern)
|
||||
logger.info("Fetching all albums in bulk...")
|
||||
all_albums = []
|
||||
start_index = 0
|
||||
limit = 10000
|
||||
consecutive_failures = 0
|
||||
|
||||
while True:
|
||||
|
||||
def _fetch_albums_page(start_index, limit):
|
||||
params = {
|
||||
'ParentId': self.music_library_id,
|
||||
'IncludeItemTypes': 'MusicAlbum',
|
||||
|
|
@ -593,41 +565,16 @@ class JellyfinClient(MediaServerClient):
|
|||
'StartIndex': start_index,
|
||||
'Limit': limit
|
||||
}
|
||||
|
||||
response = self._make_request(f'/Users/{self.user_id}/Items', params)
|
||||
|
||||
if not response:
|
||||
consecutive_failures += 1
|
||||
# Wait before retrying — the server may still be processing the timed-out request
|
||||
time.sleep(5)
|
||||
if limit > 1000:
|
||||
limit = limit // 2
|
||||
consecutive_failures = 0 # Reset — give the smaller batch a fair chance
|
||||
logger.warning(f"Album fetch failed - reducing batch size to {limit}")
|
||||
continue
|
||||
elif consecutive_failures >= 2:
|
||||
logger.warning("Multiple album fetch failures at minimum batch size - stopping")
|
||||
break
|
||||
else:
|
||||
logger.warning("Album fetch failed at minimum batch size - retrying once")
|
||||
continue
|
||||
return response.get('Items', []) if response else None # None = failed page
|
||||
|
||||
all_albums = paginate_all_items(
|
||||
_fetch_albums_page,
|
||||
report_progress=self._progress_callback,
|
||||
label="albums",
|
||||
on_retry_wait=lambda: time.sleep(5),
|
||||
)
|
||||
|
||||
consecutive_failures = 0
|
||||
batch_albums = response.get('Items', [])
|
||||
if not batch_albums:
|
||||
break
|
||||
|
||||
all_albums.extend(batch_albums)
|
||||
|
||||
if len(batch_albums) < limit:
|
||||
break
|
||||
|
||||
start_index += limit
|
||||
progress_msg = f"Fetched {len(all_albums)} albums so far..."
|
||||
logger.info(f" {progress_msg} (batch size: {limit})")
|
||||
if self._progress_callback:
|
||||
self._progress_callback(progress_msg)
|
||||
|
||||
# Group albums by artist ID for instant lookup
|
||||
self._album_cache = {}
|
||||
for album_data in all_albums:
|
||||
|
|
@ -1711,6 +1658,81 @@ class JellyfinClient(MediaServerClient):
|
|||
logger.error(f"Error reconciling Jellyfin playlist '{playlist_name}': {e}")
|
||||
return False
|
||||
|
||||
def get_playlist_track_ids(self, playlist_id: str) -> List[str]:
|
||||
"""The playlist's current track ids (Item Ids), in current order. [] on miss."""
|
||||
if not self.ensure_connection():
|
||||
return []
|
||||
try:
|
||||
resp = self._make_request(f'/Playlists/{playlist_id}/Items', {'UserId': self.user_id})
|
||||
if not resp:
|
||||
return []
|
||||
return [str(i.get('Id')) for i in resp.get('Items', []) if i.get('Id')]
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting Jellyfin playlist track ids '{playlist_id}': {e}")
|
||||
return []
|
||||
|
||||
def reorder_playlist(self, playlist_id, playlist_name: str, ordered_ids) -> bool:
|
||||
"""In-place reorder a playlist to an exact ordered track-id list ('Align
|
||||
playlists'). Removes any current entry whose track NOT in ``ordered_ids``
|
||||
('Mirror source' drops extras; 'Keep extras' keeps them in the list), then
|
||||
moves each desired track to its target index via the Jellyfin Move endpoint.
|
||||
Operates on the existing playlist (DELETE EntryIds + Items/{entryId}/Move/{i})
|
||||
so its poster, name and Id survive — no delete/recreate."""
|
||||
if not self.ensure_connection():
|
||||
return False
|
||||
try:
|
||||
import requests
|
||||
ordered = [str(i) for i in (ordered_ids or []) if str(i)]
|
||||
ordered_set = set(ordered)
|
||||
|
||||
# Entries carry both the track Id and the PlaylistItemId (entry id);
|
||||
# move/remove operate on the entry id.
|
||||
entries = [] # (track_id, entry_id) in current order
|
||||
resp = self._make_request(f'/Playlists/{playlist_id}/Items', {'UserId': self.user_id})
|
||||
if resp:
|
||||
for item in resp.get('Items', []):
|
||||
tid = str(item.get('Id') or '')
|
||||
eid = str(item.get('PlaylistItemId') or '')
|
||||
if tid:
|
||||
entries.append((tid, eid))
|
||||
if not entries:
|
||||
logger.error(f"Jellyfin reorder: no entries for playlist {playlist_id}")
|
||||
return False
|
||||
|
||||
by_tid = {tid: eid for tid, eid in entries}
|
||||
hdr = {'X-Emby-Token': self.api_key}
|
||||
|
||||
# Drop entries not in the desired list (extras, for Mirror source).
|
||||
extra_eids = [eid for tid, eid in entries if tid not in ordered_set and eid]
|
||||
for i in range(0, len(extra_eids), 100):
|
||||
batch = extra_eids[i:i + 100]
|
||||
r = requests.delete(
|
||||
f"{self.base_url}/Playlists/{playlist_id}/Items",
|
||||
params={'EntryIds': ','.join(batch)}, headers=hdr, timeout=30,
|
||||
)
|
||||
if r.status_code not in (200, 204):
|
||||
logger.error(f"Jellyfin reorder remove failed: HTTP {r.status_code}")
|
||||
return False
|
||||
|
||||
# Move each desired track to its target index, ascending — each move
|
||||
# lands the item exactly at index i without disturbing 0..i-1.
|
||||
for idx, tid in enumerate(ordered):
|
||||
eid = by_tid.get(tid)
|
||||
if not eid:
|
||||
continue
|
||||
r = requests.post(
|
||||
f"{self.base_url}/Playlists/{playlist_id}/Items/{eid}/Move/{idx}",
|
||||
headers=hdr, timeout=30,
|
||||
)
|
||||
if r.status_code not in (200, 204):
|
||||
logger.warning(f"Jellyfin reorder move failed for {tid}: HTTP {r.status_code}")
|
||||
return False
|
||||
logger.info(f"Aligned Jellyfin playlist '{playlist_name}' order ({len(ordered)} tracks)")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error reordering Jellyfin playlist '{playlist_name}': {e}")
|
||||
return False
|
||||
|
||||
def update_playlist(self, playlist_name: str, tracks) -> bool:
|
||||
"""Update an existing playlist or create it if it doesn't exist"""
|
||||
if not self.ensure_connection():
|
||||
|
|
|
|||
93
core/library/bulk_paginate.py
Normal file
93
core/library/bulk_paginate.py
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
"""Paginate a bulk media-server fetch while feeding the no-progress watchdog.
|
||||
|
||||
The library scan fetches every track/album by paging a server API. The DB-update
|
||||
watchdog (``core/database_update_health.py``) kills a job that reports no progress
|
||||
for 300s. The old Jellyfin fetch used a single 10 000-item page, so a whole
|
||||
library came back in ONE request that emitted NO progress while it was in flight —
|
||||
on a slow server that single request exceeded 300s and the watchdog declared the
|
||||
job "stuck" even though it was alive, not hung (Discord: DXP4800 NAS, 7148 tracks,
|
||||
"Fetching all tracks in bulk…").
|
||||
|
||||
``paginate_all_items`` pages at a size chosen so progress is emitted on a cadence
|
||||
set by the PAGE SIZE, not the library size — the watchdog is fed every page, so it
|
||||
can never starve mid-fetch regardless of how big the library is. It is pure: all
|
||||
I/O lives in the injected ``fetch_page``, so the pagination + progress + failure-
|
||||
shrink logic is unit-testable without a server.
|
||||
|
||||
This does NOT change WHAT is fetched (same query, same fields, same items) — only
|
||||
how it's paged and that every page reports progress (the old loop skipped progress
|
||||
on the final/only page, which is the entire bug for a sub-page-size library).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, List, Optional
|
||||
|
||||
# Page size for bulk library fetches. Small enough that a single request stays
|
||||
# well under the 300s no-progress watchdog even on a slow NAS, and that progress
|
||||
# is reported every page. NOT a performance knob — a resilience/observability one.
|
||||
DEFAULT_PAGE_SIZE = 1000
|
||||
|
||||
# Floor the failure-shrink can reach before giving up — a server that can't return
|
||||
# even this many items in one request is genuinely struggling.
|
||||
DEFAULT_MIN_PAGE_SIZE = 250
|
||||
|
||||
|
||||
def paginate_all_items(
|
||||
fetch_page: Callable[[int, int], Optional[List[Any]]],
|
||||
*,
|
||||
report_progress: Optional[Callable[[str], None]] = None,
|
||||
label: str = "items",
|
||||
page_size: int = DEFAULT_PAGE_SIZE,
|
||||
min_page_size: int = DEFAULT_MIN_PAGE_SIZE,
|
||||
on_retry_wait: Optional[Callable[[], None]] = None,
|
||||
) -> List[Any]:
|
||||
"""Page through ``fetch_page(start_index, limit)`` until the server is drained.
|
||||
|
||||
``fetch_page`` returns the page's items (a list, possibly empty = end), or
|
||||
``None`` to signal a FAILED request (timeout/error) — on failure the page size
|
||||
is halved down to ``min_page_size`` and retried, then abandoned after two
|
||||
consecutive failures at the floor.
|
||||
|
||||
Progress is reported after EVERY non-empty page (including the final/only one),
|
||||
so a no-progress watchdog is fed on a cadence set by ``page_size`` — never by
|
||||
the total library size. Returns every item gathered.
|
||||
"""
|
||||
items: List[Any] = []
|
||||
start_index = 0
|
||||
limit = page_size
|
||||
consecutive_failures = 0
|
||||
|
||||
while True:
|
||||
batch = fetch_page(start_index, limit)
|
||||
|
||||
if batch is None: # failed request
|
||||
consecutive_failures += 1
|
||||
if on_retry_wait is not None:
|
||||
on_retry_wait()
|
||||
if limit > min_page_size:
|
||||
limit = max(min_page_size, limit // 2)
|
||||
consecutive_failures = 0 # give the smaller batch a fair chance
|
||||
continue
|
||||
if consecutive_failures >= 2:
|
||||
break # struggling at the floor — stop with what we have
|
||||
continue
|
||||
|
||||
consecutive_failures = 0
|
||||
if not batch:
|
||||
break # drained
|
||||
|
||||
items.extend(batch)
|
||||
# Feed the watchdog on EVERY page — this is the line the old loop only ran
|
||||
# when there was a *next* page, so a sub-page-size library reported nothing.
|
||||
if report_progress is not None:
|
||||
report_progress(f"Fetched {len(items)} {label} so far...")
|
||||
|
||||
if len(batch) < limit:
|
||||
break # last (partial) page
|
||||
start_index += limit
|
||||
|
||||
return items
|
||||
|
||||
|
||||
__all__ = ["paginate_all_items", "DEFAULT_PAGE_SIZE", "DEFAULT_MIN_PAGE_SIZE"]
|
||||
|
|
@ -10,6 +10,7 @@ from __future__ import annotations
|
|||
from dataclasses import dataclass
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
|
@ -121,6 +122,20 @@ def import_existing_track_for_album_slot(album_id: str, payload: dict, deps: Mis
|
|||
if album_data.get("server_source") and source_track.get("server_source") and album_data["server_source"] != source_track["server_source"]:
|
||||
raise MissingTrackImportError("Selected track belongs to a different library source", 400)
|
||||
|
||||
# #917: "I have this" rebuilds the destination path from album metadata. When the album row
|
||||
# has no year, the rebuilt path drops the $year and the copied file lands in a NEW, yearless
|
||||
# directory instead of the album's existing folder. Recover the year from a sibling track so
|
||||
# the import reuses the same directory.
|
||||
if not album_data.get("year"):
|
||||
recovered_year = _existing_album_year_from_sibling(
|
||||
database, album_id, deps.resolve_library_file_path_fn,
|
||||
int(expected.get("disc_number") or 1), int(expected.get("track_number") or 1),
|
||||
)
|
||||
if recovered_year:
|
||||
album_data["year"] = recovered_year
|
||||
logger.info("[I Have This] recovered album year %s from existing folder for album %s",
|
||||
recovered_year, album_id)
|
||||
|
||||
source_path = deps.resolve_library_file_path_fn(source_track.get("file_path"))
|
||||
if not source_path:
|
||||
raise MissingTrackImportError(_file_not_found_message(source_track.get("file_path")), 404)
|
||||
|
|
@ -426,6 +441,50 @@ def _sync_imported_track(deps: MissingTrackImportDeps, track_id, expected_title:
|
|||
logger.debug("Existing-track import server sync skipped/failed: %s", sync_err)
|
||||
|
||||
|
||||
def _existing_album_year_from_sibling(
|
||||
database,
|
||||
album_id: str,
|
||||
resolve_library_file_path_fn: Callable[[Optional[str]], Optional[str]],
|
||||
target_disc: int,
|
||||
target_track: int,
|
||||
) -> Optional[str]:
|
||||
"""Find the release year already baked into this album's on-disk folder (#917).
|
||||
|
||||
Read from a sibling track — its own ``year`` column first, else a ``(YYYY)`` /
|
||||
``[YYYY]`` in the album folder name — so an "I have this" import reuses the album's
|
||||
existing directory instead of rebuilding a yearless one. Returns the 4-digit year
|
||||
string, or None when no signal exists.
|
||||
"""
|
||||
try:
|
||||
with database._get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT file_path, year FROM tracks
|
||||
WHERE album_id = ?
|
||||
AND file_path IS NOT NULL AND file_path != ''
|
||||
AND NOT (COALESCE(disc_number, 1) = ? AND track_number = ?)
|
||||
ORDER BY COALESCE(disc_number, 1), track_number
|
||||
LIMIT 12
|
||||
""",
|
||||
(album_id, target_disc, target_track),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
for row in rows:
|
||||
year = row["year"]
|
||||
if year is not None and str(year).strip()[:4].isdigit():
|
||||
return str(year).strip()[:4]
|
||||
resolved = resolve_library_file_path_fn(row["file_path"])
|
||||
if resolved:
|
||||
folder = os.path.basename(os.path.dirname(resolved))
|
||||
match = re.search(r"[(\[](\d{4})[)\]]", folder) # "Album (2019)" / "Album [2019]"
|
||||
if match:
|
||||
return match.group(1)
|
||||
except Exception as exc:
|
||||
logger.debug("Could not recover album year from sibling for %s: %s", album_id, exc)
|
||||
return None
|
||||
|
||||
|
||||
def copy_album_identity_from_target_sibling(
|
||||
database,
|
||||
album_id: str,
|
||||
|
|
|
|||
|
|
@ -138,10 +138,25 @@ def _collect_base_dirs(
|
|||
except Exception as e:
|
||||
logger.debug("music paths read failed: %s", e)
|
||||
|
||||
# Normalize to absolute forms so resolution does NOT depend on the calling
|
||||
# thread's CWD. A relative config like "./Transfer" otherwise only resolves
|
||||
# when os.path.isdir("./Transfer") happens to be true from the current CWD —
|
||||
# which fails in background workers whose CWD isn't the app root, leaving
|
||||
# base_dirs empty and every track "unresolved". For each candidate we try
|
||||
# the raw form first (cheap, preserves an already-absolute path), then its
|
||||
# os.path.abspath() form so "./Transfer" → "/app/Transfer".
|
||||
expanded: list[str] = []
|
||||
for c in candidates:
|
||||
if not c:
|
||||
continue
|
||||
expanded.append(c)
|
||||
if not os.path.isabs(c):
|
||||
expanded.append(os.path.abspath(c))
|
||||
|
||||
# De-duplicate while preserving order, drop empties / non-existent dirs.
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for c in candidates:
|
||||
for c in expanded:
|
||||
if not c or c in seen:
|
||||
continue
|
||||
seen.add(c)
|
||||
|
|
@ -224,10 +239,23 @@ def resolve_library_file_path_with_diagnostic(
|
|||
if not base_dirs:
|
||||
return None, attempt
|
||||
|
||||
# Skip index 0 to avoid drive-letter / leading-slash artifacts
|
||||
# (e.g. "E:" or "" from a leading "/").
|
||||
# Try progressively shorter path suffixes against each base dir.
|
||||
#
|
||||
# Start at index 0 so a clean RELATIVE library path is tried in FULL first.
|
||||
# SoulSync's own library scanner stores paths like
|
||||
# "Asketa/Another Side/01 - Track.flac" (no leading slash) — index 0 is the
|
||||
# artist folder and dropping it (the old range(1, ...)) meant the artist
|
||||
# segment was never joined, so nothing under transfer/ ever resolved and
|
||||
# every track looked unreadable to the quality scanner.
|
||||
#
|
||||
# For ABSOLUTE media-server paths ("/music/Artist/Album/track.flac") index 0
|
||||
# is the empty leading segment and i=0 yields os.path.join(base, "", ...) ==
|
||||
# base/Artist/... which simply won't exist and harmlessly falls through to
|
||||
# i=1 ("music/...") etc. A Windows drive part ("E:") at i=0 likewise just
|
||||
# fails on POSIX and falls through. So starting at 0 is safe for every form
|
||||
# and only ADDS the relative-full-path match that was missing.
|
||||
for base in base_dirs:
|
||||
for i in range(1, len(path_parts)):
|
||||
for i in range(0, len(path_parts)):
|
||||
candidate = os.path.join(base, *path_parts[i:])
|
||||
if os.path.exists(candidate):
|
||||
return candidate, attempt
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from core.runtime_state import (
|
|||
download_tasks,
|
||||
tasks_lock,
|
||||
)
|
||||
from core.metadata.album_tracks import get_album_for_source
|
||||
from core.metadata.registry import (
|
||||
get_deezer_client,
|
||||
get_itunes_client,
|
||||
|
|
@ -26,6 +27,27 @@ from database.music_database import get_database
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _album_data_from_source(full: dict, album_id: str, fallback_name: str) -> dict:
|
||||
"""Build the redownload `album_data` from a primary-source get_album result (#915).
|
||||
|
||||
Mirrors the Spotify branch's album_data shape so iTunes/Deezer redownloads carry the
|
||||
real release_date / album_type / total_tracks — instead of a lean {'name': ...} that
|
||||
drops the $year and forces a manual reorganize afterwards."""
|
||||
images = full.get('images') or []
|
||||
image_url = full.get('image_url') or ''
|
||||
if not image_url and images and isinstance(images[0], dict):
|
||||
image_url = images[0].get('url', '')
|
||||
return {
|
||||
'id': str(full.get('id') or album_id),
|
||||
'name': full.get('name') or fallback_name,
|
||||
'release_date': full.get('release_date', ''),
|
||||
'album_type': full.get('album_type', 'album'),
|
||||
'total_tracks': full.get('total_tracks', 0),
|
||||
'images': images,
|
||||
'image_url': image_url,
|
||||
}
|
||||
|
||||
|
||||
def _get_itunes_client():
|
||||
"""Mirror of web_server._get_itunes_client — delegates to registry."""
|
||||
return get_itunes_client()
|
||||
|
|
@ -141,12 +163,26 @@ def redownload_start(track_id):
|
|||
'images': album_images,
|
||||
'image_url': album_images[0]['url'] if album_images else '',
|
||||
}
|
||||
elif meta_source == 'itunes':
|
||||
track_number = full_track_details.get('trackNumber')
|
||||
disc_number = full_track_details.get('discNumber', 1)
|
||||
elif meta_source == 'deezer':
|
||||
track_number = full_track_details.get('track_position')
|
||||
disc_number = full_track_details.get('disk_number', 1)
|
||||
elif meta_source in ('itunes', 'deezer'):
|
||||
# #915: parity with the Spotify branch + Reorganize — pull the full album from
|
||||
# the primary source so album_data carries release_date/album_type/total_tracks
|
||||
# (was lean {'name': ...}, which dropped the $year on iTunes/Deezer redownloads).
|
||||
if meta_source == 'itunes':
|
||||
track_number = full_track_details.get('trackNumber')
|
||||
disc_number = full_track_details.get('discNumber', 1)
|
||||
_alb_id = full_track_details.get('collectionId')
|
||||
else:
|
||||
track_number = full_track_details.get('track_position')
|
||||
disc_number = full_track_details.get('disk_number', 1)
|
||||
_alb_id = (full_track_details.get('album') or {}).get('id')
|
||||
if _alb_id:
|
||||
try:
|
||||
_full_album = get_album_for_source(meta_source, str(_alb_id))
|
||||
except Exception as _alb_err: # noqa: BLE001 — never let metadata break redownload
|
||||
logger.debug("[Redownload] %s album fetch failed: %s", meta_source, _alb_err)
|
||||
_full_album = None
|
||||
if isinstance(_full_album, dict):
|
||||
album_data = _album_data_from_source(_full_album, str(_alb_id), metadata.get('album', ''))
|
||||
|
||||
track_data = {
|
||||
'id': meta_id,
|
||||
|
|
|
|||
104
core/library/standalone_scan.py
Normal file
104
core/library/standalone_scan.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
"""Decision logic for the SoulSync standalone Deep Scan's untracked → Staging move.
|
||||
|
||||
The standalone deep scan (``_run_soulsync_deep_scan`` in web_server) walks the
|
||||
Transfer folder, diffs it against the ``soulsync`` rows in the DB, and relocates
|
||||
every file it can't find a DB record for into Staging for auto-import. That's fine
|
||||
when Transfer is a scratch/landing area — files arrive, get moved, imported, and
|
||||
recorded, so a later scan only ever sees a few genuinely-new arrivals.
|
||||
|
||||
It is a DATA-LOSS trap when the DB is empty or out of sync with disk (a volume
|
||||
swap, a DB reset, external tag edits) while Transfer holds the user's real library:
|
||||
a path-only diff then flags the *entire* library as "untracked" and the scan
|
||||
relocates all of it (issue #904). The same failure mode the orphan detector and the
|
||||
media-server deep scan already guard against (``core.library.stale_guard``) — this
|
||||
path just never used the guard.
|
||||
|
||||
This module is the pure, testable decision: given the Transfer file set, the DB's
|
||||
known paths, and the user's "Transfer is my permanent library" preference, decide
|
||||
WHICH files are untracked and WHETHER it's safe to relocate them. The web layer does
|
||||
only the I/O (walk/move/delete) based on the returned plan.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable, Set
|
||||
|
||||
from core.library.stale_guard import (
|
||||
DEFAULT_MAX_ORPHAN_FRACTION,
|
||||
DEFAULT_MIN_ORPHANS,
|
||||
is_implausible_orphan_flood,
|
||||
)
|
||||
|
||||
# Block reason codes (web layer turns these into a user-facing warning).
|
||||
BLOCK_NONE = ""
|
||||
BLOCK_TRANSFER_PERMANENT = "transfer_permanent"
|
||||
BLOCK_DESYNC = "desync"
|
||||
|
||||
|
||||
def _norm(path: str) -> str:
|
||||
"""Normalize a path for cross-platform comparison (Windows vs Unix separators)."""
|
||||
return str(path).replace("\\", "/")
|
||||
|
||||
|
||||
def diff_untracked(transfer_files: Iterable[str], db_paths: Iterable[str]) -> Set[str]:
|
||||
"""Files present in ``transfer_files`` but with no matching ``db_paths`` record.
|
||||
|
||||
Comparison is separator-normalized, so a DB path stored with one separator style
|
||||
still matches the on-disk path. Pure — no I/O. Returns the original (un-normalized)
|
||||
transfer paths so the caller can act on the real filesystem entries.
|
||||
"""
|
||||
db_norm = {_norm(p) for p in db_paths if p}
|
||||
return {f for f in transfer_files if _norm(f) not in db_norm}
|
||||
|
||||
|
||||
def plan_standalone_deep_scan(
|
||||
transfer_files: Iterable[str],
|
||||
db_paths: Iterable[str],
|
||||
*,
|
||||
never_move: bool = False,
|
||||
min_untracked: int = DEFAULT_MIN_ORPHANS,
|
||||
max_fraction: float = DEFAULT_MAX_ORPHAN_FRACTION,
|
||||
) -> dict:
|
||||
"""Plan the untracked → Staging move for a standalone deep scan. Pure — no I/O.
|
||||
|
||||
Returns a dict:
|
||||
* ``untracked`` (set[str]) — Transfer files with no DB record.
|
||||
* ``move_blocked`` (bool) — True when the untracked files must NOT be relocated.
|
||||
* ``block_reason`` (str) — ``BLOCK_TRANSFER_PERMANENT`` / ``BLOCK_DESYNC`` / "".
|
||||
|
||||
The move is blocked when either:
|
||||
* ``never_move`` is set (the user marked Transfer as their permanent library), or
|
||||
* the untracked share is implausibly large (> ``min_untracked`` files AND
|
||||
> ``max_fraction`` of the folder) — the empty/desynced-DB signature, where a
|
||||
path-only diff would relocate the whole library. Below that floor a normal
|
||||
batch of new arrivals still moves as before.
|
||||
|
||||
``move_blocked`` is only ever True when there ARE untracked files; an empty scan
|
||||
or a clean library returns ``move_blocked=False`` with no reason.
|
||||
"""
|
||||
transfer_set = set(transfer_files) # concrete (handles generators) + dedups
|
||||
untracked = diff_untracked(transfer_set, db_paths)
|
||||
total = len(transfer_set)
|
||||
n_untracked = len(untracked)
|
||||
|
||||
if n_untracked == 0:
|
||||
return {"untracked": untracked, "move_blocked": False, "block_reason": BLOCK_NONE}
|
||||
|
||||
if never_move:
|
||||
return {"untracked": untracked, "move_blocked": True, "block_reason": BLOCK_TRANSFER_PERMANENT}
|
||||
|
||||
if is_implausible_orphan_flood(
|
||||
n_untracked, total, min_orphans=min_untracked, max_fraction=max_fraction
|
||||
):
|
||||
return {"untracked": untracked, "move_blocked": True, "block_reason": BLOCK_DESYNC}
|
||||
|
||||
return {"untracked": untracked, "move_blocked": False, "block_reason": BLOCK_NONE}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"diff_untracked",
|
||||
"plan_standalone_deep_scan",
|
||||
"BLOCK_NONE",
|
||||
"BLOCK_TRANSFER_PERMANENT",
|
||||
"BLOCK_DESYNC",
|
||||
]
|
||||
|
|
@ -26,14 +26,16 @@ without a source ID are reported back to the caller and skipped
|
|||
entirely.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, List, Optional, Set
|
||||
from typing import Any, Callable, Dict, List, Optional, Set, Tuple
|
||||
|
||||
# Per-album track concurrency. Matches the download workers' per-batch
|
||||
# concurrency (3) so reorganize feels comparable to a fresh download.
|
||||
|
|
@ -99,6 +101,7 @@ _ALBUM_ID_COLUMNS = {
|
|||
'deezer': 'deezer_id',
|
||||
'discogs': 'discogs_id',
|
||||
'hydrabase': 'soul_id',
|
||||
'musicbrainz': 'musicbrainz_release_id',
|
||||
}
|
||||
|
||||
# Human-facing label for each source.
|
||||
|
|
@ -407,6 +410,18 @@ def _differentiators_in(norm_title: str) -> frozenset:
|
|||
return frozenset(t for t in norm_title.split() if t in _VERSION_DIFFERENTIATORS)
|
||||
|
||||
|
||||
# Featured-artist credit: "(feat. X)" / "[ft X]" / a trailing "feat. X". The
|
||||
# parenthesised form is stripped wherever it appears; the bare form only when
|
||||
# something follows it (so a song literally named "The Feat" is left alone, and
|
||||
# "Defeat"/"Lift" never trip the word-boundary). Case-insensitive.
|
||||
_FEAT_RE = re.compile(
|
||||
r"""\s*[\(\[]\s*(?:feat|ft|featuring)\b\.?[^)\]]*[\)\]] # (feat. X) / [ft. X]
|
||||
| \s+(?:feat|ft|featuring)\b\.?\s+\S.*$ # trailing feat. X ...
|
||||
""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_title(value) -> str:
|
||||
"""Lowercase + strip cosmetic punctuation and treat brackets / dashes
|
||||
/ slashes as word separators so the same track named slightly
|
||||
|
|
@ -418,10 +433,17 @@ def _normalize_title(value) -> str:
|
|||
- ``Don't Stop Believin'`` ↔ ``Don’t Stop Believin’``
|
||||
- ``Swimming Pools (Drank) - Extended Version``
|
||||
↔ ``Swimming Pools (Drank) (Extended Version)``
|
||||
- ``The Chase (feat. Big Artist)`` ↔ ``The Chase`` (#914)
|
||||
"""
|
||||
if value is None:
|
||||
return ''
|
||||
out = str(value).strip().lower()
|
||||
out = str(value).strip()
|
||||
# #914: drop featured-artist credits FIRST (while the parens are still here to
|
||||
# bound the group). iTunes appends "(feat. X)" to track titles while a user's
|
||||
# file is often just "The Chase" — the credit is metadata, not the song's
|
||||
# identity, and leaving it in dropped the match ratio below the threshold so
|
||||
# correctly-identified tracks reported as "not in the tracklist".
|
||||
out = _FEAT_RE.sub('', out).lower()
|
||||
# Strip characters that don't carry meaning across providers.
|
||||
for ch in ('"', "'", '‘', '’', '“', '”', '.', ',', '!', '?',
|
||||
'(', ')', '[', ']', '{', '}'):
|
||||
|
|
@ -1080,6 +1102,12 @@ def preview_album_reorganize(
|
|||
'track_number': track.get('track_number', 0),
|
||||
'current_path': _trim_to_transfer(db_path, resolved, transfer_dir),
|
||||
'new_path': '',
|
||||
# Absolute on-disk paths (additive). `current_path`/`new_path` above are
|
||||
# display-trimmed; these carry the real paths so the rename-only executor
|
||||
# acts on EXACTLY what the preview computed — no separate path logic that
|
||||
# could drift from what the user saw (#875).
|
||||
'current_path_abs': resolved or '',
|
||||
'new_path_abs': '',
|
||||
'file_exists': resolved is not None,
|
||||
'unchanged': False,
|
||||
'collision': False,
|
||||
|
|
@ -1127,6 +1155,7 @@ def preview_album_reorganize(
|
|||
new_full, _ok = build_final_path_fn(
|
||||
context, spotify_artist, album_info, file_ext, create_dirs=False
|
||||
)
|
||||
item['new_path_abs'] = new_full or ''
|
||||
item['new_path'] = (
|
||||
os.path.relpath(new_full, transfer_dir)
|
||||
if transfer_dir and new_full and new_full.startswith(transfer_dir)
|
||||
|
|
@ -1786,6 +1815,150 @@ def reorganize_album(
|
|||
return summary
|
||||
|
||||
|
||||
def _rename_track_in_place(current_abs: str, new_abs: str) -> Tuple[bool, Optional[str]]:
|
||||
"""Move ONE file from ``current_abs`` to ``new_abs`` in place — no copy, no re-tag,
|
||||
no post-processing. Creates the destination folder, carries sibling-format files
|
||||
(e.g. a lossy ``.opus`` alongside the ``.flac``) along with the renamed stem, and
|
||||
falls back to a cross-device move when the rename crosses a filesystem boundary.
|
||||
|
||||
Refuses to overwrite a DIFFERENT existing file at the destination (returns an error
|
||||
instead) — never silent data loss. Returns ``(ok, error_message)``.
|
||||
"""
|
||||
try:
|
||||
if current_abs and not os.path.exists(current_abs):
|
||||
return False, 'source file no longer on disk'
|
||||
same = os.path.normpath(current_abs) == os.path.normpath(new_abs)
|
||||
if os.path.exists(new_abs) and not same:
|
||||
return False, 'destination already exists'
|
||||
os.makedirs(os.path.dirname(new_abs), exist_ok=True)
|
||||
# Carry sibling-format audio to the same destination with the renamed stem —
|
||||
# mirrors _finalize_track so lossy-copy pairs don't get orphaned.
|
||||
for sibling_src in _find_sibling_audio_files(current_abs):
|
||||
_move_sibling_to_destination(sibling_src, new_abs)
|
||||
try:
|
||||
os.rename(current_abs, new_abs)
|
||||
except OSError as e:
|
||||
if getattr(e, 'errno', None) == errno.EXDEV:
|
||||
shutil.move(current_abs, new_abs) # crosses a filesystem boundary
|
||||
else:
|
||||
raise
|
||||
return True, None
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def reorganize_album_rename_only(
|
||||
*,
|
||||
album_id: str,
|
||||
db,
|
||||
transfer_dir: str,
|
||||
resolve_file_path_fn: Callable[[Optional[str]], Optional[str]],
|
||||
build_final_path_fn: Callable,
|
||||
update_track_path_fn: Optional[Callable[[object, str], None]] = None,
|
||||
cleanup_empty_dir_fn: Optional[Callable[[str], None]] = None,
|
||||
on_progress: Optional[Callable[[dict], None]] = None,
|
||||
primary_source: Optional[str] = None,
|
||||
strict_source: bool = False,
|
||||
metadata_source: str = 'api',
|
||||
stop_check: Optional[Callable[[], bool]] = None,
|
||||
preview_fn: Optional[Callable] = None,
|
||||
) -> dict:
|
||||
"""RENAME-ONLY reorganize (#875): move each track's file to the path the current
|
||||
naming scheme dictates, and nothing else — no copy-to-staging, no re-tag, no
|
||||
quality/AcoustID checks.
|
||||
|
||||
It acts on EXACTLY what :func:`preview_album_reorganize` computed (injected via
|
||||
``preview_fn`` for testability), so the apply can never disagree with what the user
|
||||
saw, and ONLY files whose path actually changes are touched — files marked
|
||||
``unchanged`` are skipped, which is what keeps a rename from rewriting the whole
|
||||
album (the #875 complaint). Tags and audio are left byte-for-byte alone.
|
||||
|
||||
Returns the same summary shape as :func:`reorganize_album`.
|
||||
"""
|
||||
preview_fn = preview_fn or preview_album_reorganize
|
||||
summary = {
|
||||
'status': 'completed', 'source': None, 'total': 0,
|
||||
'moved': 0, 'skipped': 0, 'failed': 0, 'errors': [],
|
||||
}
|
||||
|
||||
def _emit(**updates):
|
||||
if on_progress is None:
|
||||
return
|
||||
try:
|
||||
on_progress(updates)
|
||||
except Exception as e:
|
||||
logger.debug("[Reorganize/rename] progress emit failed: %s", e)
|
||||
|
||||
preview = preview_fn(
|
||||
album_id=album_id, db=db, transfer_dir=transfer_dir,
|
||||
resolve_file_path_fn=resolve_file_path_fn,
|
||||
build_final_path_fn=build_final_path_fn,
|
||||
primary_source=primary_source, strict_source=strict_source,
|
||||
metadata_source=metadata_source,
|
||||
)
|
||||
summary['source'] = preview.get('source')
|
||||
if not preview.get('success'):
|
||||
summary['status'] = preview.get('status', 'error')
|
||||
return summary
|
||||
|
||||
tracks = preview.get('tracks', [])
|
||||
summary['total'] = len(tracks)
|
||||
src_dirs_touched: Set[str] = set()
|
||||
|
||||
for t in tracks:
|
||||
if stop_check and stop_check():
|
||||
break
|
||||
title = t.get('title', 'Unknown')
|
||||
_emit(current_track=title)
|
||||
|
||||
# Skip anything that isn't a real, changing move. `unchanged` is the key one —
|
||||
# it's why a rename no longer rewrites files whose name didn't change.
|
||||
if (not t.get('matched') or t.get('unchanged')
|
||||
or t.get('collision') or not t.get('new_path_abs')):
|
||||
summary['skipped'] += 1
|
||||
_emit(skipped=summary['skipped'])
|
||||
continue
|
||||
|
||||
current_abs = t.get('current_path_abs')
|
||||
new_abs = t.get('new_path_abs')
|
||||
ok, err = _rename_track_in_place(current_abs, new_abs)
|
||||
if not ok:
|
||||
summary['failed'] += 1
|
||||
summary['errors'].append({
|
||||
'track_id': t.get('track_id'), 'title': title,
|
||||
'error': err or 'rename failed',
|
||||
})
|
||||
_emit(failed=summary['failed'], errors=list(summary['errors']))
|
||||
continue
|
||||
|
||||
# File is at its new home — update the DB directly (authoritative; no need to
|
||||
# round-trip through a server scan to learn what we just did). On DB failure the
|
||||
# file still moved; a library scan reconciles it, so we don't fail the track.
|
||||
if update_track_path_fn:
|
||||
try:
|
||||
update_track_path_fn(t.get('track_id'), new_abs)
|
||||
except Exception as db_err:
|
||||
logger.warning(
|
||||
"[Reorganize/rename] DB path update failed for %s: %s "
|
||||
"(file moved to %s; a scan will reconcile)",
|
||||
t.get('track_id'), db_err, new_abs,
|
||||
)
|
||||
if current_abs:
|
||||
src_dirs_touched.add(os.path.dirname(current_abs))
|
||||
summary['moved'] += 1
|
||||
_emit(moved=summary['moved'],
|
||||
processed=summary['moved'] + summary['skipped'] + summary['failed'])
|
||||
|
||||
if cleanup_empty_dir_fn:
|
||||
for src_dir in src_dirs_touched:
|
||||
try:
|
||||
cleanup_empty_dir_fn(src_dir)
|
||||
except Exception as e:
|
||||
logger.debug("[Reorganize/rename] cleanup of %s failed: %s", src_dir, e)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def _find_artist_dir(dest_path: str, transfer_dir: str) -> Optional[str]:
|
||||
"""Walk up from ``dest_path`` until the parent equals ``transfer_dir``;
|
||||
the directory at that point is the artist folder. Returns None if
|
||||
|
|
|
|||
|
|
@ -47,7 +47,10 @@ class LidarrDownloadClient(DownloadSourcePlugin):
|
|||
if download_path is None:
|
||||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
|
||||
self.active_downloads: Dict[str, Dict[str, Any]] = {}
|
||||
self._download_lock = threading.Lock()
|
||||
|
|
|
|||
|
|
@ -158,6 +158,161 @@ class ListenBrainzClient:
|
|||
|
||||
return True
|
||||
|
||||
_MAX_TRACKS_PER_ADD = 100 # ListenBrainz MAX_RECORDINGS_PER_ADD
|
||||
_PLAYLIST_EXT = "https://musicbrainz.org/doc/jspf#playlist"
|
||||
|
||||
def _lb_headers(self) -> Dict:
|
||||
return {"Authorization": f"Token {self.token}", "Content-Type": "application/json"}
|
||||
|
||||
def _add_tracks_in_batches(self, playlist_mbid: str, tracks: List[Dict]) -> int:
|
||||
"""Add JSPF tracks to a playlist in <=100-track batches; return how many were added."""
|
||||
added = 0
|
||||
headers = self._lb_headers()
|
||||
for i in range(0, len(tracks or []), self._MAX_TRACKS_PER_ADD):
|
||||
batch = tracks[i:i + self._MAX_TRACKS_PER_ADD]
|
||||
try:
|
||||
r = self._make_request_with_retry(
|
||||
"POST", f"{self.base_url}/playlist/{playlist_mbid}/item/add",
|
||||
json={"playlist": {"track": batch}}, headers=headers,
|
||||
)
|
||||
if r and r.status_code in (200, 201):
|
||||
added += len(batch)
|
||||
else:
|
||||
logger.warning(f"ListenBrainz item/add batch failed: "
|
||||
f"{r.status_code if r else 'no response'}")
|
||||
except Exception as e:
|
||||
logger.error(f"ListenBrainz item/add error: {e}")
|
||||
return added
|
||||
|
||||
def get_playlist_track_count(self, playlist_mbid: str):
|
||||
"""Current track count of an LB playlist, or None if it can't be fetched (gone/404)."""
|
||||
try:
|
||||
r = self._make_request_with_retry(
|
||||
"GET", f"{self.base_url}/playlist/{playlist_mbid}",
|
||||
params={"fetch_metadata": "false"},
|
||||
headers={"Authorization": f"Token {self.token}"},
|
||||
)
|
||||
if r and r.status_code == 200:
|
||||
return len(((r.json() or {}).get("playlist") or {}).get("track", []))
|
||||
except Exception as e:
|
||||
logger.debug(f"ListenBrainz get playlist count failed: {e}")
|
||||
return None
|
||||
|
||||
def delete_playlist(self, playlist_mbid: str) -> bool:
|
||||
"""Delete an LB playlist. True on success."""
|
||||
try:
|
||||
r = self._make_request_with_retry(
|
||||
"POST", f"{self.base_url}/playlist/{playlist_mbid}/delete", headers=self._lb_headers()
|
||||
)
|
||||
return bool(r and r.status_code in (200, 201))
|
||||
except Exception as e:
|
||||
logger.error(f"ListenBrainz delete playlist error: {e}")
|
||||
return False
|
||||
|
||||
def create_playlist(self, title: str, tracks: List[Dict], public: bool = False) -> Dict:
|
||||
"""Create a NEW playlist on ListenBrainz and add its tracks (#903).
|
||||
|
||||
``tracks`` are JSPF track dicts — each MUST carry an ``identifier`` of the form
|
||||
``https://musicbrainz.org/recording/<mbid>`` (LB rejects text-only tracks). Creates
|
||||
an empty playlist for the MBID, then adds tracks in <=100 batches. Returns
|
||||
``{success, playlist_mbid, playlist_url, added, requested, error, updated}``. Never raises.
|
||||
"""
|
||||
result = {"success": False, "playlist_mbid": None, "playlist_url": None,
|
||||
"added": 0, "requested": len(tracks or []), "error": None, "updated": False}
|
||||
if not self.is_authenticated():
|
||||
result["error"] = "ListenBrainz not authenticated (no token/username)"
|
||||
return result
|
||||
|
||||
create_body = {"playlist": {
|
||||
"title": (title or "SoulSync Export").strip() or "SoulSync Export",
|
||||
"extension": {self._PLAYLIST_EXT: {"public": bool(public)}},
|
||||
}}
|
||||
try:
|
||||
resp = self._make_request_with_retry(
|
||||
"POST", f"{self.base_url}/playlist/create", json=create_body, headers=self._lb_headers()
|
||||
)
|
||||
except Exception as e:
|
||||
result["error"] = f"create request failed: {e}"
|
||||
return result
|
||||
if not resp or resp.status_code not in (200, 201):
|
||||
result["error"] = f"create returned {resp.status_code if resp else 'no response'}"
|
||||
return result
|
||||
try:
|
||||
playlist_mbid = (resp.json() or {}).get("playlist_mbid")
|
||||
except Exception:
|
||||
playlist_mbid = None
|
||||
if not playlist_mbid:
|
||||
result["error"] = "create succeeded but no playlist_mbid in response"
|
||||
return result
|
||||
|
||||
result["playlist_mbid"] = playlist_mbid
|
||||
result["playlist_url"] = f"https://listenbrainz.org/playlist/{playlist_mbid}"
|
||||
result["added"] = self._add_tracks_in_batches(playlist_mbid, tracks)
|
||||
result["success"] = True
|
||||
return result
|
||||
|
||||
def update_playlist(self, playlist_mbid: str, title: str, tracks: List[Dict], public: bool = False) -> Dict:
|
||||
"""Replace an existing LB playlist's contents IN PLACE (stable URL/MBID) (#903).
|
||||
|
||||
Verifies the playlist still exists, clears its current items, re-adds the new tracks,
|
||||
and updates the title. If the playlist is gone (deleted on LB), returns success=False
|
||||
with ``gone=True`` so the caller can fall back to creating a fresh one.
|
||||
Returns the same shape as ``create_playlist`` plus ``updated=True``.
|
||||
"""
|
||||
result = {"success": False, "playlist_mbid": playlist_mbid,
|
||||
"playlist_url": f"https://listenbrainz.org/playlist/{playlist_mbid}",
|
||||
"added": 0, "requested": len(tracks or []), "error": None,
|
||||
"updated": True, "gone": False}
|
||||
if not self.is_authenticated():
|
||||
result["error"] = "ListenBrainz not authenticated (no token/username)"
|
||||
return result
|
||||
|
||||
count = self.get_playlist_track_count(playlist_mbid)
|
||||
if count is None:
|
||||
result["error"] = "playlist not found on ListenBrainz"
|
||||
result["gone"] = True
|
||||
return result
|
||||
|
||||
headers = self._lb_headers()
|
||||
# Clear existing items (one range delete from the top).
|
||||
if count > 0:
|
||||
try:
|
||||
self._make_request_with_retry(
|
||||
"POST", f"{self.base_url}/playlist/{playlist_mbid}/item/delete",
|
||||
json={"index": 0, "count": count}, headers=headers,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"ListenBrainz item/delete (clear) failed: {e}")
|
||||
|
||||
result["added"] = self._add_tracks_in_batches(playlist_mbid, tracks)
|
||||
|
||||
# Refresh the title (best-effort — content already replaced).
|
||||
try:
|
||||
self._make_request_with_retry(
|
||||
"POST", f"{self.base_url}/playlist/edit/{playlist_mbid}",
|
||||
json={"playlist": {"title": (title or "SoulSync Export").strip() or "SoulSync Export"}},
|
||||
headers=headers,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"ListenBrainz playlist title edit failed: {e}")
|
||||
|
||||
result["success"] = True
|
||||
return result
|
||||
|
||||
def create_or_update_playlist(self, title: str, tracks: List[Dict],
|
||||
existing_mbid: str = None, public: bool = False) -> Dict:
|
||||
"""Update the existing LB playlist in place when we've pushed this one before, else
|
||||
create a fresh one — so re-exporting the same SoulSync playlist never duplicates it.
|
||||
Falls back to create if the remembered playlist was deleted on LB."""
|
||||
if existing_mbid:
|
||||
res = self.update_playlist(existing_mbid, title, tracks, public)
|
||||
if res.get("success"):
|
||||
return res
|
||||
# Remembered playlist gone/failed -> create a new one instead of erroring out.
|
||||
logger.info(f"ListenBrainz playlist {existing_mbid} unavailable for update "
|
||||
f"({res.get('error')}); creating a new one.")
|
||||
return self.create_playlist(title, tracks, public)
|
||||
|
||||
def get_playlists_created_for_user(self, count: int = 25, offset: int = 0) -> List[Dict]:
|
||||
"""
|
||||
Fetch playlists created FOR the user (recommendations, personalized playlists)
|
||||
|
|
|
|||
|
|
@ -291,6 +291,13 @@ class ListeningStatsWorker:
|
|||
if not (top_artists or top_albums or top_tracks):
|
||||
return
|
||||
|
||||
# Normalize image URLs HERE, at cache-build time, not on every /api/stats/cached
|
||||
# read. normalize_image_url registers each URL in the image cache (a SQLite write
|
||||
# under a lock) — doing that per-request made the "instant" stats endpoint take ~20s
|
||||
# on HDD-backed installs (#935). Done once per background rebuild it's off the hot path,
|
||||
# and the read just returns the already-browser-safe URLs.
|
||||
from core.metadata import normalize_image_url as _fix_image
|
||||
|
||||
conn = None
|
||||
try:
|
||||
conn = self.db._get_connection()
|
||||
|
|
@ -324,7 +331,7 @@ class ListeningStatsWorker:
|
|||
key = (artist.get('name') or '').lower()
|
||||
r = artist_rows.get(key)
|
||||
if r:
|
||||
artist['image_url'] = r[1] or None
|
||||
artist['image_url'] = _fix_image(r[1]) or None
|
||||
artist['id'] = r[2]
|
||||
artist['global_listeners'] = r[3]
|
||||
artist['global_playcount'] = r[4]
|
||||
|
|
@ -356,7 +363,7 @@ class ListeningStatsWorker:
|
|||
key = (album.get('name') or '').lower()
|
||||
r = album_rows.get(key)
|
||||
if r:
|
||||
album['image_url'] = r[1] or None
|
||||
album['image_url'] = _fix_image(r[1]) or None
|
||||
album['id'] = r[2]
|
||||
album['artist_id'] = r[3]
|
||||
|
||||
|
|
@ -395,7 +402,7 @@ class ListeningStatsWorker:
|
|||
(track.get('artist') or '').lower())
|
||||
r = track_rows.get(key)
|
||||
if r:
|
||||
track['image_url'] = r[2] or None
|
||||
track['image_url'] = _fix_image(r[2]) or None
|
||||
track['id'] = r[3]
|
||||
track['artist_id'] = r[4]
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -438,10 +438,19 @@ def embed_album_art_metadata(audio_file, metadata: dict):
|
|||
|
||||
if not image_data:
|
||||
art_url = metadata.get("album_art_url")
|
||||
if not art_url:
|
||||
logger.warning("No album art URL available for embedding.")
|
||||
# Prefer the pinned release's OWN cover over a release-group / provider
|
||||
# representative (usually the standard edition); fall back to art_url
|
||||
# when the release has no art of its own. Keeps embedded art in sync
|
||||
# with cover.jpg (same preference + fetch).
|
||||
from core.metadata.caa_art import fetch_release_preferred_art
|
||||
image_data, mime_type, used_url = fetch_release_preferred_art(
|
||||
release_mbid, art_url, fetch_fn=_fetch_art_bytes)
|
||||
if not image_data:
|
||||
if not art_url and not release_mbid:
|
||||
logger.warning("No album art URL available for embedding.")
|
||||
return False
|
||||
image_data, mime_type = _fetch_art_bytes(art_url)
|
||||
if release_mbid and used_url and used_url != art_url:
|
||||
logger.info("Embedding release-specific art (edition match): %s", release_mbid)
|
||||
|
||||
if not image_data:
|
||||
logger.error("Failed to download album art data.")
|
||||
|
|
@ -560,13 +569,20 @@ def download_cover_art(album_info: dict, target_dir: str, context: dict = None,
|
|||
art_url = images[0].get("url", "")
|
||||
if art_url:
|
||||
logger.info("Using cover art URL from album context")
|
||||
if not art_url:
|
||||
logger.warning("No cover art URL available for download.")
|
||||
# Prefer the pinned release's OWN cover over a release-group / provider
|
||||
# representative (which is usually the standard edition), falling back
|
||||
# to art_url when the release has no art of its own. Upgrades to the
|
||||
# source's highest resolution via _fetch_art_bytes (shared with the
|
||||
# tag-embed path so cover.jpg and embedded art match).
|
||||
from core.metadata.caa_art import fetch_release_preferred_art
|
||||
image_data, _, used_url = fetch_release_preferred_art(
|
||||
release_mbid, art_url, fetch_fn=_fetch_art_bytes)
|
||||
if not image_data:
|
||||
if not art_url and not release_mbid:
|
||||
logger.warning("No cover art URL available for download.")
|
||||
return
|
||||
# Upgrade to the source's highest resolution (Spotify master /
|
||||
# iTunes 3000 / Deezer 1900) with a one-level fallback — shared
|
||||
# with the tag-embed path so cover.jpg and embedded art match.
|
||||
image_data, _ = _fetch_art_bytes(art_url)
|
||||
if release_mbid and used_url and used_url != art_url:
|
||||
logger.info("Using release-specific cover.jpg (edition match): %s", release_mbid)
|
||||
|
||||
if not image_data:
|
||||
return
|
||||
|
|
|
|||
74
core/metadata/caa_art.py
Normal file
74
core/metadata/caa_art.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""Cover Art Archive helper: prefer a pinned release's OWN cover over the
|
||||
release-group representative.
|
||||
|
||||
On the Cover Art Archive a release-group ``front`` is a single REPRESENTATIVE
|
||||
cover — CAA designates one release in the group to stand for the whole thing,
|
||||
which is almost always the standard / most-common edition. So when a download
|
||||
has pinned a SPECIFIC release (e.g. a "Gustave Edition" the user picked), using
|
||||
the release-group cover silently swaps in the standard art.
|
||||
|
||||
This helper tries the specific release's own ``/release/<mbid>/front`` first and
|
||||
only falls back to the caller's existing URL (a release-group representative or a
|
||||
provider cover) when the release has no art of its own — so it can only ever
|
||||
*improve* on today's behaviour, never strip a cover that was already showing.
|
||||
|
||||
Pure: the network fetch is injected, so the preference logic is unit-testable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
COVER_ART_ARCHIVE_URL = "https://coverartarchive.org"
|
||||
|
||||
|
||||
def caa_front_url(mbid: Optional[str], scope: str = "release", size: int = 1200) -> Optional[str]:
|
||||
"""Build a Cover Art Archive front-cover URL, or None for a falsy mbid.
|
||||
``scope`` is 'release' (a specific edition) or 'release-group' (the group's
|
||||
representative). ``size`` selects the CDN thumbnail (e.g. 250/500/1200); 0
|
||||
requests the bare ``/front`` original."""
|
||||
if not mbid:
|
||||
return None
|
||||
if scope not in ("release", "release-group"):
|
||||
scope = "release"
|
||||
suffix = f"-{size}" if size else ""
|
||||
return f"{COVER_ART_ARCHIVE_URL}/{scope}/{mbid}/front{suffix}"
|
||||
|
||||
|
||||
def fetch_release_preferred_art(
|
||||
release_mbid: Optional[str],
|
||||
fallback_url: Optional[str],
|
||||
*,
|
||||
fetch_fn: Callable[[str], Tuple[Optional[bytes], Optional[str]]],
|
||||
size: int = 1200,
|
||||
min_bytes: int = 0,
|
||||
) -> Tuple[Optional[bytes], Optional[str], Optional[str]]:
|
||||
"""Fetch the best cover, preferring the specific release's own art.
|
||||
|
||||
Tries ``/release/<release_mbid>/front`` first; on any miss (no such art,
|
||||
404, or smaller than ``min_bytes``) falls back to ``fallback_url`` (a
|
||||
release-group / provider cover). ``fetch_fn(url) -> (bytes|None, mime|None)``;
|
||||
it is expected to return ``(None, None)`` on a 404, which is how a release
|
||||
with no art of its own advances to the fallback. ``min_bytes`` defaults to 0
|
||||
(accept any non-empty image) to preserve the fallback path's prior behaviour;
|
||||
callers can raise it to reject placeholder/error images. Returns
|
||||
``(bytes|None, mime|None, url_used|None)``. Never raises for a missing cover —
|
||||
a failed candidate just advances to the next, so coverage never regresses."""
|
||||
candidates = []
|
||||
release_url = caa_front_url(release_mbid, "release", size) if release_mbid else None
|
||||
if release_url:
|
||||
candidates.append(release_url)
|
||||
if fallback_url and fallback_url not in candidates:
|
||||
candidates.append(fallback_url)
|
||||
|
||||
for url in candidates:
|
||||
try:
|
||||
data, mime = fetch_fn(url)
|
||||
except Exception:
|
||||
data, mime = None, None
|
||||
if data and len(data) > min_bytes:
|
||||
return data, mime, url
|
||||
return None, None, None
|
||||
|
||||
|
||||
__all__ = ["COVER_ART_ARCHIVE_URL", "caa_front_url", "fetch_release_preferred_art"]
|
||||
|
|
@ -211,7 +211,7 @@ def pick_canonical_release(
|
|||
# core.library_reorganize._ALBUM_ID_COLUMNS — a test pins them in sync). A manual
|
||||
# match on any of these should pin/lock the canonical version (#758); a match on
|
||||
# a source the canonical tools don't read (e.g. lastfm) has no version to pin.
|
||||
CANONICAL_ALBUM_SOURCES = frozenset({'spotify', 'itunes', 'deezer', 'discogs', 'hydrabase'})
|
||||
CANONICAL_ALBUM_SOURCES = frozenset({'spotify', 'itunes', 'deezer', 'discogs', 'hydrabase', 'musicbrainz'})
|
||||
|
||||
|
||||
def should_pin_manual_canonical(entity_type: str, source: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -167,6 +167,7 @@ def track_already_owned(
|
|||
album_name: str,
|
||||
server_source: Optional[str],
|
||||
confidence_threshold: float = 0.7,
|
||||
candidate_tracks: Optional[List[Any]] = None,
|
||||
) -> bool:
|
||||
"""Return True if the track is already in the user's library.
|
||||
|
||||
|
|
@ -186,6 +187,15 @@ def track_already_owned(
|
|||
original deleted) matches just fine — track_name + artist + album
|
||||
don't change with format.
|
||||
|
||||
``candidate_tracks`` (when not None) is the artist's library tracks,
|
||||
pre-fetched ONCE by the caller, so the check scores in-memory instead
|
||||
of firing per-track fuzzy SQL scans against the whole library. Pass an
|
||||
empty list for an artist the user owns nothing of — it still routes
|
||||
through the fast in-memory path (scores against zero candidates →
|
||||
instant "not owned") rather than the slow per-track search. None
|
||||
preserves the original per-track-SQL behaviour for callers that don't
|
||||
pre-fetch.
|
||||
|
||||
Returns False on any exception so a transient DB hiccup doesn't
|
||||
silently nuke a discography fetch — a redundant wishlist add is
|
||||
much cheaper to recover from than a missed track.
|
||||
|
|
@ -198,6 +208,7 @@ def track_already_owned(
|
|||
confidence_threshold=confidence_threshold,
|
||||
server_source=server_source,
|
||||
album=album_name or None,
|
||||
candidate_tracks=candidate_tracks,
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -131,6 +131,14 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf
|
|||
track_num_str = format_track_number_tag(
|
||||
metadata.get('track_number'), metadata.get('total_tracks')
|
||||
)
|
||||
# Disc number is written UNCONDITIONALLY (floored to >=1), like the
|
||||
# track number above. The old code only wrote it when truthy, so a
|
||||
# track whose disc came back 0/None/'' (e.g. matched to a different
|
||||
# edition) lost its disc tag on the clear-then-rewrite and floated
|
||||
# ungrouped above the disc sections in Jellyfin/Plex (Sokhi).
|
||||
from core.imports.track_number import normalize_disc_number
|
||||
_disc_num = normalize_disc_number(metadata.get('disc_number'))
|
||||
disc_num_str = str(_disc_num)
|
||||
write_multi = cfg.get("metadata_enhancement.tags.write_multi_artist", False)
|
||||
artists_list = metadata.get("_artists_list", [])
|
||||
|
||||
|
|
@ -162,8 +170,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf
|
|||
if metadata.get("genre"):
|
||||
audio_file.tags.add(symbols.TCON(encoding=3, text=[metadata["genre"]]))
|
||||
audio_file.tags.add(symbols.TRCK(encoding=3, text=[track_num_str]))
|
||||
if metadata.get("disc_number"):
|
||||
audio_file.tags.add(symbols.TPOS(encoding=3, text=[str(metadata["disc_number"])]))
|
||||
audio_file.tags.add(symbols.TPOS(encoding=3, text=[disc_num_str]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
if metadata.get("title"):
|
||||
audio_file["title"] = [metadata["title"]]
|
||||
|
|
@ -180,8 +187,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf
|
|||
if metadata.get("genre"):
|
||||
audio_file["genre"] = [metadata["genre"]]
|
||||
audio_file["tracknumber"] = [track_num_str]
|
||||
if metadata.get("disc_number"):
|
||||
audio_file["discnumber"] = [str(metadata["disc_number"])]
|
||||
audio_file["discnumber"] = [disc_num_str]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
if metadata.get("title"):
|
||||
audio_file["\xa9nam"] = [metadata["title"]]
|
||||
|
|
@ -198,8 +204,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf
|
|||
audio_file["trkn"] = [format_track_number_tuple(
|
||||
metadata.get("track_number"), metadata.get("total_tracks")
|
||||
)]
|
||||
if metadata.get("disc_number"):
|
||||
audio_file["disk"] = [(metadata["disc_number"], 0)]
|
||||
audio_file["disk"] = [(_disc_num, 0)]
|
||||
|
||||
embed_source_ids(audio_file, metadata, context, runtime=runtime)
|
||||
|
||||
|
|
|
|||
|
|
@ -230,15 +230,20 @@ def get_spotify_client_for_profile(profile_id: Optional[int] = None):
|
|||
return client
|
||||
|
||||
try:
|
||||
from core.spotify_client import SpotifyClient
|
||||
from core.spotify_client import SpotifyClient, normalize_spotify_oauth_config, SPOTIFY_OAUTH_SCOPE
|
||||
from spotipy.oauth2 import SpotifyOAuth
|
||||
import spotipy
|
||||
|
||||
normalized_creds = normalize_spotify_oauth_config({
|
||||
"client_id": client_id,
|
||||
"client_secret": client_secret,
|
||||
"redirect_uri": redirect_uri,
|
||||
})
|
||||
auth_manager = SpotifyOAuth(
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
redirect_uri=redirect_uri,
|
||||
scope="user-library-read user-read-private playlist-read-private playlist-read-collaborative user-read-email user-follow-read",
|
||||
client_id=normalized_creds.get("client_id", client_id),
|
||||
client_secret=normalized_creds.get("client_secret", client_secret),
|
||||
redirect_uri=normalized_creds.get("redirect_uri", redirect_uri),
|
||||
scope=SPOTIFY_OAUTH_SCOPE,
|
||||
cache_path=cache_path,
|
||||
state=f"profile_{profile_id}",
|
||||
)
|
||||
|
|
@ -352,10 +357,26 @@ def get_hydrabase_client(allow_fallback: bool = True, require_enabled: bool = Tr
|
|||
return None
|
||||
|
||||
|
||||
def get_configured_primary_source() -> str:
|
||||
"""Return metadata.fallback_source as stored in config, without runtime downgrade.
|
||||
|
||||
Unlike get_primary_source(), this never probes Spotify auth. Use at boot and
|
||||
anywhere blocking network I/O must be avoided (gunicorn worker import, Docker
|
||||
cold start when Spotify is unreachable).
|
||||
"""
|
||||
_default = METADATA_SOURCE_PRIORITY[0]
|
||||
return _get_config_value("metadata.fallback_source", _default) or _default
|
||||
|
||||
|
||||
def get_primary_source(spotify_client_factory: Optional[MetadataClientFactory] = None) -> str:
|
||||
"""Return configured primary metadata source."""
|
||||
from core.boot_phase import is_boot_phase
|
||||
|
||||
if is_boot_phase():
|
||||
return get_configured_primary_source()
|
||||
|
||||
_default = METADATA_SOURCE_PRIORITY[0]
|
||||
source = _get_config_value("metadata.fallback_source", _default) or _default
|
||||
source = get_configured_primary_source()
|
||||
|
||||
if source == "spotify":
|
||||
try:
|
||||
|
|
@ -368,6 +389,24 @@ def get_primary_source(spotify_client_factory: Optional[MetadataClientFactory] =
|
|||
return source
|
||||
|
||||
|
||||
def get_primary_source_label() -> str:
|
||||
"""Configured primary source for UI that *names* "your primary source" (the
|
||||
import-search fallback banner, etc.).
|
||||
|
||||
Identical to ``get_primary_source()`` except it does NOT downgrade a no-auth
|
||||
Spotify Free user to the working fallback: Spotify Free (fallback_source=
|
||||
'spotify' + metadata.spotify_free) is reported as 'spotify', because that IS
|
||||
their configured source — even though free-text album search itself has no
|
||||
free-path implementation and legitimately falls back to another provider.
|
||||
``get_primary_source()`` keeps the downgrade so client routing always yields a
|
||||
usable client; labels want the user's actual intent, not the fallback."""
|
||||
_default = METADATA_SOURCE_PRIORITY[0]
|
||||
source = _get_config_value("metadata.fallback_source", _default) or _default
|
||||
if source == "spotify" and _get_config_value("metadata.spotify_free", False):
|
||||
return "spotify"
|
||||
return get_primary_source()
|
||||
|
||||
|
||||
def get_spotify_disconnect_source(configured_source: Optional[str] = None) -> str:
|
||||
"""Return the active metadata source after Spotify is disconnected."""
|
||||
_default = METADATA_SOURCE_PRIORITY[0]
|
||||
|
|
@ -424,7 +463,19 @@ def get_primary_source_status(
|
|||
musicbrainz_client_factory: Optional[MetadataClientFactory] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Return a generic status snapshot for the active primary metadata source."""
|
||||
from core.boot_phase import is_boot_phase
|
||||
|
||||
source = _get_config_value("metadata.fallback_source", "deezer") or "deezer"
|
||||
if is_boot_phase():
|
||||
display_source = source
|
||||
if source == "spotify" and _get_config_value("metadata.spotify_free", False):
|
||||
display_source = "spotify_free"
|
||||
return {
|
||||
"source": display_source,
|
||||
"connected": False,
|
||||
"response_time": 0,
|
||||
}
|
||||
|
||||
started = time.time()
|
||||
connected = False
|
||||
|
||||
|
|
@ -492,9 +543,13 @@ def get_client_for_source(
|
|||
musicbrainz_client_factory: Optional[MetadataClientFactory] = None,
|
||||
):
|
||||
"""Return exact client for a source, or None if unavailable."""
|
||||
from core.boot_phase import is_boot_phase
|
||||
|
||||
if source == "spotify":
|
||||
try:
|
||||
client = get_spotify_client(client_factory=spotify_client_factory)
|
||||
if is_boot_phase():
|
||||
return client if client and getattr(client, "sp", None) else None
|
||||
if client and client.is_spotify_authenticated():
|
||||
return client
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -52,6 +52,27 @@ from typing import List, Optional, Sequence
|
|||
from core.metadata.types import Track
|
||||
|
||||
|
||||
def build_combined_search_query(track: str = '', artist: str = '',
|
||||
legacy: str = '') -> str:
|
||||
"""Combine a track + artist into a PLAIN, source-agnostic search query.
|
||||
|
||||
Deliberately NOT field-scoped (``track:"X" artist:"Y"``). That Spotify/Lucene
|
||||
filter syntax leaks to non-Spotify sources when a search falls back: Deezer
|
||||
closed the connection on it outright (``RemoteDisconnected``), and even sources
|
||||
that parse it over-constrain and miss the canonical cut (the iTunes/Deezer search
|
||||
endpoints already dropped it for that reason). The matching ``/search_tracks``
|
||||
endpoints rerank by expected title/artist afterward, so precision is recovered
|
||||
without the brittle field syntax.
|
||||
|
||||
Falls back to ``legacy`` when neither track nor artist is given. Returns ``''``
|
||||
when there's nothing to search (caller decides how to handle empty).
|
||||
"""
|
||||
parts = [p.strip() for p in (track, artist) if p and p.strip()]
|
||||
if parts:
|
||||
return ' '.join(parts)
|
||||
return (legacy or '').strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pattern tables — public so tests can introspect, callers can extend
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -1089,10 +1089,12 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di
|
|||
metadata["track_number"] = 1
|
||||
metadata["total_tracks"] = 1
|
||||
|
||||
disc_num = original_search.get("disc_number")
|
||||
if disc_num is None and album_info:
|
||||
disc_num = album_info.get("disc_number")
|
||||
metadata["disc_number"] = disc_num if disc_num is not None else 1
|
||||
# Resolve via the SHARED resolver so the embedded tag and the "Disc N" folder
|
||||
# (computed in the import pipeline from album_info) can never disagree — same
|
||||
# function, same inputs. Floors to >=1 (a 0/''/non-numeric disc must not read
|
||||
# as disc-less and ungroup the track in Jellyfin/Plex).
|
||||
from core.imports.track_number import resolve_disc_for_track
|
||||
metadata["disc_number"] = resolve_disc_for_track(original_search, album_info)
|
||||
|
||||
if album_ctx and album_ctx.get("release_date"):
|
||||
release_date = _normalize_release_date_tag(album_ctx.get("release_date"))
|
||||
|
|
|
|||
|
|
@ -49,8 +49,10 @@ from core.metadata.registry import (
|
|||
get_discogs_client,
|
||||
get_hydrabase_client,
|
||||
get_itunes_client,
|
||||
get_configured_primary_source,
|
||||
get_primary_client,
|
||||
get_primary_source,
|
||||
get_primary_source_label,
|
||||
get_spotify_client_for_profile,
|
||||
get_registered_runtime_client,
|
||||
get_source_priority,
|
||||
|
|
@ -116,7 +118,9 @@ __all__ = [
|
|||
"get_metadata_service",
|
||||
"get_musicmap_similar_artists",
|
||||
"get_primary_client",
|
||||
"get_configured_primary_source",
|
||||
"get_primary_source",
|
||||
"get_primary_source_label",
|
||||
"get_spotify_client_for_profile",
|
||||
"get_registered_runtime_client",
|
||||
"get_spotify_client",
|
||||
|
|
|
|||
|
|
@ -110,6 +110,7 @@ class NavidromeTrack:
|
|||
self.title = navidrome_data.get('title', 'Unknown Track')
|
||||
self.duration = navidrome_data.get('duration', 0) * 1000 # Convert to milliseconds
|
||||
self.trackNumber = navidrome_data.get('track')
|
||||
self.discNumber = navidrome_data.get('discNumber') # multi-disc: disc number
|
||||
self.year = navidrome_data.get('year')
|
||||
self.userRating = navidrome_data.get('userRating')
|
||||
self.addedAt = self._parse_date(navidrome_data.get('created'))
|
||||
|
|
@ -1022,6 +1023,34 @@ class NavidromeClient(MediaServerClient):
|
|||
logger.error(f"Error {'updating' if playlist_id else 'creating'} Navidrome playlist '{name}': {e}")
|
||||
return False
|
||||
|
||||
def rewrite_playlist_order(self, playlist_id: str, name: str, ordered_song_ids) -> bool:
|
||||
"""Rewrite a playlist's tracks to an exact ordered id list (Subsonic has no
|
||||
per-track move — the only reorder primitive is overwriting the whole song
|
||||
list). Overwrites in place via createPlaylist + playlistId, so the playlist
|
||||
identity (id/name) survives. Used ONLY by the 'Align playlists' path with
|
||||
ids already present in the playlist — never adds a new track.
|
||||
|
||||
NOTE: like every createPlaylist+playlistId overwrite (same as replace-mode
|
||||
sync), Navidrome may not carry the playlist's comment forward — the caller
|
||||
re-applies it if needed."""
|
||||
if not self.ensure_connection():
|
||||
return False
|
||||
ids = [str(i) for i in (ordered_song_ids or []) if str(i)]
|
||||
if not ids:
|
||||
logger.warning(f"rewrite_playlist_order: no song ids for '{name}'")
|
||||
return False
|
||||
try:
|
||||
params = {'name': name, 'songId': ids, 'playlistId': playlist_id}
|
||||
response = self._make_request('createPlaylist', params)
|
||||
if response and response.get('status') == 'ok':
|
||||
logger.info(f"Aligned Navidrome playlist '{name}' order ({len(ids)} tracks)")
|
||||
return True
|
||||
logger.error(f"rewrite_playlist_order failed for '{name}'")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error rewriting Navidrome playlist order '{name}': {e}")
|
||||
return False
|
||||
|
||||
def copy_playlist(self, source_name: str, target_name: str) -> bool:
|
||||
"""Copy a playlist to create a backup"""
|
||||
if not self.ensure_connection():
|
||||
|
|
@ -1192,7 +1221,11 @@ class NavidromeClient(MediaServerClient):
|
|||
|
||||
primary = existing_playlists[0]
|
||||
existing_tracks = self.get_playlist_tracks(primary.id)
|
||||
current_ids = [str(t.id) for t in existing_tracks if getattr(t, 'id', None)]
|
||||
# #905: NavidromeTrack exposes the Subsonic song id as `ratingKey` (NOT `.id`,
|
||||
# which doesn't exist) — same as append_to_playlist reads it. Reading `t.id` here
|
||||
# made current_ids ALWAYS empty, so reconcile thought the playlist was empty and
|
||||
# re-added every track each sync (playlists doubling) while removing nothing.
|
||||
current_ids = [str(t.ratingKey) for t in existing_tracks if getattr(t, 'ratingKey', None)]
|
||||
desired_ids = []
|
||||
for t in tracks:
|
||||
tid = (str(t.ratingKey) if hasattr(t, 'ratingKey')
|
||||
|
|
|
|||
|
|
@ -25,3 +25,4 @@ from core.personalized.generators import daily_mix # noqa: F401
|
|||
from core.personalized.generators import fresh_tape # noqa: F401
|
||||
from core.personalized.generators import archives # noqa: F401
|
||||
from core.personalized.generators import seasonal_mix # noqa: F401
|
||||
from core.personalized.generators import listening_mix # noqa: F401
|
||||
|
|
|
|||
68
core/personalized/generators/listening_mix.py
Normal file
68
core/personalized/generators/listening_mix.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
"""Listening Mix (#913) generator.
|
||||
|
||||
Reads the full, render-ready track dicts the watchlist scan stored under the
|
||||
``listening_recs_tracks_full`` metadata key — built from the recommended artists'
|
||||
top tracks (see ``core.watchlist_scanner._build_listening_recommendations``) — and
|
||||
coerces them into ``Track`` records.
|
||||
|
||||
Unlike Fresh Tape / Archives this needs NO discovery-pool hydration: the stored
|
||||
dicts are already complete, so the snapshot can't shrink when the pool rotates. It
|
||||
also means the generator is a pure read — generation/network all happen during the
|
||||
scan; this just hands the stored tracks to the personalized manager so the mix can
|
||||
participate in the Sync-page mirror + Auto-Sync pipeline like every other kind."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, List
|
||||
|
||||
from core.personalized.specs import PlaylistKindSpec, get_registry
|
||||
from core.personalized.types import PlaylistConfig, Track
|
||||
|
||||
|
||||
KIND = 'listening_mix'
|
||||
METADATA_KEY = 'listening_recs_tracks_full'
|
||||
|
||||
|
||||
def generate(deps: Any, variant: str, config: PlaylistConfig) -> List[Track]:
|
||||
"""Return the stored Listening Mix tracks, trimmed to ``config.limit``.
|
||||
|
||||
Empty (not an error) when the scan hasn't produced a mix yet — the manager
|
||||
preserves any prior snapshot rather than dropping it. Tolerates a missing/garbage
|
||||
metadata blob the same way.
|
||||
"""
|
||||
db = getattr(deps, 'database', None) or (deps.get('database') if isinstance(deps, dict) else None)
|
||||
if db is None:
|
||||
raise RuntimeError("Listening Mix generator deps missing `database`")
|
||||
|
||||
raw = db.get_metadata(METADATA_KEY)
|
||||
if not raw:
|
||||
return []
|
||||
try:
|
||||
rows = json.loads(raw) or []
|
||||
except (ValueError, TypeError):
|
||||
return []
|
||||
|
||||
tracks: List[Track] = []
|
||||
for d in rows:
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
tracks.append(Track.from_dict(d))
|
||||
if len(tracks) >= config.limit:
|
||||
break
|
||||
return tracks
|
||||
|
||||
|
||||
SPEC = PlaylistKindSpec(
|
||||
kind=KIND,
|
||||
name_template='Your Listening Mix',
|
||||
description='Tracks from artists matched to what you actually listen to.',
|
||||
default_config=PlaylistConfig(limit=50, max_per_album=5, max_per_artist=3),
|
||||
generator=generate,
|
||||
requires_variant=False,
|
||||
tags=['curated', 'listening'],
|
||||
)
|
||||
|
||||
|
||||
if get_registry().get(KIND) is None:
|
||||
get_registry().register(SPEC)
|
||||
|
|
@ -111,7 +111,10 @@ class PersonalizedPlaylistsService:
|
|||
Either value can be None to skip that filter for that source.
|
||||
|
||||
- Spotify: 60 / 40 (the existing 0-100 popularity scale)
|
||||
- Deezer: 500000 / 100000 (rank values — ballpark from real data)
|
||||
- Deezer: 60 / 50 (ALSO 0-100 — the discovery pool SYNTHESIZES deezer popularity to a
|
||||
0-100 score at scan time, capped at 100; it is NOT Deezer's raw rank. The old
|
||||
500000/100000 rank thresholds matched nothing, so Popular Picks came back empty for
|
||||
deezer users while Hidden Gems' < 100000 caught the whole pool.)
|
||||
- iTunes / others: None / None (no popularity data; fall back to no
|
||||
threshold filter, just diversity-on-random)
|
||||
|
||||
|
|
@ -121,7 +124,7 @@ class PersonalizedPlaylistsService:
|
|||
if normalized == 'spotify':
|
||||
return 60, 40
|
||||
if normalized == 'deezer':
|
||||
return 500_000, 100_000
|
||||
return 60, 50
|
||||
# iTunes, hydrabase, anything else — no usable popularity data
|
||||
return None, None
|
||||
|
||||
|
|
|
|||
94
core/playlists/item_naming.py
Normal file
94
core/playlists/item_naming.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
"""Optional custom naming for the FILES inside an organize-by-playlist folder.
|
||||
|
||||
By default a playlist entry keeps the real library filename (the materialized
|
||||
folder is a view onto Artist/Album/track.ext). A user can opt into a flat
|
||||
filename template — e.g. ``$position - $artist - $title`` — so the folder sorts
|
||||
and plays the way they want (most commonly: in playlist order on a dumb DAP).
|
||||
|
||||
It is a **filename** template, never a path:
|
||||
- it may NOT contain a path separator (``/`` or ``\\``) — it names the file,
|
||||
not a folder tree, and
|
||||
- it MUST contain ``$title`` — so every file has a real, non-empty name.
|
||||
|
||||
Both rules are validated up front (so the Settings UI can reject a bad value
|
||||
with a reason) AND re-checked at apply time, where an invalid/empty template or
|
||||
an empty render falls back to the library filename. So a bad value can never
|
||||
produce a broken name — the worst case is "no change from today".
|
||||
|
||||
Pure logic: no DB, no config, no filesystem. The caller supplies the metadata.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from core.imports.paths import sanitize_filename
|
||||
|
||||
# Tokens a user may use in the template (for docs / UI hints).
|
||||
PLAYLIST_ITEM_TOKENS = ("$position", "$artist", "$album", "$track", "$title")
|
||||
|
||||
|
||||
def validate_playlist_item_template(template: Optional[str]) -> Tuple[bool, str]:
|
||||
"""Return ``(ok, reason)``. An empty template is VALID and means "feature off"
|
||||
(keep the library filename). ``reason`` is '' when ok."""
|
||||
t = (template or "").strip()
|
||||
if not t:
|
||||
return True, "" # empty == disabled, not an error
|
||||
if "/" in t or "\\" in t:
|
||||
return False, ("Playlist file naming can't contain a folder separator "
|
||||
"( / or \\ ) — it names the file, not a path.")
|
||||
if "$title" not in t:
|
||||
return False, "Playlist file naming must include $title so every file has a name."
|
||||
return True, ""
|
||||
|
||||
|
||||
def render_playlist_item_name(
|
||||
template: Optional[str],
|
||||
*,
|
||||
title: str,
|
||||
artist: str = "",
|
||||
album: str = "",
|
||||
track: object = None,
|
||||
position: object = None,
|
||||
ext: str = "",
|
||||
fallback_name: str = "",
|
||||
) -> str:
|
||||
"""Render ``template`` to a sanitized filename WITH ``ext`` appended.
|
||||
|
||||
Falls back to ``fallback_name`` (the library filename) when the template is
|
||||
empty/invalid or renders to nothing after sanitizing — so the result is
|
||||
never broken. ``position`` is used verbatim (the caller pre-pads it for
|
||||
correct sorting); ``track`` is zero-padded to two digits when numeric."""
|
||||
ok, _ = validate_playlist_item_template(template)
|
||||
t = (template or "").strip()
|
||||
if not ok or not t:
|
||||
return fallback_name
|
||||
|
||||
pos_str = "" if position is None else str(position)
|
||||
if track is None:
|
||||
trk_str = ""
|
||||
else:
|
||||
try:
|
||||
trk_str = f"{int(track):02d}"
|
||||
except (TypeError, ValueError):
|
||||
trk_str = str(track)
|
||||
|
||||
# No token is a prefix of another, so replacement order is irrelevant.
|
||||
out = t
|
||||
out = out.replace("$position", pos_str)
|
||||
out = out.replace("$artist", str(artist or ""))
|
||||
out = out.replace("$album", str(album or ""))
|
||||
out = out.replace("$track", trk_str)
|
||||
out = out.replace("$title", str(title or ""))
|
||||
|
||||
out = sanitize_filename(out).strip()
|
||||
if not out:
|
||||
return fallback_name
|
||||
return out + (ext or "")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PLAYLIST_ITEM_TOKENS",
|
||||
"validate_playlist_item_template",
|
||||
"render_playlist_item_name",
|
||||
]
|
||||
|
|
@ -72,16 +72,25 @@ def playlist_dir_for(playlists_root: str, playlist_name: str) -> str:
|
|||
return candidate
|
||||
|
||||
|
||||
def _desired_entries(playlist_dir: str, real_paths: Sequence[str]) -> "list[tuple[str, str]]":
|
||||
"""Map each real file to a flat destination inside ``playlist_dir``, preserving
|
||||
the source filename. On a basename collision between two *different* sources,
|
||||
disambiguate with a numeric suffix rather than silently overwriting."""
|
||||
def _desired_entries(
|
||||
playlist_dir: str,
|
||||
real_paths: Sequence[str],
|
||||
dest_names: Optional[Sequence[Optional[str]]] = None,
|
||||
) -> "list[tuple[str, str]]":
|
||||
"""Map each real file to a flat destination inside ``playlist_dir``.
|
||||
|
||||
By default the source filename is preserved. ``dest_names`` (parallel to
|
||||
``real_paths``) lets a caller override the name per entry — e.g. a custom
|
||||
playlist file-naming template; a falsy override falls back to the source
|
||||
basename. On a name collision between two *different* sources, disambiguate
|
||||
with a numeric suffix rather than silently overwriting."""
|
||||
entries: list[tuple[str, str]] = []
|
||||
used: dict[str, str] = {} # dest basename -> source real path
|
||||
for real in real_paths:
|
||||
for i, real in enumerate(real_paths):
|
||||
if not real:
|
||||
continue
|
||||
base = os.path.basename(real)
|
||||
override = dest_names[i] if (dest_names is not None and i < len(dest_names)) else None
|
||||
base = override or os.path.basename(real)
|
||||
name = base
|
||||
stem, ext = os.path.splitext(base)
|
||||
counter = 1
|
||||
|
|
@ -156,6 +165,7 @@ def rebuild_playlist_folder(
|
|||
real_paths: Sequence[str],
|
||||
mode: str = DEFAULT_MODE,
|
||||
*,
|
||||
dest_names: Optional[Sequence[Optional[str]]] = None,
|
||||
prune_stale: bool = True,
|
||||
symlink_fn: Callable[[str, str], None] = os.symlink,
|
||||
copy_fn: Callable[[str, str], object] = shutil.copy2,
|
||||
|
|
@ -163,13 +173,15 @@ def rebuild_playlist_folder(
|
|||
"""(Re)build ``playlists_root/<playlist_name>/`` so it contains exactly one
|
||||
entry per real file in ``real_paths`` — adding missing entries, leaving correct
|
||||
ones untouched, and (when ``prune_stale``) removing entries no longer present.
|
||||
``dest_names`` (parallel to ``real_paths``) optionally overrides each entry's
|
||||
filename (custom playlist naming); falsy entries keep the source basename.
|
||||
Idempotent and safe to re-run any time. Filesystem ops are injectable."""
|
||||
mode = normalize_mode(mode)
|
||||
pdir = playlist_dir_for(playlists_root, playlist_name)
|
||||
summary = RebuildSummary(playlist_dir=pdir, mode_requested=mode)
|
||||
os.makedirs(pdir, exist_ok=True)
|
||||
|
||||
entries = _desired_entries(pdir, real_paths)
|
||||
entries = _desired_entries(pdir, real_paths, dest_names)
|
||||
keep = {dest for _real, dest in entries}
|
||||
|
||||
for real, dest in entries:
|
||||
|
|
|
|||
|
|
@ -155,11 +155,20 @@ def _rebuild_one_from_db(db, config_manager, playlist: dict):
|
|||
source IDs), resolving to disk, and rebuilding WITH prune. Because it's driven
|
||||
by current membership, a track that has LEFT the playlist drops out of the set
|
||||
and its symlink is pruned. Returns ``(playlist_name, RebuildSummary)``."""
|
||||
import os as _os
|
||||
|
||||
from core.library.path_resolver import resolve_library_file_path
|
||||
from core.playlists.item_naming import render_playlist_item_name
|
||||
|
||||
root = docker_resolve_path(config_manager.get("playlists.materialize_path", "./Playlists"))
|
||||
mode = normalize_mode(config_manager.get("playlists.materialize_mode", "symlink"))
|
||||
real_paths: List[str] = []
|
||||
item_template = ((config_manager.get("file_organization.templates", {}) or {})
|
||||
.get("playlist_item", "") or "").strip()
|
||||
|
||||
# Resolve owned tracks to real paths IN PLAYLIST ORDER, keeping the metadata
|
||||
# so an optional custom filename template ($position/$artist/$title/...) can
|
||||
# be applied. $position is the playlist index, which is exactly this order.
|
||||
resolved: List[dict] = []
|
||||
seen = set()
|
||||
for t in (db.get_mirrored_playlist_tracks(playlist["id"]) or []):
|
||||
title = (t.get("track_name") or "").strip()
|
||||
|
|
@ -175,9 +184,32 @@ def _rebuild_one_from_db(db, config_manager, playlist: dict):
|
|||
real = resolve_library_file_path(getattr(db_track, "file_path", None), config_manager=config_manager)
|
||||
if real and real not in seen:
|
||||
seen.add(real)
|
||||
real_paths.append(real)
|
||||
resolved.append({
|
||||
"real": real,
|
||||
"title": title,
|
||||
"artist": artist,
|
||||
"album": (t.get("album_name") or t.get("album") or "").strip(),
|
||||
"track": getattr(db_track, "track_number", None),
|
||||
})
|
||||
|
||||
real_paths: List[str] = [r["real"] for r in resolved]
|
||||
|
||||
dest_names = None
|
||||
if item_template:
|
||||
width = max(2, len(str(len(resolved)))) # zero-pad $position for correct sorting
|
||||
dest_names = [
|
||||
render_playlist_item_name(
|
||||
item_template,
|
||||
title=r["title"], artist=r["artist"], album=r["album"], track=r["track"],
|
||||
position=f"{i:0{width}d}",
|
||||
ext=_os.path.splitext(r["real"])[1],
|
||||
fallback_name=_os.path.basename(r["real"]),
|
||||
)
|
||||
for i, r in enumerate(resolved, start=1)
|
||||
]
|
||||
|
||||
name = playlist.get("name") or "Unnamed Playlist"
|
||||
return name, rebuild_playlist_folder(root, name, real_paths, mode) # prune_stale=True
|
||||
return name, rebuild_playlist_folder(root, name, real_paths, mode, dest_names=dest_names) # prune_stale=True
|
||||
|
||||
|
||||
def rebuild_organized_playlists_from_db(db, config_manager, *, profile_id: int = 1):
|
||||
|
|
|
|||
|
|
@ -19,6 +19,35 @@ from urllib.parse import parse_qs, urlparse
|
|||
|
||||
_SPOTIFY_ID_RE = re.compile(r"^[A-Za-z0-9]{16,32}$")
|
||||
|
||||
|
||||
def stable_source_track_id(track: Mapping, existing: Optional[str] = None) -> str:
|
||||
"""A stable per-track id for a mirrored-playlist track.
|
||||
|
||||
Spotify / YouTube / Deezer tracks carry a native id. File-import (CSV / M3U /
|
||||
TXT) and iTunes-only sources don't — they arrive with an empty
|
||||
``source_track_id``. The whole manual-match system (Find & Add ↔ sync) keys on
|
||||
``source_track_id``, and an empty key can neither be recorded (the persist is a
|
||||
no-op) nor looked up — so a manual match on a file-import track is silently
|
||||
dropped and the track re-appears as "extra" (#901).
|
||||
|
||||
When a native id is present it's used verbatim. Otherwise we derive a
|
||||
DETERMINISTIC id from the track's identity (artist|title|album, normalized) so
|
||||
the SAME song gets the SAME id across re-imports and discovery passes — which
|
||||
is exactly what the match lookup needs. Prefixed ``file:`` so it's recognizable
|
||||
and never collides with a real upstream id. Returns '' only when there's no
|
||||
usable identity at all (no title)."""
|
||||
native = (existing if existing is not None else track.get("source_track_id")) or ""
|
||||
native = str(native).strip()
|
||||
if native:
|
||||
return native
|
||||
title = str(track.get("track_name") or track.get("name") or "").strip().lower()
|
||||
if not title:
|
||||
return ""
|
||||
artist = str(track.get("artist_name") or track.get("artist") or "").strip().lower()
|
||||
album = str(track.get("album_name") or track.get("album") or "").strip().lower()
|
||||
digest = hashlib.md5(f"{artist}|{title}|{album}".encode("utf-8")).hexdigest()[:16]
|
||||
return f"file:{digest}"
|
||||
|
||||
# Synthetic batch playlist_id prefixes that wrap a mirrored_playlists PK.
|
||||
# Download/discovery flows build a batch playlist_id as f"{prefix}{pk}" — e.g.
|
||||
# auto_mirror_<pk> (core/automation/handlers/sync_playlist.py), youtube_mirrored_<pk>
|
||||
|
|
|
|||
|
|
@ -714,6 +714,88 @@ class PlexClient(MediaServerClient):
|
|||
logger.error(f"Error reconciling Plex playlist '{playlist_name}': {e}")
|
||||
return False
|
||||
|
||||
def get_playlist_track_ids(self, playlist_id, playlist_name: str = "") -> List[str]:
|
||||
"""The playlist's current track ratingKeys, in current order. [] if missing."""
|
||||
if not self.ensure_connection():
|
||||
return []
|
||||
try:
|
||||
playlist = None
|
||||
try:
|
||||
playlist = self.server.fetchItem(int(playlist_id))
|
||||
except Exception:
|
||||
playlist = None
|
||||
if playlist is None and playlist_name:
|
||||
try:
|
||||
playlist = self.server.playlist(playlist_name)
|
||||
except Exception:
|
||||
playlist = None
|
||||
if playlist is None:
|
||||
return []
|
||||
return [str(i.ratingKey) for i in playlist.items() if hasattr(i, 'ratingKey')]
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting Plex playlist track ids '{playlist_name}': {e}")
|
||||
return []
|
||||
|
||||
def reorder_playlist(self, playlist_id, playlist_name: str, ordered_ids) -> bool:
|
||||
"""In-place reorder a playlist to an exact ordered ratingKey list ('Align
|
||||
playlists'). Moves items into sequence and removes any current item NOT in
|
||||
``ordered_ids`` (so 'Mirror source' drops extras; 'Keep extras' includes
|
||||
them in the list). Uses moveItem/removeItems so the playlist's poster,
|
||||
summary and ratingKey survive — never deletes/recreates. Order-only: every
|
||||
id in ``ordered_ids`` is already in the playlist (the caller validated)."""
|
||||
if not self.ensure_connection():
|
||||
return False
|
||||
try:
|
||||
playlist = None
|
||||
try:
|
||||
playlist = self.server.fetchItem(int(playlist_id))
|
||||
except Exception as e:
|
||||
logger.debug("Plex reorder fetchItem failed: %s", e)
|
||||
if playlist is None and playlist_name:
|
||||
try:
|
||||
playlist = self.server.playlist(playlist_name)
|
||||
except Exception as e:
|
||||
logger.debug("Plex reorder by-name failed: %s", e)
|
||||
if playlist is None:
|
||||
logger.error(f"Plex reorder: playlist not found (id={playlist_id}, name='{playlist_name}')")
|
||||
return False
|
||||
|
||||
ordered = [str(i) for i in (ordered_ids or []) if str(i)]
|
||||
ordered_set = set(ordered)
|
||||
items = [i for i in playlist.items() if hasattr(i, 'ratingKey')]
|
||||
by_rk = {str(i.ratingKey): i for i in items}
|
||||
|
||||
# Drop items not in the desired list (extras, for Mirror source).
|
||||
to_remove = [i for i in items if str(i.ratingKey) not in ordered_set]
|
||||
if to_remove:
|
||||
try:
|
||||
playlist.removeItems(to_remove)
|
||||
except (AttributeError, TypeError):
|
||||
for it in to_remove:
|
||||
try:
|
||||
playlist.removeItem(it)
|
||||
except Exception as one_err:
|
||||
logger.debug("Plex reorder: removeItem failed: %s", one_err)
|
||||
|
||||
# Move each desired item into place: first to the front (after=None),
|
||||
# then each subsequent one after its predecessor.
|
||||
prev = None
|
||||
for sid in ordered:
|
||||
it = by_rk.get(sid)
|
||||
if it is None:
|
||||
continue
|
||||
try:
|
||||
playlist.moveItem(it, after=prev)
|
||||
except Exception as mv_err:
|
||||
logger.warning(f"Plex reorder moveItem failed for {sid}: {mv_err}")
|
||||
return False
|
||||
prev = it
|
||||
logger.info(f"Aligned Plex playlist '{playlist_name}' order ({len(ordered)} tracks)")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error reordering Plex playlist '{playlist_name}': {e}")
|
||||
return False
|
||||
|
||||
def update_playlist(self, playlist_name: str, tracks: List[TrackInfo]) -> bool:
|
||||
if not self.ensure_connection():
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from config.settings import config_manager
|
|||
|
||||
# Import Soulseek data structures for drop-in replacement compatibility
|
||||
from core.download_plugins.types import TrackResult, AlbumResult, DownloadStatus
|
||||
from core.quality.source_map import quality_from_qobuz, quality_tier_for_source
|
||||
|
||||
logger = get_logger("qobuz_client")
|
||||
|
||||
|
|
@ -119,7 +120,10 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
|
||||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify download path {self.download_path}: {e}")
|
||||
|
||||
logger.info(f"Qobuz client using download path: {self.download_path}")
|
||||
|
||||
|
|
@ -160,6 +164,8 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
|
||||
def _restore_session(self):
|
||||
"""Try to restore saved session from config."""
|
||||
from core.boot_phase import is_boot_phase
|
||||
|
||||
saved = config_manager.get('qobuz.session', {})
|
||||
app_id = saved.get('app_id', '')
|
||||
app_secret = saved.get('app_secret', '')
|
||||
|
|
@ -174,6 +180,10 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
'X-User-Auth-Token': self.user_auth_token,
|
||||
})
|
||||
|
||||
if is_boot_phase():
|
||||
logger.info("Loaded Qobuz session from config (verification deferred until after boot)")
|
||||
return
|
||||
|
||||
# Verify the token is still valid
|
||||
try:
|
||||
resp = self.session.get(
|
||||
|
|
@ -711,6 +721,26 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
logger.error(f"Error getting Qobuz track {track_id}: {e}")
|
||||
return None
|
||||
|
||||
def get_track_result(self, track_id):
|
||||
"""Fetch ONE track by ID and convert it to a downloadable TrackResult.
|
||||
|
||||
Used when a pasted Qobuz link must inject the EXACT track: a text search
|
||||
for an obscure track's name often doesn't surface it at all, so bubbling
|
||||
the matching id is a no-op (#932). Returns None on any failure so the
|
||||
caller falls back to the text-search path."""
|
||||
try:
|
||||
track = self.get_track(track_id)
|
||||
if not track:
|
||||
return None
|
||||
quality_key = quality_tier_for_source('qobuz', default='lossless')
|
||||
quality_info = QOBUZ_QUALITY_MAP.get(quality_key, QOBUZ_QUALITY_MAP['lossless'])
|
||||
# require_streamable=False: this is the exact track the user linked — don't
|
||||
# let a missing/absent 'streamable' flag in track/get drop it (#932).
|
||||
return self._qobuz_to_track_result(track, quality_info, require_streamable=False)
|
||||
except Exception as e:
|
||||
logger.debug(f"get_track_result failed for Qobuz {track_id}: {e}")
|
||||
return None
|
||||
|
||||
# ===================== Playlists & Favorites =====================
|
||||
#
|
||||
# Qobuz playlist sync surface — mirrors the Tidal client contract
|
||||
|
|
@ -987,7 +1017,7 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
return ([], [])
|
||||
|
||||
# Get configured quality for display
|
||||
quality_key = config_manager.get('qobuz.quality', 'lossless')
|
||||
quality_key = quality_tier_for_source('qobuz', default='lossless')
|
||||
quality_info = QOBUZ_QUALITY_MAP.get(quality_key, QOBUZ_QUALITY_MAP['lossless'])
|
||||
|
||||
track_results = []
|
||||
|
|
@ -1008,14 +1038,20 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
traceback.print_exc()
|
||||
return ([], [])
|
||||
|
||||
def _qobuz_to_track_result(self, track: Dict, quality_info: dict) -> Optional[TrackResult]:
|
||||
"""Convert Qobuz track dict to TrackResult (Soulseek-compatible format)."""
|
||||
def _qobuz_to_track_result(self, track: Dict, quality_info: dict,
|
||||
require_streamable: bool = True) -> Optional[TrackResult]:
|
||||
"""Convert Qobuz track dict to TrackResult (Soulseek-compatible format).
|
||||
|
||||
``require_streamable`` gates bulk SEARCH results on the ``streamable`` flag
|
||||
(filters noise). For a track the user explicitly pasted a link to, pass
|
||||
False: ``track/get`` may omit the flag (which would default-False and wrongly
|
||||
drop the exact track they asked for), and they should get to try it (#932)."""
|
||||
track_id = track.get('id')
|
||||
if not track_id:
|
||||
return None
|
||||
|
||||
# Check if track is streamable
|
||||
if not track.get('streamable', False):
|
||||
# Check if track is streamable (skipped for explicit by-id link fetches).
|
||||
if require_streamable and not track.get('streamable', False):
|
||||
return None
|
||||
|
||||
performer = track.get('performer', {})
|
||||
|
|
@ -1070,8 +1106,20 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
title=title,
|
||||
album=album_name,
|
||||
track_number=track.get('track_number'),
|
||||
# Stamp the Qobuz track id so a pasted-link manual search can float the
|
||||
# exact track to the top (#932). Without this the bubble never matched
|
||||
# and the linked track stayed buried among fuzzy lookalikes.
|
||||
_source_metadata={'source': 'qobuz', 'track_id': str(track.get('id') or '')},
|
||||
)
|
||||
|
||||
# Stamp real API quality so the global ranker sees actual
|
||||
# sample_rate/bit_depth. Hi-res only when the track itself is
|
||||
# hires-streamable; otherwise it streams as CD-quality FLAC.
|
||||
if hires_streamable and max_bit_depth >= 24:
|
||||
track_result.set_quality(quality_from_qobuz(max_sample_rate, max_bit_depth))
|
||||
else:
|
||||
track_result.set_quality(quality_from_qobuz(44.1, 16))
|
||||
|
||||
return track_result
|
||||
|
||||
# ===================== Download =====================
|
||||
|
|
@ -1181,7 +1229,7 @@ class QobuzClient(DownloadSourcePlugin):
|
|||
|
||||
try:
|
||||
# Determine quality
|
||||
quality_key = config_manager.get('qobuz.quality', 'lossless')
|
||||
quality_key = quality_tier_for_source('qobuz', default='lossless')
|
||||
quality_info = QOBUZ_QUALITY_MAP.get(quality_key, QOBUZ_QUALITY_MAP['lossless'])
|
||||
|
||||
# Quality fallback chain: hires_max → hires → lossless → mp3
|
||||
|
|
|
|||
0
core/quality/__init__.py
Normal file
0
core/quality/__init__.py
Normal file
103
core/quality/lossless.py
Normal file
103
core/quality/lossless.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
"""Single source of truth for "is this audio lossless?" + the lossy-copy
|
||||
overwrite invariant.
|
||||
|
||||
The "create a lossy copy of lossless tracks" feature lives in two places (the
|
||||
import post-processing path and the Lossy Converter repair job). Both used to
|
||||
hardcode ``.flac``, which is exactly how ALAC/WAV/DSD ended up being quality-
|
||||
profile options but NOT lossy-copy sources (#941). The knowledge of which
|
||||
formats are lossless now lives HERE, derived from the same format names the
|
||||
quality model ranks, so adding a format lights it up in both sites at once and
|
||||
they can never drift.
|
||||
|
||||
Two seams, both pure (no I/O) so they're unit-testable without real files:
|
||||
|
||||
* :func:`is_lossless_format` / :func:`is_lossless_audio_path` — eligibility.
|
||||
``.m4a``/``.mp4`` are ambiguous (ALAC=lossless, AAC=lossy) and can only be told
|
||||
apart by codec, so the path check delegates them to an *injected* ``probe_codec``
|
||||
callable — the file I/O stays at the edge, the decision stays pure.
|
||||
* :func:`lossy_output_would_overwrite_source` — the safety invariant: a lossy copy
|
||||
must NEVER be written over its own source (which becomes possible once ``.m4a``
|
||||
is eligible and the target codec is AAC → same ``.m4a`` path).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from core.quality.source_map import AUDIO_EXTENSIONS, format_from_extension
|
||||
|
||||
# Canonical lossless format set. MUST stay in sync with the lossless tiers in
|
||||
# core.quality.model.tier_score and the frontend RT_LOSSLESS_FORMATS — the
|
||||
# consistency test in tests/quality/test_lossless.py pins the tier agreement.
|
||||
LOSSLESS_FORMATS = frozenset({'flac', 'alac', 'wav', 'dsf'})
|
||||
|
||||
# Container extensions that may hold EITHER lossless (ALAC) or lossy (AAC) audio —
|
||||
# only a codec probe can decide, so they're never lossless on extension alone.
|
||||
_AMBIGUOUS_EXTS = frozenset({'m4a', 'mp4'})
|
||||
|
||||
|
||||
def is_lossless_format(fmt: Any) -> bool:
|
||||
"""True when a unified format name (as returned by ``format_from_extension``)
|
||||
is lossless. ``'aac'`` is False here — an ALAC-in-m4a file reports format
|
||||
``'aac'`` by extension and must be resolved by codec, not by name."""
|
||||
return str(fmt or '').lower() in LOSSLESS_FORMATS
|
||||
|
||||
|
||||
# Extensions worth checking as possibly-lossless (a caller can pre-filter SQL by
|
||||
# these, then confirm each via is_lossless_audio_path). Derived from the format
|
||||
# map so it can't drift: every extension whose format is lossless, plus the
|
||||
# ambiguous ALAC containers. Leading dot included (e.g. '.flac', '.m4a').
|
||||
LOSSLESS_CANDIDATE_EXTENSIONS = frozenset(
|
||||
e for e in AUDIO_EXTENSIONS
|
||||
if is_lossless_format(format_from_extension(e)) or e.lstrip('.') in _AMBIGUOUS_EXTS
|
||||
)
|
||||
|
||||
|
||||
def _ext(path: Any) -> str:
|
||||
return os.path.splitext(str(path or ''))[1].lower().lstrip('.')
|
||||
|
||||
|
||||
def is_lossless_audio_path(
|
||||
path: Any,
|
||||
*,
|
||||
probe_codec: Optional[Callable[[str], Optional[str]]] = None,
|
||||
) -> bool:
|
||||
"""True when the file at ``path`` is lossless.
|
||||
|
||||
Unambiguous extensions (flac/wav/aiff/dsf/dff/alac) are decided by extension.
|
||||
``.m4a``/``.mp4`` are decided by ``probe_codec(path)`` (returns the codec, e.g.
|
||||
``'alac'`` or ``'aac'``) — without a probe they're treated as NOT lossless, so
|
||||
a missing probe can never misclassify an AAC file as lossless.
|
||||
"""
|
||||
ext = _ext(path)
|
||||
if is_lossless_format(format_from_extension(ext)):
|
||||
return True
|
||||
if ext in _AMBIGUOUS_EXTS and probe_codec is not None:
|
||||
try:
|
||||
codec = (probe_codec(str(path)) or '').lower()
|
||||
except Exception:
|
||||
return False
|
||||
return 'alac' in codec
|
||||
return False
|
||||
|
||||
|
||||
def lossy_output_would_overwrite_source(source_path: Any, output_path: Any) -> bool:
|
||||
"""True when the computed lossy-copy output path is the source file itself.
|
||||
|
||||
Safety invariant: the converter must skip (never run ffmpeg with ``-y``) when
|
||||
this is True, or it would destroy the original. Happens when an ``.m4a`` ALAC
|
||||
source is converted with the AAC codec (output is also ``.m4a``)."""
|
||||
if not source_path or not output_path:
|
||||
return False
|
||||
a = os.path.normcase(os.path.normpath(str(source_path)))
|
||||
b = os.path.normcase(os.path.normpath(str(output_path)))
|
||||
return a == b
|
||||
|
||||
|
||||
__all__ = [
|
||||
'LOSSLESS_FORMATS',
|
||||
'is_lossless_format',
|
||||
'is_lossless_audio_path',
|
||||
'lossy_output_would_overwrite_source',
|
||||
]
|
||||
285
core/quality/model.py
Normal file
285
core/quality/model.py
Normal file
|
|
@ -0,0 +1,285 @@
|
|||
"""Source-agnostic audio quality model.
|
||||
|
||||
Every download source maps its result into ``AudioQuality``.
|
||||
The ``QualityTarget`` list in the user's profile defines the
|
||||
priority order (1st choice, 2nd choice, …). ``rank_candidate``
|
||||
scores any ``AudioQuality`` against that list so the same
|
||||
logic drives Soulseek, Tidal, Deezer, torrent — no per-source
|
||||
quality pipelines needed.
|
||||
|
||||
Soulseek attribute type codes (Soulseek protocol spec):
|
||||
0 = bitrate (kbps)
|
||||
1 = duration (seconds)
|
||||
2 = VBR flag
|
||||
4 = sample rate (Hz) — FLAC / WAV only
|
||||
5 = bit depth — FLAC / WAV only
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioQuality:
|
||||
"""Unified audio quality descriptor — source-agnostic."""
|
||||
|
||||
format: str # 'flac', 'mp3', 'aac', 'ogg', 'wav', 'unknown'
|
||||
bitrate: Optional[int] = None # kbps
|
||||
sample_rate: Optional[int] = None # Hz (e.g. 44100, 96000, 192000)
|
||||
bit_depth: Optional[int] = None # bits per sample (16, 24, 32)
|
||||
|
||||
def tier_score(self) -> float:
|
||||
"""Continuous score for ranking within a matched target bucket.
|
||||
Higher = better. Used as a tiebreaker after target-list matching.
|
||||
"""
|
||||
# NOTE: this only orders the *fallback* path (nothing matched a ranked
|
||||
# target) and tie-breaks candidates of the SAME format within one
|
||||
# matched target. Cross-format PRIORITY is decided solely by the user's
|
||||
# ranked-target list (target index), never by these numbers.
|
||||
format_base: dict[str, float] = {
|
||||
'dsf': 102.0, # DSD — 1-bit hi-res lossless, ranks at/above FLAC (#939)
|
||||
'flac': 100.0,
|
||||
'alac': 98.0, # lossless (Apple)
|
||||
'wav': 95.0,
|
||||
'ogg': 70.0,
|
||||
'opus': 65.0,
|
||||
'aac': 60.0,
|
||||
'mp3': 50.0,
|
||||
'wma': 30.0,
|
||||
}
|
||||
base = format_base.get(self.format.lower(), 10.0)
|
||||
|
||||
if self.format.lower() in ('flac', 'wav'):
|
||||
sr = self.sample_rate or 44100
|
||||
bd = self.bit_depth or 16
|
||||
# sample-rate contribution: 44.1 kHz = 0, 192 kHz = +20
|
||||
sr_score = min(sr / 192_000, 1.0) * 20
|
||||
# bit-depth contribution: 16-bit = 0, 24-bit = +10
|
||||
bd_score = max(bd - 16, 0) / 8 * 10
|
||||
return base + sr_score + bd_score
|
||||
else:
|
||||
br = self.bitrate or 0
|
||||
return base + min(br / 320, 1.0) * 10
|
||||
|
||||
def matches_target(self, target: QualityTarget) -> bool:
|
||||
"""True when this quality satisfies every constraint in *target*."""
|
||||
if target.format and target.format.lower() != self.format.lower():
|
||||
return False
|
||||
if target.min_bitrate and (self.bitrate or 0) < target.min_bitrate:
|
||||
return False
|
||||
if target.min_sample_rate:
|
||||
if self.sample_rate is not None:
|
||||
if self.sample_rate < target.min_sample_rate:
|
||||
return False
|
||||
else:
|
||||
# No sample-rate metadata (common on slskd FLAC). Use the kbps
|
||||
# heuristic when a bitrate is present; otherwise we CANNOT
|
||||
# confirm the spec, so fail the strict target rather than
|
||||
# over-claim it — an unknown-spec FLAC must not outrank a known
|
||||
# 16/44 FLAC under a hi-res target (#896 review #4). It falls to
|
||||
# the plain-flac bucket instead.
|
||||
# 16-bit/44.1 kHz ≈ 1411 kbps; 24-bit/96 kHz ≈ 4608 kbps.
|
||||
if self.format.lower() == 'flac' and self.bitrate:
|
||||
required_kbps = _sample_rate_to_min_kbps(target.min_sample_rate, target.bit_depth or 24)
|
||||
if self.bitrate < required_kbps:
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
if target.bit_depth:
|
||||
if self.bit_depth is not None:
|
||||
if self.bit_depth < target.bit_depth:
|
||||
return False
|
||||
else:
|
||||
# No bit-depth metadata. A hi-res (>=24-bit) target needs proof:
|
||||
# use the kbps heuristic if a bitrate is present, else fail
|
||||
# rather than over-claim. The 16-bit baseline still matches an
|
||||
# unknown-spec FLAC (any FLAC is at least CD quality). #896 review #4.
|
||||
if self.format.lower() == 'flac' and target.bit_depth >= 24:
|
||||
if self.bitrate:
|
||||
if self.bitrate < 1450:
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
def label(self) -> str:
|
||||
"""Human-readable label, e.g. 'FLAC 24-bit/192kHz' or 'MP3 320kbps'."""
|
||||
fmt = self.format.upper()
|
||||
if self.format.lower() in ('flac', 'wav'):
|
||||
bd = f"{self.bit_depth}-bit/" if self.bit_depth else ""
|
||||
sr = f"{self.sample_rate // 1000}kHz" if self.sample_rate else ""
|
||||
detail = f" {bd}{sr}".rstrip()
|
||||
return f"{fmt}{detail}" if detail.strip() else fmt
|
||||
else:
|
||||
br = f" {self.bitrate}kbps" if self.bitrate else ""
|
||||
return f"{fmt}{br}"
|
||||
|
||||
@classmethod
|
||||
def from_slskd_file(cls, file_data: dict, extension: str) -> 'AudioQuality':
|
||||
"""Build from a raw slskd API file entry.
|
||||
|
||||
slskd exposes Soulseek protocol file attributes as:
|
||||
``{"attributes": [{"type": 4, "value": 96000}, {"type": 5, "value": 24}, ...]}``
|
||||
"""
|
||||
attrs = {a['type']: a['value'] for a in file_data.get('attributes', [])}
|
||||
return cls(
|
||||
format=extension.lower().lstrip('.'),
|
||||
bitrate=file_data.get('bitRate') or attrs.get(0),
|
||||
sample_rate=attrs.get(4),
|
||||
bit_depth=attrs.get(5),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_tier(cls, fmt: str, bitrate: int, sample_rate: Optional[int] = None, bit_depth: Optional[int] = None) -> 'AudioQuality':
|
||||
"""Build from a hardcoded quality tier (Tidal, Deezer, Qobuz)."""
|
||||
return cls(format=fmt, bitrate=bitrate, sample_rate=sample_rate, bit_depth=bit_depth)
|
||||
|
||||
@classmethod
|
||||
def from_extension_and_bitrate(cls, extension: str, bitrate: Optional[int]) -> 'AudioQuality':
|
||||
"""Minimal constructor when only format + bitrate are known (torrent, YouTube)."""
|
||||
return cls(format=extension.lower().lstrip('.'), bitrate=bitrate)
|
||||
|
||||
|
||||
@dataclass
|
||||
class QualityTarget:
|
||||
"""One ranked entry in the user's quality priority list."""
|
||||
|
||||
label: str = ""
|
||||
format: Optional[str] = None # 'flac', 'mp3', 'aac', …
|
||||
bit_depth: Optional[int] = None # 16, 24
|
||||
min_sample_rate: Optional[int] = None # Hz
|
||||
min_bitrate: Optional[int] = None # kbps (lossy)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {k: v for k, v in self.__dict__.items() if v not in (None, "")}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict) -> 'QualityTarget':
|
||||
known = {f.name for f in cls.__dataclass_fields__.values()} # type: ignore[attr-defined]
|
||||
return cls(**{k: v for k, v in d.items() if k in known})
|
||||
|
||||
|
||||
# ── Default priority list ──────────────────────────────────────────────────────
|
||||
|
||||
DEFAULT_RANKED_TARGETS: List[QualityTarget] = [
|
||||
QualityTarget(label='FLAC 24-bit/192kHz', format='flac', bit_depth=24, min_sample_rate=192_000),
|
||||
QualityTarget(label='FLAC 24-bit/96kHz', format='flac', bit_depth=24, min_sample_rate=96_000),
|
||||
QualityTarget(label='FLAC 24-bit/48kHz', format='flac', bit_depth=24, min_sample_rate=48_000),
|
||||
QualityTarget(label='FLAC 24-bit/44.1kHz',format='flac', bit_depth=24, min_sample_rate=44_100),
|
||||
QualityTarget(label='FLAC 16-bit', format='flac', bit_depth=16),
|
||||
QualityTarget(label='MP3 320kbps', format='mp3', min_bitrate=320),
|
||||
QualityTarget(label='MP3 256kbps', format='mp3', min_bitrate=256),
|
||||
QualityTarget(label='MP3 192kbps', format='mp3', min_bitrate=192),
|
||||
]
|
||||
|
||||
|
||||
# ── Ranking helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
def rank_candidate(aq: AudioQuality, targets: List[QualityTarget]) -> Tuple[int, float]:
|
||||
"""Return *(target_index, tier_score)* for sorting.
|
||||
|
||||
Lower ``target_index`` → higher priority match.
|
||||
Candidates that satisfy no target get ``index = len(targets)``
|
||||
(they sort last but are not discarded — the caller decides that).
|
||||
"""
|
||||
for i, target in enumerate(targets):
|
||||
if aq.matches_target(target):
|
||||
return (i, aq.tier_score())
|
||||
return (len(targets), aq.tier_score())
|
||||
|
||||
|
||||
def filter_and_rank(
|
||||
candidates: list,
|
||||
targets: List[QualityTarget],
|
||||
*,
|
||||
fallback_enabled: bool = True,
|
||||
) -> list:
|
||||
"""Sort *candidates* (any objects with an ``audio_quality`` attribute)
|
||||
by quality priority.
|
||||
|
||||
Returns the subset that matched the *highest-priority* satisfied target,
|
||||
sorted by ``tier_score`` descending within that group.
|
||||
Falls back to all candidates sorted by score when ``fallback_enabled``
|
||||
and nothing matches, or when targets list is empty.
|
||||
"""
|
||||
if not targets:
|
||||
candidates_copy = list(candidates)
|
||||
candidates_copy.sort(key=lambda c: c.audio_quality.tier_score(), reverse=True)
|
||||
return candidates_copy
|
||||
|
||||
scored = [(rank_candidate(c.audio_quality, targets), c) for c in candidates]
|
||||
|
||||
# Best target index that any candidate reached
|
||||
best_idx = min((s[0][0] for s in scored), default=len(targets))
|
||||
|
||||
if best_idx < len(targets):
|
||||
winners = [c for (idx, _), c in scored if idx == best_idx]
|
||||
winners.sort(key=lambda c: c.audio_quality.tier_score(), reverse=True)
|
||||
return winners
|
||||
|
||||
if fallback_enabled:
|
||||
all_sorted = list(candidates)
|
||||
all_sorted.sort(key=lambda c: c.audio_quality.tier_score(), reverse=True)
|
||||
return all_sorted
|
||||
|
||||
return []
|
||||
|
||||
|
||||
# ── Migration helper ───────────────────────────────────────────────────────────
|
||||
|
||||
def v2_qualities_to_ranked_targets(qualities: dict) -> List[dict]:
|
||||
"""Convert old v2 ``qualities`` dict to a ranked-targets list.
|
||||
|
||||
Preserves the user's existing priority order while upgrading to the
|
||||
richer target format.
|
||||
"""
|
||||
_FORMAT_MAP = {
|
||||
'flac': {'format': 'flac', 'bit_depth': None},
|
||||
'mp3_320': {'format': 'mp3', 'min_bitrate': 320},
|
||||
'mp3_256': {'format': 'mp3', 'min_bitrate': 256},
|
||||
'mp3_192': {'format': 'mp3', 'min_bitrate': 192},
|
||||
# AAC (#886): opt-in tier. Match on format alone — Soulseek AAC/.m4a
|
||||
# rarely carries a bitrate attribute, so a min_bitrate gate would
|
||||
# reject every bitrate-less AAC. Priority order (above MP3, below FLAC)
|
||||
# is preserved by the caller's priority sort, not by min_bitrate.
|
||||
'aac': {'format': 'aac'},
|
||||
}
|
||||
enabled = [
|
||||
(cfg.get('priority', 999), name, cfg)
|
||||
for name, cfg in qualities.items()
|
||||
if cfg.get('enabled', False)
|
||||
]
|
||||
enabled.sort()
|
||||
targets = []
|
||||
for _, name, cfg in enabled:
|
||||
base = _FORMAT_MAP.get(name, {}).copy()
|
||||
if not base:
|
||||
continue
|
||||
if name == 'flac':
|
||||
bd = cfg.get('bit_depth', 'any')
|
||||
if bd == '24':
|
||||
base['bit_depth'] = 24
|
||||
base['label'] = 'FLAC 24-bit'
|
||||
elif bd == '16':
|
||||
base['bit_depth'] = 16
|
||||
base['label'] = 'FLAC 16-bit'
|
||||
else:
|
||||
base['label'] = 'FLAC (any)'
|
||||
else:
|
||||
base['label'] = name.upper().replace('_', ' ')
|
||||
targets.append(base)
|
||||
return targets
|
||||
|
||||
|
||||
# ── Internal helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
def _sample_rate_to_min_kbps(sample_rate: int, bit_depth: int) -> int:
|
||||
"""Approximate minimum kbps for a lossless file at the given spec.
|
||||
Used as heuristic when actual sample-rate metadata is absent.
|
||||
"""
|
||||
# kbps = sample_rate * channels * bit_depth / 1000 * compression_ratio
|
||||
# Assume stereo (2 ch) and ~0.6 FLAC compression ratio
|
||||
raw_kbps = sample_rate * 2 * bit_depth / 1000
|
||||
return int(raw_kbps * 0.55) # conservative compressed estimate
|
||||
148
core/quality/selection.py
Normal file
148
core/quality/selection.py
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
"""Quality-aware candidate selection shared by the search engine and the
|
||||
download orchestrator.
|
||||
|
||||
``rank_with_targets`` is the pure core: it ranks candidates against a target
|
||||
list and reports whether any candidate met a *real* target (strict, fallback
|
||||
off). The engine uses that ``satisfied`` flag to decide whether the current
|
||||
source is good enough or it should fall through to the next source in the
|
||||
hybrid chain.
|
||||
|
||||
``rank_for_profile`` is the thin DB-backed wrapper that loads the user's
|
||||
quality profile (with v2->v3 migration) and delegates.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Tuple
|
||||
|
||||
from core.quality.model import (
|
||||
QualityTarget,
|
||||
filter_and_rank,
|
||||
v2_qualities_to_ranked_targets,
|
||||
)
|
||||
|
||||
|
||||
def rank_with_targets(
|
||||
candidates: list,
|
||||
targets: List[QualityTarget],
|
||||
*,
|
||||
fallback_enabled: bool = True,
|
||||
) -> Tuple[list, bool]:
|
||||
"""Rank *candidates* against *targets*.
|
||||
|
||||
Returns ``(ranked, satisfied)`` where ``satisfied`` is True when at least
|
||||
one candidate meets a real target. When no targets are configured the
|
||||
profile imposes no constraint, so any non-empty result counts as
|
||||
satisfied (the first source wins, quality-sorted).
|
||||
"""
|
||||
if not candidates:
|
||||
return [], False
|
||||
|
||||
if not targets:
|
||||
ranked = filter_and_rank(candidates, targets, fallback_enabled=True)
|
||||
return ranked, bool(ranked)
|
||||
|
||||
strict = filter_and_rank(candidates, targets, fallback_enabled=False)
|
||||
if strict:
|
||||
return strict, True
|
||||
|
||||
if fallback_enabled:
|
||||
return filter_and_rank(candidates, targets, fallback_enabled=True), False
|
||||
return [], False
|
||||
|
||||
|
||||
def targets_from_profile(profile: dict) -> Tuple[List[QualityTarget], bool]:
|
||||
"""Convert a quality-profile dict into ``(targets, fallback_enabled)`` with
|
||||
v2->v3 migration applied. The single conversion path shared by the import
|
||||
guard, the download ranker and the library quality scanner."""
|
||||
raw_targets = profile.get('ranked_targets')
|
||||
if not raw_targets and 'qualities' in profile:
|
||||
raw_targets = v2_qualities_to_ranked_targets(profile['qualities'])
|
||||
|
||||
targets = [QualityTarget.from_dict(t) for t in (raw_targets or [])]
|
||||
fallback_enabled = profile.get('fallback_enabled', True)
|
||||
return targets, fallback_enabled
|
||||
|
||||
|
||||
def load_profile_targets() -> Tuple[List[QualityTarget], bool]:
|
||||
"""Load the user's quality profile from the DB and return
|
||||
``(targets, fallback_enabled)`` with v2->v3 migration applied.
|
||||
|
||||
Callers that rank across many sources should load once and reuse via
|
||||
:func:`rank_with_targets` rather than calling :func:`rank_for_profile`
|
||||
per source.
|
||||
"""
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
return targets_from_profile(MusicDatabase().get_quality_profile())
|
||||
|
||||
|
||||
def quality_meets_profile(aq, targets: List[QualityTarget]) -> bool:
|
||||
"""Strict: True iff *aq* satisfies at least one ranked *target*.
|
||||
|
||||
The shared definition of "good enough" for both the import guard and the
|
||||
library scanner — bit depth + sample rate are minimums (see
|
||||
:meth:`AudioQuality.matches_target`). Fallback is NOT consulted here; it's a
|
||||
download-time last-resort concession, not part of what counts as meeting the
|
||||
profile. ``targets`` empty → no constraint (True). ``aq`` None (probe
|
||||
failed) → True, so an unreadable file is never falsely flagged.
|
||||
"""
|
||||
if not targets:
|
||||
return True
|
||||
if aq is None:
|
||||
return True
|
||||
from core.quality.model import rank_candidate
|
||||
|
||||
idx, _ = rank_candidate(aq, targets)
|
||||
return idx < len(targets)
|
||||
|
||||
|
||||
_VALID_SEARCH_MODES = ("priority", "best_quality")
|
||||
|
||||
|
||||
def load_search_mode() -> str:
|
||||
"""Return the download search strategy from the user's quality profile.
|
||||
|
||||
``'priority'`` (default) keeps today's behaviour — the first source in the
|
||||
hybrid chain that meets a quality target wins. ``'best_quality'`` pools
|
||||
candidates across all sources and works them best→worst by actual audio
|
||||
quality. Any missing/unknown value resolves to ``'priority'`` so existing
|
||||
installs are unaffected.
|
||||
"""
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
try:
|
||||
profile = MusicDatabase().get_quality_profile()
|
||||
mode = profile.get("search_mode", "priority")
|
||||
except Exception:
|
||||
return "priority"
|
||||
return mode if mode in _VALID_SEARCH_MODES else "priority"
|
||||
|
||||
|
||||
def load_rank_candidates_by_quality() -> bool:
|
||||
"""Opt-in: order the priority-mode download walk (and thus the
|
||||
version-mismatch force-import pick, which takes the first-tried = best)
|
||||
by ranked-target quality instead of confidence-first.
|
||||
|
||||
Best-quality search mode is always quality-first regardless of this flag;
|
||||
this toggle only affects *priority* mode. Default ``False`` keeps the
|
||||
byte-for-byte old behaviour (confidence/peer-speed first), so existing
|
||||
installs are unaffected unless they opt in. Any missing value or DB error
|
||||
resolves to ``False``.
|
||||
"""
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
try:
|
||||
profile = MusicDatabase().get_quality_profile()
|
||||
return bool(profile.get("rank_candidates_by_quality", False))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def rank_for_profile(candidates: list) -> Tuple[list, bool]:
|
||||
"""Load the user's quality profile and rank *candidates* against it.
|
||||
|
||||
Returns ``(ranked, satisfied)`` — see :func:`rank_with_targets`.
|
||||
"""
|
||||
targets, fallback_enabled = load_profile_targets()
|
||||
return rank_with_targets(candidates, targets, fallback_enabled=fallback_enabled)
|
||||
206
core/quality/source_map.py
Normal file
206
core/quality/source_map.py
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
"""Per-source quality mappers — turn each download source's tier string or
|
||||
API values into a unified :class:`~core.quality.model.AudioQuality`.
|
||||
|
||||
Every streaming source describes quality differently (Tidal/HiFi use tier
|
||||
strings, Qobuz reports real kHz + bit depth, Deezer uses config codes,
|
||||
Amazon mixes real values with HD/UHD tiers). Centralising the knowledge
|
||||
here keeps the per-client code to a single call and keeps the tier tables
|
||||
in one auditable place.
|
||||
|
||||
Each value is a *claim*: the download client populates its ``TrackResult``
|
||||
from it so the global ranker can choose a source, and the post-download
|
||||
quality guard later verifies the real file. Over-claiming is the danger —
|
||||
an unknown tier maps to ``format='unknown'`` rather than pretending to be
|
||||
lossless.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from core.quality.model import AudioQuality
|
||||
from core.quality.selection import load_profile_targets
|
||||
|
||||
|
||||
# ── Extension → format string (source-agnostic) ────────────────────────────
|
||||
#
|
||||
# The single source of truth for mapping a file extension to the unified
|
||||
# AudioQuality ``format``. Every extension-based download source (Soulseek,
|
||||
# torrent/usenet file lists, …) classifies through this, so the ranked-target
|
||||
# system behaves identically across sources and adding a format here lights it
|
||||
# up everywhere at once. Unknown extensions → 'unknown' (never matches a
|
||||
# target, so it only ever comes through via the fallback toggle).
|
||||
#
|
||||
# AIFF/AIF are uncompressed PCM like WAV → the same 'wav' tier. ``.m4a``
|
||||
# defaults to 'aac'; an ALAC-in-m4a file can't be told apart by extension
|
||||
# alone, so probe_audio_quality corrects it from the real codec post-download.
|
||||
_EXTENSION_FORMAT_MAP = {
|
||||
'flac': 'flac',
|
||||
'alac': 'alac',
|
||||
'wav': 'wav', 'wave': 'wav',
|
||||
'aiff': 'wav', 'aif': 'wav', 'aifc': 'wav',
|
||||
'mp3': 'mp3',
|
||||
'm4a': 'aac', 'mp4': 'aac', 'aac': 'aac',
|
||||
'ogg': 'ogg', 'oga': 'ogg',
|
||||
'opus': 'opus',
|
||||
'wma': 'wma',
|
||||
# DSD (DSD Stream File / DSDIFF) — 1-bit hi-res lossless (e.g. DSD64 ≈ 11 Mbps).
|
||||
# Both container types map to the single 'dsf' tier (#939).
|
||||
'dsf': 'dsf', 'dff': 'dsf',
|
||||
}
|
||||
|
||||
# Audio extensions worth probing/classifying at all — derived from the map so
|
||||
# the allow-list and the classifier never drift apart.
|
||||
AUDIO_EXTENSIONS = {f'.{e}' for e in _EXTENSION_FORMAT_MAP}
|
||||
|
||||
|
||||
def format_from_extension(ext: str) -> str:
|
||||
"""Map a file extension (with or without leading dot) to the unified
|
||||
AudioQuality format string. Unknown → 'unknown'."""
|
||||
return _EXTENSION_FORMAT_MAP.get(str(ext or '').lower().lstrip('.'), 'unknown')
|
||||
|
||||
|
||||
# ── Tidal / HiFi (Monochrome is Tidal-backed) ──────────────────────────────
|
||||
#
|
||||
# Tidal exposes UPPER_SNAKE tier strings (``HI_RES_LOSSLESS``); HiFi's config
|
||||
# uses lowercase keys (``hires``/``lossless``). We normalise both into the
|
||||
# same lookup so one mapper serves both sources.
|
||||
|
||||
_TIDAL_HIRES = AudioQuality(format='flac', sample_rate=96000, bit_depth=24)
|
||||
_TIDAL_LOSSLESS = AudioQuality(format='flac', sample_rate=44100, bit_depth=16)
|
||||
_TIDAL_HIGH = AudioQuality(format='aac', bitrate=320)
|
||||
_TIDAL_LOW = AudioQuality(format='aac', bitrate=96)
|
||||
|
||||
TIDAL_TIER_MAP = {
|
||||
'HI_RES_LOSSLESS': _TIDAL_HIRES,
|
||||
'HI_RES': _TIDAL_HIRES,
|
||||
'HIRES': _TIDAL_HIRES,
|
||||
'LOSSLESS': _TIDAL_LOSSLESS,
|
||||
'HIGH': _TIDAL_HIGH,
|
||||
'LOW': _TIDAL_LOW,
|
||||
}
|
||||
|
||||
|
||||
def quality_from_tidal_tier(tier: str) -> AudioQuality:
|
||||
"""Map a Tidal/HiFi quality tier string to an AudioQuality.
|
||||
|
||||
Case-insensitive; accepts both ``HI_RES`` and ``hires`` spellings.
|
||||
Unrecognised tiers map to ``format='unknown'`` so they never
|
||||
over-claim lossless quality.
|
||||
"""
|
||||
key = (tier or '').strip().upper()
|
||||
return TIDAL_TIER_MAP.get(key, AudioQuality(format='unknown'))
|
||||
|
||||
|
||||
# ── Qobuz (real API values) ────────────────────────────────────────────────
|
||||
|
||||
def quality_from_qobuz(sampling_rate_khz: float, bit_depth: int) -> AudioQuality:
|
||||
"""Qobuz reports ``maximum_sampling_rate`` in kHz (e.g. 44.1, 96, 192)
|
||||
and ``maximum_bit_depth``. These are real values from the API.
|
||||
"""
|
||||
sample_rate = int(round(sampling_rate_khz * 1000)) if sampling_rate_khz else None
|
||||
return AudioQuality(format='flac', sample_rate=sample_rate, bit_depth=bit_depth)
|
||||
|
||||
|
||||
# ── Deezer (config code) ───────────────────────────────────────────────────
|
||||
|
||||
DEEZER_CODE_MAP = {
|
||||
'flac': AudioQuality(format='flac', sample_rate=44100, bit_depth=16),
|
||||
'mp3_320': AudioQuality(format='mp3', bitrate=320),
|
||||
'mp3_128': AudioQuality(format='mp3', bitrate=128),
|
||||
}
|
||||
|
||||
|
||||
def quality_from_deezer(code: str) -> AudioQuality:
|
||||
"""Map a Deezer download quality code to AudioQuality.
|
||||
|
||||
Deezer FLAC is always CD-quality (16-bit/44.1 kHz).
|
||||
"""
|
||||
return DEEZER_CODE_MAP.get((code or '').lower(), AudioQuality(format='unknown'))
|
||||
|
||||
|
||||
# ── Amazon Music (real sampleRate preferred, HD/UHD tier fallback) ─────────
|
||||
|
||||
_AMAZON_TIER_MAP = {
|
||||
'UHD': AudioQuality(format='flac', sample_rate=96000, bit_depth=24),
|
||||
'HD': AudioQuality(format='flac', sample_rate=44100, bit_depth=16),
|
||||
}
|
||||
|
||||
|
||||
def quality_from_amazon(
|
||||
tier: str,
|
||||
sample_rate: Optional[int] = None,
|
||||
bit_depth: Optional[int] = None,
|
||||
) -> AudioQuality:
|
||||
"""Amazon Music is FLAC; prefer the real ``sampleRate``/``bitDepth`` from
|
||||
the stream info when present, otherwise fall back to the HD/UHD tier.
|
||||
"""
|
||||
base = _AMAZON_TIER_MAP.get((tier or '').strip().upper(), AudioQuality(format='flac'))
|
||||
return AudioQuality(
|
||||
format='flac',
|
||||
sample_rate=sample_rate if sample_rate is not None else base.sample_rate,
|
||||
bit_depth=bit_depth if bit_depth is not None else base.bit_depth,
|
||||
)
|
||||
|
||||
|
||||
# ── Profile-driven download tier (replaces per-source quality settings) ─────
|
||||
#
|
||||
# Each source's selectable download tiers, ordered best → worst, with the
|
||||
# AudioQuality the tier delivers. ``quality_tier_for_source`` walks these to
|
||||
# request the LOWEST tier that satisfies the user's top global target — so the
|
||||
# global quality profile, not a per-source dropdown, decides what each source
|
||||
# fetches.
|
||||
|
||||
_SOURCE_TIER_LADDERS: dict[str, list[tuple[str, AudioQuality]]] = {
|
||||
'tidal': [
|
||||
('hires', AudioQuality('flac', sample_rate=96000, bit_depth=24)),
|
||||
('lossless', AudioQuality('flac', sample_rate=44100, bit_depth=16)),
|
||||
('high', AudioQuality('aac', bitrate=320)),
|
||||
('low', AudioQuality('aac', bitrate=96)),
|
||||
],
|
||||
'hifi': [
|
||||
('hires', AudioQuality('flac', sample_rate=96000, bit_depth=24)),
|
||||
('lossless', AudioQuality('flac', sample_rate=44100, bit_depth=16)),
|
||||
('high', AudioQuality('aac', bitrate=320)),
|
||||
('low', AudioQuality('aac', bitrate=96)),
|
||||
],
|
||||
'qobuz': [
|
||||
('hires_max', AudioQuality('flac', sample_rate=192000, bit_depth=24)),
|
||||
('hires', AudioQuality('flac', sample_rate=96000, bit_depth=24)),
|
||||
('lossless', AudioQuality('flac', sample_rate=44100, bit_depth=16)),
|
||||
('mp3', AudioQuality('mp3', bitrate=320)),
|
||||
],
|
||||
'deezer': [
|
||||
('flac', AudioQuality('flac', sample_rate=44100, bit_depth=16)),
|
||||
('mp3_320', AudioQuality('mp3', bitrate=320)),
|
||||
('mp3_128', AudioQuality('mp3', bitrate=128)),
|
||||
],
|
||||
'amazon': [
|
||||
('flac', AudioQuality('flac', sample_rate=48000, bit_depth=24)),
|
||||
('opus', AudioQuality('aac', bitrate=320)),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def quality_tier_for_source(source_name: str, *, default: Optional[str] = None) -> Optional[str]:
|
||||
"""Return the source tier key to request, derived from the global profile.
|
||||
|
||||
Picks the lowest tier in the source's ladder that satisfies the user's
|
||||
top (most-preferred) target — respecting the quality ceiling and saving
|
||||
bandwidth. Falls back to the source's max tier when none can satisfy it
|
||||
(best effort), or to the source's max when no targets are configured.
|
||||
Returns *default* for an unknown source.
|
||||
"""
|
||||
ladder = _SOURCE_TIER_LADDERS.get(source_name)
|
||||
if not ladder:
|
||||
return default
|
||||
|
||||
targets, _ = load_profile_targets()
|
||||
if not targets:
|
||||
return ladder[0][0]
|
||||
|
||||
top = targets[0]
|
||||
for key, aq in reversed(ladder): # low → high
|
||||
if aq.matches_target(top):
|
||||
return key
|
||||
return ladder[0][0] # best effort: max tier
|
||||
|
|
@ -70,6 +70,9 @@ class QueueItem:
|
|||
# 'tags' = read each file's embedded tags as the source
|
||||
# of truth (issue #592). Zero API calls.
|
||||
metadata_source: str = 'api'
|
||||
# Rename-only mode (#875): move files to the current naming scheme WITHOUT the
|
||||
# copy + post-processing (re-tag / quality / AcoustID) the full flow runs.
|
||||
rename_only: bool = False
|
||||
status: str = 'queued' # queued | running | done | failed | cancelled
|
||||
started_at: Optional[float] = None
|
||||
finished_at: Optional[float] = None
|
||||
|
|
@ -96,6 +99,7 @@ class QueueItem:
|
|||
'artist_name': self.artist_name,
|
||||
'source': self.source,
|
||||
'metadata_source': self.metadata_source,
|
||||
'rename_only': self.rename_only,
|
||||
'enqueued_at': self.enqueued_at,
|
||||
'started_at': self.started_at,
|
||||
'finished_at': self.finished_at,
|
||||
|
|
@ -161,6 +165,7 @@ class ReorganizeQueue:
|
|||
artist_name: str,
|
||||
source: Optional[str] = None,
|
||||
metadata_source: str = 'api',
|
||||
rename_only: bool = False,
|
||||
) -> dict:
|
||||
"""Add an album to the queue. Returns a result dict:
|
||||
|
||||
|
|
@ -190,6 +195,7 @@ class ReorganizeQueue:
|
|||
source=source,
|
||||
enqueued_at=time.time(),
|
||||
metadata_source=metadata_source or 'api',
|
||||
rename_only=bool(rename_only),
|
||||
)
|
||||
self._items.append(item)
|
||||
position = sum(1 for i in self._items if i.status == 'queued')
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ def build_runner(
|
|||
is_shutting_down_fn: Callable[[], bool],
|
||||
get_download_path: Callable[[], str],
|
||||
get_transfer_path: Callable[[], str],
|
||||
build_final_path_fn: Optional[Callable] = None,
|
||||
) -> Callable[[object], dict]:
|
||||
"""Return the closure the queue worker invokes per item.
|
||||
|
||||
|
|
@ -59,7 +60,7 @@ def build_runner(
|
|||
A callable ``runner(item)`` suitable for
|
||||
:meth:`core.reorganize_queue.ReorganizeQueue.set_runner`.
|
||||
"""
|
||||
from core.library_reorganize import reorganize_album
|
||||
from core.library_reorganize import reorganize_album, reorganize_album_rename_only
|
||||
from core.reorganize_queue import get_queue
|
||||
|
||||
def _update_track_path(track_id, new_path):
|
||||
|
|
@ -80,17 +81,6 @@ def build_runner(
|
|||
# server restart.
|
||||
download_dir = get_download_path()
|
||||
transfer_dir = get_transfer_path()
|
||||
staging_root = os.path.join(download_dir, 'ssync_staging')
|
||||
try:
|
||||
os.makedirs(staging_root, exist_ok=True)
|
||||
except OSError as mk_err:
|
||||
logger.error(f"[Reorganize] Cannot create staging dir {staging_root}: {mk_err}")
|
||||
return {
|
||||
'status': 'setup_failed',
|
||||
'source': None,
|
||||
'total': 0, 'moved': 0, 'skipped': 0, 'failed': 0,
|
||||
'errors': [{'error': f'Could not create staging dir: {mk_err}'}],
|
||||
}
|
||||
|
||||
def _cleanup_empty(src_dir):
|
||||
try:
|
||||
|
|
@ -105,6 +95,42 @@ def build_runner(
|
|||
# Progress fan-out failures must never break a run.
|
||||
logger.debug("reorganize progress fan-out: %s", e)
|
||||
|
||||
# Rename-only mode (#875): just move files to the current scheme — no staging,
|
||||
# no copy, no post-processing. Falls through to the full pipeline otherwise.
|
||||
if getattr(item, 'rename_only', False):
|
||||
if build_final_path_fn is None:
|
||||
return {
|
||||
'status': 'setup_failed', 'source': None,
|
||||
'total': 0, 'moved': 0, 'skipped': 0, 'failed': 0,
|
||||
'errors': [{'error': 'Rename-only mode unavailable (no path builder)'}],
|
||||
}
|
||||
return reorganize_album_rename_only(
|
||||
album_id=item.album_id,
|
||||
db=get_database(),
|
||||
transfer_dir=transfer_dir,
|
||||
resolve_file_path_fn=resolve_file_path_fn,
|
||||
build_final_path_fn=build_final_path_fn,
|
||||
update_track_path_fn=_update_track_path,
|
||||
cleanup_empty_dir_fn=_cleanup_empty,
|
||||
on_progress=_on_progress,
|
||||
primary_source=item.source,
|
||||
strict_source=bool(item.source),
|
||||
metadata_source=getattr(item, 'metadata_source', 'api') or 'api',
|
||||
stop_check=is_shutting_down_fn,
|
||||
)
|
||||
|
||||
staging_root = os.path.join(download_dir, 'ssync_staging')
|
||||
try:
|
||||
os.makedirs(staging_root, exist_ok=True)
|
||||
except OSError as mk_err:
|
||||
logger.error(f"[Reorganize] Cannot create staging dir {staging_root}: {mk_err}")
|
||||
return {
|
||||
'status': 'setup_failed',
|
||||
'source': None,
|
||||
'total': 0, 'moved': 0, 'skipped': 0, 'failed': 0,
|
||||
'errors': [{'error': f'Could not create staging dir: {mk_err}'}],
|
||||
}
|
||||
|
||||
return reorganize_album(
|
||||
album_id=item.album_id,
|
||||
db=get_database(),
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ _JOB_MODULES = [
|
|||
'core.repair_jobs.metadata_gap_filler',
|
||||
'core.repair_jobs.album_completeness',
|
||||
'core.repair_jobs.fake_lossless_detector',
|
||||
'core.repair_jobs.quality_upgrade_scanner',
|
||||
'core.repair_jobs.library_reorganize',
|
||||
'core.repair_jobs.mbid_mismatch_detector',
|
||||
'core.repair_jobs.single_album_dedup',
|
||||
|
|
@ -51,6 +52,7 @@ _JOB_MODULES = [
|
|||
'core.repair_jobs.canonical_version_resolve',
|
||||
'core.repair_jobs.library_retag',
|
||||
'core.repair_jobs.quality_upgrade',
|
||||
'core.repair_jobs.short_preview_track',
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -389,14 +389,43 @@ class AcoustIDScannerJob(RepairJob):
|
|||
cur.execute(
|
||||
"UPDATE tracks SET verification_status = ? WHERE id = ?",
|
||||
(status, track_id))
|
||||
matched = 0
|
||||
exp = expected or {}
|
||||
# Find the canonical history row for this file. The stored path is frozen at
|
||||
# import time while the file has since moved (media-server import / reorganize),
|
||||
# so an exact-path match alone misses it — then the status never lands and a
|
||||
# duplicate row gets inserted every scan (#934). Match exact path first, then
|
||||
# filename guarded by title, and HEAL the row's path so future scans match cleanly.
|
||||
from core.downloads.history_match import pick_history_row, like_filename_filter
|
||||
current = fpath or db_path
|
||||
basename = os.path.basename(current) if current else ''
|
||||
clauses, params = [], []
|
||||
for p in {p for p in (fpath, db_path) if p}:
|
||||
clauses.append("file_path = ?")
|
||||
params.append(p)
|
||||
if basename:
|
||||
clauses.append("file_path LIKE ? ESCAPE '\\'")
|
||||
params.append(like_filename_filter(basename))
|
||||
row_id = None
|
||||
if clauses:
|
||||
cur.execute(
|
||||
"UPDATE library_history SET verification_status = ? WHERE file_path = ?",
|
||||
(status, p))
|
||||
matched += max(getattr(cur, 'rowcount', 0) or 0, 0)
|
||||
if status == 'unverified' and matched == 0:
|
||||
exp = expected or {}
|
||||
"SELECT id, file_path, title, download_source FROM library_history WHERE "
|
||||
+ " OR ".join(clauses),
|
||||
params)
|
||||
row_id = pick_history_row(
|
||||
cur.fetchall(),
|
||||
current_paths=(fpath, db_path),
|
||||
basename=basename, title=exp.get('title') or '')
|
||||
if row_id is not None:
|
||||
cur.execute(
|
||||
"UPDATE library_history SET verification_status = ?, file_path = ? WHERE id = ?",
|
||||
(status, current, row_id))
|
||||
# Drop synthetic scan-created duplicates for this exact file (the #934
|
||||
# leftovers). Exact path → collision-free; never touches a real download row.
|
||||
cur.execute(
|
||||
"DELETE FROM library_history WHERE id != ? AND download_source = 'acoustid_scan' "
|
||||
"AND file_path = ?",
|
||||
(row_id, current))
|
||||
elif status == 'unverified':
|
||||
cur.execute(
|
||||
"""INSERT INTO library_history
|
||||
(event_type, title, artist_name, album_name, file_path,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -91,13 +91,19 @@ class EmptyFolderCleanerJob(RepairJob):
|
|||
ignore_disposable = False
|
||||
try:
|
||||
if context.config_manager:
|
||||
ignore_junk = bool(context.config_manager.get(
|
||||
'repair.jobs.empty_folder_cleaner.remove_junk_files', True))
|
||||
# #891: also clear folders left holding only images / .lrc / sidecars
|
||||
# (what a reorganize leaves behind). Opt-in — default off.
|
||||
ignore_disposable = bool(context.config_manager.get(
|
||||
'repair.jobs.empty_folder_cleaner.remove_residual_files', False))
|
||||
except Exception: # noqa: S110 — setting read is best-effort; defaults to True
|
||||
# #912: job settings are persisted as a nested dict under
|
||||
# `repair.jobs.<id>.settings` (see RepairWorker.set_job_settings / get_job_config).
|
||||
# The old flat-key reads ('repair.jobs.empty_folder_cleaner.remove_residual_files')
|
||||
# never matched what the UI saves, so the #891 opt-in toggle silently did nothing —
|
||||
# the scan always fell back to the False default and skipped every image/.lrc folder.
|
||||
job_settings = context.config_manager.get(
|
||||
'repair.jobs.empty_folder_cleaner.settings', {}) or {}
|
||||
if isinstance(job_settings, dict):
|
||||
ignore_junk = bool(job_settings.get('remove_junk_files', True))
|
||||
# #891: also clear folders left holding only images / .lrc / sidecars
|
||||
# (what a reorganize leaves behind). Opt-in — default off.
|
||||
ignore_disposable = bool(job_settings.get('remove_residual_files', False))
|
||||
except Exception: # noqa: S110 — setting read is best-effort; defaults to junk-only
|
||||
pass
|
||||
|
||||
flagged = set() # dir paths we'd remove → a parent sees them as "gone"
|
||||
|
|
|
|||
|
|
@ -1,13 +1,19 @@
|
|||
"""Lossy Converter Job — finds FLAC files that don't have a lossy copy.
|
||||
"""Lossy Converter Job — finds lossless files that don't have a lossy copy.
|
||||
|
||||
Scans the library for FLAC files without a corresponding lossy copy alongside
|
||||
Scans the library for lossless files without a corresponding lossy copy alongside
|
||||
them, and creates a finding for each. The fix action converts the file using
|
||||
ffmpeg with the user's configured codec/bitrate settings.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from core.imports.file_ops import m4a_codec
|
||||
from core.library.path_resolver import resolve_library_file_path
|
||||
from core.quality.lossless import (
|
||||
LOSSLESS_CANDIDATE_EXTENSIONS,
|
||||
is_lossless_audio_path,
|
||||
lossy_output_would_overwrite_source,
|
||||
)
|
||||
from core.repair_jobs import register_job
|
||||
from core.repair_jobs.base import JobContext, JobResult, RepairJob
|
||||
from utils.logging_config import get_logger
|
||||
|
|
@ -21,6 +27,16 @@ CODEC_MAP = {
|
|||
}
|
||||
|
||||
|
||||
def _lossless_ext_where(col: str) -> str:
|
||||
"""SQL pre-filter matching files whose extension *might* be lossless. The
|
||||
final decision (including ALAC-in-.m4a, which needs a codec probe) is made
|
||||
per-file by is_lossless_audio_path. Extensions are trusted constants from the
|
||||
quality model, never user input — safe to interpolate."""
|
||||
return '(' + ' OR '.join(
|
||||
f"LOWER({col}) LIKE '%{ext}'" for ext in sorted(LOSSLESS_CANDIDATE_EXTENSIONS)
|
||||
) + ')'
|
||||
|
||||
|
||||
def _resolve_file_path(file_path, transfer_folder, download_folder=None, config_manager=None):
|
||||
"""Backwards-compat wrapper. Use ``resolve_library_file_path`` directly."""
|
||||
return resolve_library_file_path(
|
||||
|
|
@ -35,15 +51,15 @@ def _resolve_file_path(file_path, transfer_folder, download_folder=None, config_
|
|||
class LossyConverterJob(RepairJob):
|
||||
job_id = 'lossy_converter'
|
||||
display_name = 'Lossy Converter'
|
||||
description = 'Finds FLAC files without a lossy copy'
|
||||
description = 'Finds lossless files without a lossy copy'
|
||||
help_text = (
|
||||
'Scans your library for FLAC files that don\'t already have a lossy copy '
|
||||
'Scans your library for lossless files (FLAC/ALAC/WAV/AIFF/DSD) that don\'t already have a lossy copy '
|
||||
'(MP3, Opus, or AAC) alongside them.\n\n'
|
||||
'Uses the codec setting from your Lossy Copy configuration on the Settings '
|
||||
'page. Enable Lossy Copy in Settings first, then run this job to find FLAC '
|
||||
'files missing a lossy copy.\n\n'
|
||||
'Each finding can be fixed individually or in bulk — the fix action converts '
|
||||
'the FLAC file using ffmpeg at your configured bitrate.\n\n'
|
||||
'the lossless file using ffmpeg at your configured bitrate.\n\n'
|
||||
'Requires ffmpeg to be installed.'
|
||||
)
|
||||
icon = 'repair-icon-lossy'
|
||||
|
|
@ -81,14 +97,14 @@ class LossyConverterJob(RepairJob):
|
|||
try:
|
||||
conn = context.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
cursor.execute(f"""
|
||||
SELECT t.id, t.title, ar.name, al.title, t.file_path,
|
||||
al.thumb_url, ar.thumb_url
|
||||
FROM tracks t
|
||||
LEFT JOIN artists ar ON ar.id = t.artist_id
|
||||
LEFT JOIN albums al ON al.id = t.album_id
|
||||
WHERE t.file_path IS NOT NULL AND t.file_path != ''
|
||||
AND LOWER(t.file_path) LIKE '%.flac'
|
||||
AND {_lossless_ext_where('t.file_path')}
|
||||
""")
|
||||
tracks = cursor.fetchall()
|
||||
except Exception as e:
|
||||
|
|
@ -104,7 +120,7 @@ class LossyConverterJob(RepairJob):
|
|||
context.update_progress(0, total)
|
||||
if context.report_progress:
|
||||
context.report_progress(
|
||||
phase=f'Scanning {total} FLAC files for missing {quality_label} copies...',
|
||||
phase=f'Scanning {total} lossless files for missing {quality_label} copies...',
|
||||
total=total
|
||||
)
|
||||
|
||||
|
|
@ -135,8 +151,17 @@ class LossyConverterJob(RepairJob):
|
|||
if not resolved or not os.path.exists(resolved):
|
||||
continue
|
||||
|
||||
# Confirm it's actually lossless — the SQL pre-filter lets .m4a through,
|
||||
# which is ALAC (lossless) OR AAC (lossy); only a codec probe decides.
|
||||
if not is_lossless_audio_path(resolved, probe_codec=m4a_codec):
|
||||
continue
|
||||
|
||||
# Check if lossy copy already exists
|
||||
out_path = os.path.splitext(resolved)[0] + out_ext
|
||||
# Never offer to convert a file onto itself (e.g. .m4a ALAC + AAC target
|
||||
# lands on the same path) — that conversion would destroy the original.
|
||||
if lossy_output_would_overwrite_source(resolved, out_path):
|
||||
continue
|
||||
if os.path.exists(out_path):
|
||||
continue
|
||||
|
||||
|
|
@ -159,7 +184,7 @@ class LossyConverterJob(RepairJob):
|
|||
file_path=file_path,
|
||||
title=f'No {quality_label} copy: {title or "Unknown"}',
|
||||
description=(
|
||||
f'FLAC file "{title}" by {artist_name or "Unknown"} does not have '
|
||||
f'Lossless file "{title}" by {artist_name or "Unknown"} does not have '
|
||||
f'a {quality_label} copy alongside it'
|
||||
),
|
||||
details={
|
||||
|
|
@ -195,7 +220,7 @@ class LossyConverterJob(RepairJob):
|
|||
context.report_progress(
|
||||
scanned=total, total=total,
|
||||
phase='Complete',
|
||||
log_line=f'Found {result.findings_created} FLAC files without {quality_label} copies',
|
||||
log_line=f'Found {result.findings_created} lossless files without {quality_label} copies',
|
||||
log_type='success' if result.findings_created == 0 else 'info'
|
||||
)
|
||||
|
||||
|
|
@ -208,10 +233,10 @@ class LossyConverterJob(RepairJob):
|
|||
try:
|
||||
conn = context.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
cursor.execute(f"""
|
||||
SELECT COUNT(*) FROM tracks
|
||||
WHERE file_path IS NOT NULL AND file_path != ''
|
||||
AND LOWER(file_path) LIKE '%.flac'
|
||||
AND {_lossless_ext_where('file_path')}
|
||||
""")
|
||||
row = cursor.fetchone()
|
||||
return row[0] if row else 0
|
||||
|
|
|
|||
|
|
@ -14,9 +14,10 @@ is queued until you review and Apply the finding — at which point the matched
|
|||
track (carrying its album context) is added to the wishlist, exactly like every
|
||||
other acquisition path.
|
||||
|
||||
The quality decision (``meets_preferred_quality``) is a pure function so it can be
|
||||
unit-tested without a database or network. Transcode/"fake lossless" detection is
|
||||
intentionally NOT done here — that's the separate Fake Lossless Detector job.
|
||||
Quality is judged using the real file (mutagen-measured bit depth / sample rate /
|
||||
bitrate) checked against the user's v3 ranked profile targets — fully profile-driven,
|
||||
no hardcoded thresholds. Transcode/"fake lossless" detection is the separate Fake
|
||||
Lossless Detector job.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -39,30 +40,25 @@ from core.discovery.quality_scanner import (
|
|||
)
|
||||
from core.library.file_tags import read_embedded_tags
|
||||
from core.library.path_resolver import resolve_library_file_path
|
||||
# v3 quality: probe the real file + check it against the ranked profile targets,
|
||||
# the SAME definition the download import guard uses. Module-level so they're
|
||||
# monkeypatchable in tests.
|
||||
from core.imports.file_ops import probe_audio_quality
|
||||
from core.quality.model import rank_candidate
|
||||
from core.quality.selection import targets_from_profile, quality_meets_profile
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("repair_jobs.quality_upgrade")
|
||||
|
||||
|
||||
# Quality ranks — higher is better. Lossless tops everything; lossy tiers fall out
|
||||
# of bitrate. 0 means "below the lowest tracked tier / unknown".
|
||||
RANK_LOSSLESS = 4
|
||||
RANK_320 = 3
|
||||
RANK_256 = 2
|
||||
RANK_192 = 1
|
||||
RANK_BELOW = 0
|
||||
def _to_bool(val) -> bool:
|
||||
"""Coerce a setting value to bool. Handles Python bool, string 'true'/'false', and int."""
|
||||
if isinstance(val, bool):
|
||||
return val
|
||||
if isinstance(val, str):
|
||||
return val.lower() == 'true'
|
||||
return bool(val) if val is not None else False
|
||||
|
||||
LOSSLESS_EXTENSIONS = {'.flac', '.alac', '.ape', '.wav', '.aiff', '.aif', '.dsf', '.dff', '.m4a'}
|
||||
# NB: .m4a is ambiguous (ALAC vs AAC); we treat the *format* as lossy-capable and
|
||||
# rely on bitrate below — a true ALAC .m4a reports a lossless-scale bitrate.
|
||||
|
||||
# Quality-profile bucket key -> rank.
|
||||
_PROFILE_KEY_RANK = {
|
||||
'flac': RANK_LOSSLESS,
|
||||
'mp3_320': RANK_320,
|
||||
'mp3_256': RANK_256,
|
||||
'mp3_192': RANK_192,
|
||||
}
|
||||
|
||||
# Per-source file-tag key holding that source's own track ID (written by enrichment).
|
||||
_SOURCE_TRACK_ID_TAG = {
|
||||
|
|
@ -79,93 +75,6 @@ _SOURCE_TRACK_ID_TAG = {
|
|||
_DURATION_TOLERANCE_MS = 5000
|
||||
|
||||
|
||||
def _normalize_kbps(bitrate: Optional[int]) -> Optional[int]:
|
||||
"""Library bitrate may be stored in bps (e.g. 320000) or kbps (320).
|
||||
Normalize to kbps. Returns None when unknown/zero."""
|
||||
if not bitrate:
|
||||
return None
|
||||
try:
|
||||
b = int(bitrate)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if b <= 0:
|
||||
return None
|
||||
return b // 1000 if b > 4000 else b
|
||||
|
||||
|
||||
def classify_track_quality(file_path: str, bitrate: Optional[int]) -> Optional[int]:
|
||||
"""Rank a file by format + bitrate. Returns a RANK_* value, or None when it
|
||||
can't be judged (a lossy file with no known bitrate)."""
|
||||
ext = os.path.splitext(file_path or '')[1].lower()
|
||||
kbps = _normalize_kbps(bitrate)
|
||||
|
||||
# Lossless containers: a real lossless file has a high bitrate; a low one is a
|
||||
# lossy stream in a lossless container — but flagging that is the Fake Lossless
|
||||
# Detector's job, so here we treat the lossless *format* as top rank.
|
||||
if ext in {'.flac', '.alac', '.ape', '.wav', '.aiff', '.aif', '.dsf', '.dff'}:
|
||||
return RANK_LOSSLESS
|
||||
# .m4a / lossy: judge purely by bitrate. A lossless-scale bitrate (ALAC in m4a,
|
||||
# or a mislabeled lossless) ranks as lossless.
|
||||
if kbps is None:
|
||||
return None
|
||||
if kbps >= 800:
|
||||
return RANK_LOSSLESS
|
||||
if kbps >= 280:
|
||||
return RANK_320
|
||||
if kbps >= 200:
|
||||
return RANK_256
|
||||
if kbps >= 150:
|
||||
return RANK_192
|
||||
return RANK_BELOW
|
||||
|
||||
|
||||
def preferred_quality_floor(quality_profile: Dict[str, Any]) -> Optional[int]:
|
||||
"""The lowest acceptable quality rank from the profile's ENABLED buckets — the
|
||||
floor a track must meet. Returns None when nothing is enabled (caller should
|
||||
then flag nothing, rather than flagging everything)."""
|
||||
qualities = (quality_profile or {}).get('qualities', {}) or {}
|
||||
enabled_ranks = [
|
||||
_PROFILE_KEY_RANK[key]
|
||||
for key, cfg in qualities.items()
|
||||
if isinstance(cfg, dict) and cfg.get('enabled') and key in _PROFILE_KEY_RANK
|
||||
]
|
||||
if not enabled_ranks:
|
||||
return None
|
||||
return min(enabled_ranks)
|
||||
|
||||
|
||||
def meets_preferred_quality(file_path: str, bitrate: Optional[int],
|
||||
quality_profile: Dict[str, Any]) -> bool:
|
||||
"""Pure decision: does this track already meet the user's preferred quality?
|
||||
|
||||
A track meets quality when its format+bitrate rank is at least the profile's
|
||||
floor (the worst quality the user still accepts). This honors a profile that
|
||||
enables, say, FLAC *and* MP3-320: a 320 kbps MP3 passes, a 128 kbps MP3 does
|
||||
not. With nothing enabled, everything passes (we never flag the whole library
|
||||
on an empty profile)."""
|
||||
floor = preferred_quality_floor(quality_profile)
|
||||
if floor is None:
|
||||
return True
|
||||
|
||||
file_rank = classify_track_quality(file_path, bitrate)
|
||||
if file_rank is None:
|
||||
# Lossy file with unknown bitrate: only judgeable when the floor is
|
||||
# lossless (then any lossy file is below it). Otherwise don't flag.
|
||||
ext = os.path.splitext(file_path or '')[1].lower()
|
||||
if floor == RANK_LOSSLESS and ext not in LOSSLESS_EXTENSIONS:
|
||||
return False
|
||||
return True
|
||||
|
||||
return file_rank >= floor
|
||||
|
||||
|
||||
def _rank_label(rank: Optional[int]) -> str:
|
||||
return {
|
||||
RANK_LOSSLESS: 'Lossless', RANK_320: 'MP3 320', RANK_256: 'MP3 256',
|
||||
RANK_192: 'MP3 192', RANK_BELOW: 'low bitrate',
|
||||
}.get(rank, 'unknown')
|
||||
|
||||
|
||||
def _norm_isrc(value: Any) -> str:
|
||||
"""Canonicalize an ISRC for comparison: uppercase, strip dashes/spaces."""
|
||||
if not value:
|
||||
|
|
@ -173,14 +82,14 @@ def _norm_isrc(value: Any) -> str:
|
|||
return str(value).upper().replace('-', '').replace(' ', '').strip()
|
||||
|
||||
|
||||
def _read_file_ids(file_path: str) -> Dict[str, str]:
|
||||
def _read_file_ids(file_path: str, resolved_path: Optional[str] = None) -> Dict[str, str]:
|
||||
"""Read the identifiers enrichment embedded in the file's tags.
|
||||
|
||||
Enrichment matches every track to the metadata sources and writes the IDs
|
||||
(ISRC + per-source track IDs) into the file — so an already-enriched track
|
||||
carries its exact identity. Returns a dict with a normalized ``isrc`` plus any
|
||||
``<source>_track_id`` tags present; empty dict when unreadable / not enriched."""
|
||||
resolved = resolve_library_file_path(file_path) if file_path else None
|
||||
resolved = resolved_path or (resolve_library_file_path(file_path) if file_path else None)
|
||||
if not resolved and file_path and os.path.isfile(file_path):
|
||||
resolved = file_path
|
||||
if not resolved:
|
||||
|
|
@ -436,45 +345,61 @@ def _find_best_match(engine: Any, source_priority: List[str], title: str, artist
|
|||
@register_job
|
||||
class QualityUpgradeJob(RepairJob):
|
||||
job_id = 'quality_upgrade'
|
||||
display_name = 'Quality Upgrade Finder'
|
||||
description = 'Finds library tracks below your preferred quality and proposes a better version'
|
||||
display_name = 'Quality Upgrade Finder (active — proposes a replacement)'
|
||||
description = 'Finds library tracks below your quality profile and actively searches a better version to add to the wishlist'
|
||||
help_text = (
|
||||
'Scans your library (or just your watchlist artists) and compares each '
|
||||
"track against your Quality Profile using BOTH the file format and its "
|
||||
'bitrate — so a 128 kbps MP3 is no longer treated the same as a 320 kbps '
|
||||
'one, and enabling MP3-320/256 in your profile actually counts.\n\n'
|
||||
'For every track below your preferred quality it resolves the exact better '
|
||||
'version using the most precise identity available, in order: the source '
|
||||
"track ID enrichment wrote into the file → the file's ISRC → the album's "
|
||||
'tracklist (by stored album ID or album search) → a name/artist search. The '
|
||||
'fuzzy steps also reject candidates whose length is off (wrong live/edit cut). '
|
||||
'It skips tracks it already proposed, so re-runs are cheap. Nothing is queued '
|
||||
'automatically: applying a finding adds that matched track — with its album '
|
||||
'context — to the wishlist, the same as any other download.\n\n'
|
||||
'ACTIVE quality job. For every library track below your quality profile it '
|
||||
'goes one step further than the Quality Check (flag-only) scanner: it '
|
||||
'actively SEARCHES your metadata source for the exact better version and '
|
||||
'attaches it to the finding, so Apply adds that track straight to your '
|
||||
'wishlist.\n\n'
|
||||
'Quality is judged the SAME way as the download/import pipeline — it reads '
|
||||
'the REAL file with mutagen (measured bit depth / sample rate / bitrate) and '
|
||||
'checks it against your v3 quality profile targets (strict: fallback is '
|
||||
'ignored, that\'s a download-time concession, not "good enough" for an '
|
||||
'upgrade). So a 128 kbps MP3, a 16-bit FLAC where you want 24-bit, etc. are '
|
||||
'all caught accurately.\n\n'
|
||||
'For every below-profile track it resolves the better version by the most '
|
||||
'precise identity available, in order: the source track ID enrichment wrote '
|
||||
"into the file → the file's ISRC → the album's tracklist (by stored album ID "
|
||||
'or album search) → a name/artist search (with a duration guard against wrong '
|
||||
'live/edit cuts). It skips tracks it already proposed, so re-runs are cheap. '
|
||||
'Nothing is queued automatically — applying a finding adds the matched track '
|
||||
'(with album context) to the wishlist, like any other download.\n\n'
|
||||
'Settings:\n'
|
||||
'- Scope: "watchlist" (watchlisted artists only) or "all" (whole library)\n'
|
||||
'- Min confidence: minimum match confidence (0-1) to surface a finding\n\n'
|
||||
'Note: detecting fake/transcoded lossless files is handled by the separate '
|
||||
'Fake Lossless Detector job.'
|
||||
'- Min confidence: minimum match confidence (0-1) to surface a finding\n'
|
||||
'- Deep audio verify (default OFF): also run the ffmpeg decode guard '
|
||||
'(truncation + silence) per track — catches broken/incomplete files the '
|
||||
'header hides, but decodes every file (seconds per track, CPU-heavy).\n\n'
|
||||
'Sibling job: "Quality Check (flag only)" finds the same below-profile tracks '
|
||||
'but only flags them for you to decide per finding (re-download / delete / '
|
||||
'ignore) instead of searching a replacement. Fake/transcoded lossless '
|
||||
'detection is the separate Fake Lossless Detector job.'
|
||||
)
|
||||
icon = 'repair-icon-lossy'
|
||||
default_enabled = False
|
||||
default_interval_hours = 168
|
||||
default_settings = {'scope': 'watchlist', 'min_confidence': 0.7}
|
||||
setting_options = {'scope': ['watchlist', 'all']}
|
||||
default_settings = {'scope': 'all', 'min_confidence': 0.7, 'deep_audio_verify': False, 'require_top_target': False}
|
||||
setting_options = {'scope': ['all', 'watchlist'], 'deep_audio_verify': [True, False], 'require_top_target': [True, False]}
|
||||
auto_fix = False
|
||||
|
||||
def _get_settings(self, context: JobContext) -> Dict[str, Any]:
|
||||
cfg = context.config_manager
|
||||
scope = 'watchlist'
|
||||
min_conf = 0.7
|
||||
if cfg:
|
||||
scope = cfg.get(self.get_config_key('settings.scope'), 'watchlist') or 'watchlist'
|
||||
merged = dict(self.default_settings)
|
||||
if context.config_manager:
|
||||
try:
|
||||
min_conf = float(cfg.get(self.get_config_key('settings.min_confidence'), 0.7))
|
||||
except (TypeError, ValueError):
|
||||
min_conf = 0.7
|
||||
return {'scope': scope, 'min_confidence': min_conf}
|
||||
cfg = context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {})
|
||||
if isinstance(cfg, dict):
|
||||
merged.update(cfg)
|
||||
except Exception as e:
|
||||
logger.debug("settings read failed: %s", e)
|
||||
try:
|
||||
merged['min_confidence'] = float(merged.get('min_confidence', 0.7))
|
||||
except (TypeError, ValueError):
|
||||
merged['min_confidence'] = 0.7
|
||||
merged['deep_audio_verify'] = _to_bool(merged.get('deep_audio_verify'))
|
||||
merged['require_top_target'] = _to_bool(merged.get('require_top_target'))
|
||||
return merged
|
||||
|
||||
def _load_tracks(self, db: Any, scope: str) -> List[dict]:
|
||||
conn = db._get_connection()
|
||||
|
|
@ -530,13 +455,27 @@ class QualityUpgradeJob(RepairJob):
|
|||
settings = self._get_settings(context)
|
||||
scope = settings['scope']
|
||||
min_conf = settings['min_confidence']
|
||||
deep_verify = settings['deep_audio_verify']
|
||||
require_top = settings['require_top_target']
|
||||
|
||||
# v3 quality: judge the REAL file (mutagen-measured bit depth / sample rate
|
||||
# / bitrate) against the profile's ranked targets — the SAME definition the
|
||||
# download import guard uses. Strict: fallback is ignored (a download-time
|
||||
# concession, not "good enough" for an upgrade). targets_from_profile /
|
||||
# quality_meets_profile / probe_audio_quality are imported at module level.
|
||||
db = context.db
|
||||
quality_profile = db.get_quality_profile()
|
||||
if preferred_quality_floor(quality_profile) is None:
|
||||
logger.info("[Quality Upgrade] No quality buckets enabled in profile — nothing to flag")
|
||||
targets, _fallback = targets_from_profile(quality_profile)
|
||||
if not targets:
|
||||
logger.info("[Quality Upgrade] No quality targets in profile — nothing to flag")
|
||||
return result
|
||||
|
||||
logger.info(
|
||||
"[Quality Upgrade] scope=%s require_top=%s · all targets: %s",
|
||||
scope, require_top,
|
||||
[t.label for t in targets] if targets else '(none — all pass)',
|
||||
)
|
||||
|
||||
try:
|
||||
tracks = self._load_tracks(db, scope)
|
||||
except Exception as e:
|
||||
|
|
@ -583,13 +522,53 @@ class QualityUpgradeJob(RepairJob):
|
|||
result.findings_skipped_dedup += 1
|
||||
continue
|
||||
|
||||
if meets_preferred_quality(file_path, bitrate, quality_profile):
|
||||
# v3 quality decision — probe the REAL file. Resolve the library path
|
||||
# first (the DB stores a possibly-relative path). Pass config_manager so
|
||||
# the resolver can find the transfer/music folders and expand relative paths.
|
||||
resolved_path = resolve_library_file_path(
|
||||
file_path,
|
||||
transfer_folder=context.transfer_folder,
|
||||
config_manager=context.config_manager,
|
||||
) if file_path else None
|
||||
if not resolved_path and file_path and os.path.isfile(file_path):
|
||||
resolved_path = file_path
|
||||
|
||||
measured_aq = probe_audio_quality(resolved_path) if resolved_path else None
|
||||
|
||||
# Optional ffmpeg deep verify (default off): a truncated/silent file is
|
||||
# treated as "needs a replacement" just like a below-profile one.
|
||||
broken_reason = None
|
||||
if deep_verify and resolved_path:
|
||||
try:
|
||||
from core.imports.silence import detect_broken_audio
|
||||
broken_reason = detect_broken_audio(resolved_path)
|
||||
except Exception as e:
|
||||
logger.debug("[Quality Upgrade] deep verify failed for %s: %s", file_path, e)
|
||||
|
||||
if measured_aq is None and not broken_reason:
|
||||
# Can't read the file → can't judge it; leave it alone.
|
||||
result.skipped += 1
|
||||
if context.update_progress and (i + 1) % 25 == 0:
|
||||
context.update_progress(i + 1, total)
|
||||
continue
|
||||
|
||||
# Below preferred quality — find a better version to propose.
|
||||
if not broken_reason and measured_aq is not None:
|
||||
if require_top:
|
||||
# ranking-based: skip only if the file already sits at rank 0
|
||||
# (the top configured target). Any lower rank → flag for upgrade.
|
||||
idx, _ = rank_candidate(measured_aq, targets)
|
||||
already_best = (idx == 0)
|
||||
else:
|
||||
# default: skip if the file meets ANY configured target (i.e.
|
||||
# it's not below the acceptable floor).
|
||||
already_best = quality_meets_profile(measured_aq, targets)
|
||||
if already_best:
|
||||
result.skipped += 1
|
||||
if context.update_progress and (i + 1) % 25 == 0:
|
||||
context.update_progress(i + 1, total)
|
||||
continue
|
||||
|
||||
# Below profile (or broken) — find a better version to propose.
|
||||
if engine is None:
|
||||
from core.matching_engine import MusicMatchingEngine
|
||||
engine = MusicMatchingEngine()
|
||||
|
|
@ -602,17 +581,20 @@ class QualityUpgradeJob(RepairJob):
|
|||
logger.info("[Quality Upgrade] Spotify rate-limited — stopping scan early")
|
||||
return result
|
||||
|
||||
current_rank = classify_track_quality(file_path, bitrate)
|
||||
current_label = _rank_label(current_rank)
|
||||
current_label = measured_aq.label() if measured_aq is not None else 'broken/unreadable'
|
||||
if broken_reason:
|
||||
current_label = f'{current_label} (broken: {broken_reason})' if measured_aq is not None else f'broken ({broken_reason})'
|
||||
if context.report_progress:
|
||||
_why = 'broken audio' if broken_reason else 'low quality'
|
||||
context.report_progress(
|
||||
scanned=i + 1, total=total,
|
||||
log_line=f'Low quality ({current_label}): {artist_name} - {title}',
|
||||
log_line=f'{_why} ({current_label}): {artist_name} - {title}',
|
||||
log_type='info')
|
||||
|
||||
# Read the identifiers enrichment embedded in the file once (ISRC +
|
||||
# per-source track IDs), used by the two most-exact tiers below.
|
||||
file_ids = _read_file_ids(file_path)
|
||||
# Pass resolved_path so the inner resolver doesn't redo the lookup.
|
||||
file_ids = _read_file_ids(file_path, resolved_path=resolved_path)
|
||||
|
||||
# Tiered match, best identity first, loosest last:
|
||||
# 0. The active source's OWN track ID, embedded in the file by
|
||||
|
|
@ -685,8 +667,9 @@ class QualityUpgradeJob(RepairJob):
|
|||
file_path=file_path,
|
||||
title=f'Upgrade: {artist_name} - {title} ({current_label})',
|
||||
description=(
|
||||
f'"{title}" by {artist_name} is {current_label}, below your preferred '
|
||||
f'quality. Best match: "{_track_name(best)}" via {source} '
|
||||
f'"{title}" by {artist_name} is {current_label}'
|
||||
+ (f', below your preferred quality ({targets[0].label})' if require_top and targets else ', below your preferred quality')
|
||||
+ f'. Best match: "{_track_name(best)}" via {source} '
|
||||
f'({_MATCH_NOTE.get(matched_via, "matched") if matched_via != "search" else f"confidence {conf:.0%}"}). '
|
||||
'Apply to add it to the wishlist.'),
|
||||
details={
|
||||
|
|
@ -715,6 +698,13 @@ class QualityUpgradeJob(RepairJob):
|
|||
|
||||
if context.update_progress:
|
||||
context.update_progress(total, total)
|
||||
logger.info("[Quality Upgrade] %d scanned, %d upgrades found, %d met/skip",
|
||||
logger.info("[Quality Upgrade] %d scanned, %d upgrades found, %d already met profile / skipped",
|
||||
result.scanned, result.findings_created, result.skipped)
|
||||
if result.scanned > 0 and result.findings_created == 0 and result.errors == 0:
|
||||
logger.info(
|
||||
"[Quality Upgrade] All tracks already satisfy the configured targets. "
|
||||
"If you expected upgrades, check your quality profile — the current "
|
||||
"top target is: %s",
|
||||
targets[0].label if targets else '(none)',
|
||||
)
|
||||
return result
|
||||
|
|
|
|||
416
core/repair_jobs/quality_upgrade_scanner.py
Normal file
416
core/repair_jobs/quality_upgrade_scanner.py
Normal file
|
|
@ -0,0 +1,416 @@
|
|||
"""Quality Upgrade Scanner Job — flags library tracks below the user's profile.
|
||||
|
||||
Walks the music folders ON DISK (transfer + download + every configured library
|
||||
path) exactly like the Orphan / Fake-Lossless detectors — those reliably "see"
|
||||
files because they os.walk real directories instead of trying to resolve the
|
||||
DB's stored (often relative) paths. For each audio file it probes the ACTUAL
|
||||
measured audio quality (bit depth / sample rate / bitrate via the same
|
||||
`probe_audio_quality` the download import guard uses) and checks it against the
|
||||
user's v3 ranked targets with `quality_meets_profile` (strict — fallback
|
||||
ignored, that's a download-time concession, not a definition of "good enough").
|
||||
|
||||
Every file that satisfies none of the targets becomes a finding the user can:
|
||||
- 'redownload': add the track to the wishlist and delete the low-quality file
|
||||
- 'delete': remove the low-quality file (+ DB row when known)
|
||||
- 'ignore': dismiss the finding (handled in the UI via the dismiss endpoint)
|
||||
|
||||
Each walked file is matched back to its DB track (by path suffix) so the finding
|
||||
carries the real title/artist/album + track id; when no DB row matches, the
|
||||
file's own tags are used and the finding is filed as a loose 'file'.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from core.repair_jobs import register_job
|
||||
from core.repair_jobs.base import JobContext, JobResult, RepairJob
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("repair_job.quality_upgrade")
|
||||
|
||||
AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'}
|
||||
|
||||
|
||||
@register_job
|
||||
class QualityUpgradeScannerJob(RepairJob):
|
||||
job_id = 'quality_upgrade_scanner'
|
||||
display_name = 'Quality Check (flag only — you decide per finding)'
|
||||
description = 'Flags library tracks below your quality profile; you choose re-download / delete / ignore per finding'
|
||||
help_text = (
|
||||
'FLAG-ONLY quality job. Walks your music library folder on disk (so it also '
|
||||
'catches loose files not in the DB) and checks every track against your v3 '
|
||||
'quality profile — then just FLAGS what is below profile. Unlike the active '
|
||||
'"Quality Upgrade Finder", it does NOT search a replacement; you decide what '
|
||||
'to do per finding: Re-download / Delete / Ignore.\n\n'
|
||||
'Two-stage check (same as the download/import pipeline):\n'
|
||||
'1. Real-audio guard (optional, ffmpeg) — decodes the file (truncation + '
|
||||
'silence detection) to catch broken/incomplete audio the header hides.\n'
|
||||
'2. Quality gate — measured bit depth / sample rate / bitrate vs your '
|
||||
'profile targets.\n\n'
|
||||
'Settings:\n'
|
||||
'- Deep audio verify (default OFF): run the ffmpeg decode guard. Off = fast '
|
||||
'header-only quality pass (milliseconds/track). On = full decode '
|
||||
'(seconds/track, CPU-heavy) but catches broken/silent audio.\n'
|
||||
'- library_tracks_only (default off): only check files matched to a '
|
||||
'library DB track (skip loose/orphan files).\n\n'
|
||||
'The scan only reports — it never deletes or re-downloads on its own. '
|
||||
'Use the sibling "Quality Upgrade Finder" instead if you want it to actively '
|
||||
'find and queue a better version for you.'
|
||||
)
|
||||
icon = 'repair-icon-lossless'
|
||||
default_enabled = False
|
||||
default_interval_hours = 168
|
||||
# library_tracks_only: when ON, only check files that match a library DB
|
||||
# track (skips loose/orphan files). Default OFF — the scan checks EVERY
|
||||
# audio file in the Music Library output folder, which is what users expect
|
||||
# ("check my library folder"). DB matching after a reset is unreliable and
|
||||
# would wrongly skip everything. Turn ON to ignore non-DB files.
|
||||
#
|
||||
# deep_audio_verify default OFF: the ffmpeg decode is the CPU-heavy step. Most
|
||||
# users want the fast header-only quality pass; turn it on for a deep scan that
|
||||
# also catches broken/silent audio. (Matches the download pipeline's default.)
|
||||
default_settings = {'library_tracks_only': False, 'deep_audio_verify': False, 'require_top_target': False}
|
||||
setting_options = {'library_tracks_only': [True, False],
|
||||
'deep_audio_verify': [True, False],
|
||||
'require_top_target': [True, False]}
|
||||
auto_fix = False # User chooses fix action per finding
|
||||
|
||||
def scan(self, context: JobContext) -> JobResult:
|
||||
result = JobResult()
|
||||
|
||||
# Load the user's v3 ranked targets — the SAME definition the download
|
||||
# import guard uses. Strict: a track is below-profile when its measured
|
||||
# quality satisfies NONE of the targets (fallback is not consulted).
|
||||
from core.quality.selection import targets_from_profile, quality_meets_profile
|
||||
try:
|
||||
profile = context.db.get_quality_profile()
|
||||
except Exception as e:
|
||||
logger.warning("Could not load quality profile: %s", e)
|
||||
return result
|
||||
targets, _fallback = targets_from_profile(profile)
|
||||
if not targets:
|
||||
logger.info("Quality profile has no targets — nothing to check against")
|
||||
return result
|
||||
|
||||
logger.info("Quality upgrade scan — profile targets (strict): %s",
|
||||
[t.label for t in targets])
|
||||
|
||||
from core.imports.file_ops import probe_audio_quality
|
||||
# Same real-file AudioGuard the download/import pipeline runs: ffmpeg
|
||||
# DECODES the file (astats + silencedetect) to catch truncated or
|
||||
# mostly-silent audio the header can't reveal.
|
||||
from core.imports.silence import detect_broken_audio
|
||||
|
||||
# --- Collect the music folders to walk (real dirs, abspath'd) ---
|
||||
base_dirs = self._collect_music_dirs(context)
|
||||
if not base_dirs:
|
||||
logger.warning(
|
||||
"[QualityScan] No existing music folder to walk (transfer=%r, cwd=%r). "
|
||||
"Set soulseek.transfer_path to the real mount or add your library under "
|
||||
"Settings → Library → Music Paths.",
|
||||
context.transfer_folder, os.getcwd())
|
||||
return result
|
||||
logger.info("[QualityScan] Walking %d folder(s): %r", len(base_dirs), base_dirs)
|
||||
|
||||
# --- Gather audio files (dedup by real path) ---
|
||||
audio_files = []
|
||||
seen = set()
|
||||
for base in base_dirs:
|
||||
for root, _dirs, files in os.walk(base):
|
||||
if context.check_stop():
|
||||
return result
|
||||
for fname in files:
|
||||
if os.path.splitext(fname)[1].lower() in AUDIO_EXTENSIONS:
|
||||
fpath = os.path.join(root, fname)
|
||||
rp = os.path.realpath(fpath)
|
||||
if rp in seen:
|
||||
continue
|
||||
seen.add(rp)
|
||||
audio_files.append(fpath)
|
||||
|
||||
total = len(audio_files)
|
||||
logger.info("[QualityScan] Found %d audio file(s) to check", total)
|
||||
if context.report_progress:
|
||||
context.report_progress(phase=f'Checking {total} files...', total=total)
|
||||
if context.update_progress:
|
||||
context.update_progress(0, total)
|
||||
|
||||
# --- DB suffix index so a walked file maps back to its track row ---
|
||||
db_index = self._build_db_suffix_index(context)
|
||||
# Only check files that are part of the LIBRARY (have a DB track row).
|
||||
# The transfer/download folders also hold pre-import leftovers (e.g.
|
||||
# residue after a DB reset) — those are orphans, not library tracks, and
|
||||
# belong to the Orphan File Detector, not a quality upgrade scan. Default
|
||||
# ON so the scan reflects the user's actual library, not download junk.
|
||||
_settings = self._get_settings(context)
|
||||
library_only = _settings.get('library_tracks_only', False)
|
||||
# Deep verify = run the ffmpeg AudioGuard (real decode) per file, exactly
|
||||
# like the download pipeline. Slower than a header read (seconds vs ms) but
|
||||
# it verifies the REAL audio, not just the metadata. OFF by default (the
|
||||
# decode is the CPU-heavy step); turn on for a deep scan.
|
||||
deep_verify = _settings.get('deep_audio_verify', False)
|
||||
# require_top_target: flag files that meet a lower target but not the
|
||||
# highest-priority one (e.g. 16-bit FLAC when 24-bit is preferred).
|
||||
require_top = _settings.get('require_top_target', False)
|
||||
check_targets = targets[:1] if require_top and len(targets) > 1 else targets
|
||||
|
||||
probe_failed = 0
|
||||
not_in_library = 0
|
||||
for i, fpath in enumerate(audio_files):
|
||||
if context.check_stop():
|
||||
return result
|
||||
if i % 20 == 0 and context.wait_if_paused():
|
||||
return result
|
||||
|
||||
fname = os.path.basename(fpath)
|
||||
|
||||
# Map to a DB track up front (cheap suffix lookup). When scoping to
|
||||
# the library, skip anything with no DB row BEFORE probing — no point
|
||||
# reading hundreds of orphan files.
|
||||
meta = self._match_db(fpath, db_index)
|
||||
if library_only and meta is None:
|
||||
not_in_library += 1
|
||||
result.skipped += 1
|
||||
continue
|
||||
if meta is None:
|
||||
meta = self._read_file_tags(fpath)
|
||||
|
||||
result.scanned += 1
|
||||
if context.report_progress and i % 25 == 0:
|
||||
context.report_progress(
|
||||
scanned=i + 1, total=total,
|
||||
phase=f'Checking {i + 1} / {total}',
|
||||
log_line=f'Checking: {fname}',
|
||||
log_type='info',
|
||||
)
|
||||
|
||||
# === Real-file verification — the SAME two stages the download /
|
||||
# import pipeline runs on every file ===
|
||||
# 1) AudioGuard: ffmpeg DECODES the audio (astats / silencedetect)
|
||||
# to catch truncated or mostly-silent files the header hides.
|
||||
# 2) Quality gate: measured quality (mutagen) vs the ranked profile.
|
||||
try:
|
||||
broken_reason = detect_broken_audio(fpath) if deep_verify else None
|
||||
except Exception as e:
|
||||
logger.debug("AudioGuard failed for %s: %s", fname, e)
|
||||
broken_reason = None
|
||||
|
||||
try:
|
||||
aq = probe_audio_quality(fpath)
|
||||
except Exception as e:
|
||||
logger.debug("Probe failed for %s: %s", fname, e)
|
||||
aq = None
|
||||
|
||||
if broken_reason:
|
||||
issue = 'broken_audio'
|
||||
current_label = aq.label() if aq is not None else 'unknown'
|
||||
elif aq is None:
|
||||
# Header unreadable → can't judge quality; leave it unflagged.
|
||||
probe_failed += 1
|
||||
result.skipped += 1
|
||||
continue
|
||||
elif not quality_meets_profile(aq, check_targets):
|
||||
issue = 'below_profile'
|
||||
current_label = aq.label()
|
||||
else:
|
||||
# Decodes fully AND meets the profile → genuinely good.
|
||||
if context.update_progress and (i + 1) % 25 == 0:
|
||||
context.update_progress(i + 1, total)
|
||||
continue
|
||||
|
||||
# Build the finding (broken audio OR below profile).
|
||||
target_labels = [t.label for t in targets]
|
||||
disp_title = meta.get('title') or os.path.splitext(fname)[0]
|
||||
disp_artist = meta.get('artist') or 'Unknown'
|
||||
if issue == 'broken_audio':
|
||||
_title = f'Broken/incomplete audio: {disp_title}'
|
||||
_desc = (f'"{disp_title}" by {disp_artist} failed real-audio '
|
||||
f'verification (ffmpeg): {broken_reason}')
|
||||
_severity = 'warning'
|
||||
else:
|
||||
_pref = targets[0].label if require_top and len(targets) > 1 else None
|
||||
_title = f'{"Upgradeable" if _pref else "Below quality"}: {disp_title} ({current_label})'
|
||||
_desc = (f'"{disp_title}" by {disp_artist} is {current_label}'
|
||||
+ (f', below your preferred quality ({_pref}).' if _pref else
|
||||
f', which does not meet your quality profile '
|
||||
f'({", ".join(target_labels[:3])}'
|
||||
f'{"…" if len(target_labels) > 3 else ""}).'))
|
||||
_severity = 'info'
|
||||
|
||||
if context.report_progress:
|
||||
context.report_progress(log_line=_title, log_type='error')
|
||||
if context.create_finding:
|
||||
inserted = context.create_finding(
|
||||
job_id=self.job_id,
|
||||
finding_type='quality_upgrade',
|
||||
severity=_severity,
|
||||
entity_type='track' if meta.get('track_id') else 'file',
|
||||
entity_id=str(meta['track_id']) if meta.get('track_id') else None,
|
||||
file_path=fpath,
|
||||
title=_title,
|
||||
description=_desc,
|
||||
details={
|
||||
'quality_issue': issue,
|
||||
'broken_audio_reason': broken_reason or '',
|
||||
'current_quality': current_label,
|
||||
'current_format': aq.format if aq is not None else '',
|
||||
'current_bitrate': aq.bitrate if aq is not None else None,
|
||||
'current_sample_rate': aq.sample_rate if aq is not None else None,
|
||||
'current_bit_depth': aq.bit_depth if aq is not None else None,
|
||||
'target_qualities': target_labels,
|
||||
'expected_title': disp_title,
|
||||
'expected_artist': disp_artist,
|
||||
'album_title': meta.get('album', ''),
|
||||
'track_number': meta.get('track_number'),
|
||||
'album_thumb_url': meta.get('album_thumb_url'),
|
||||
'artist_thumb_url': meta.get('artist_thumb_url'),
|
||||
},
|
||||
)
|
||||
if inserted:
|
||||
result.findings_created += 1
|
||||
else:
|
||||
result.findings_skipped_dedup += 1
|
||||
|
||||
if context.update_progress and (i + 1) % 25 == 0:
|
||||
context.update_progress(i + 1, total)
|
||||
|
||||
if context.update_progress:
|
||||
context.update_progress(total, total)
|
||||
|
||||
if probe_failed:
|
||||
logger.warning("[QualityScan] %d/%d files could not be probed (unreadable)",
|
||||
probe_failed, total)
|
||||
if not_in_library:
|
||||
logger.info(
|
||||
"[QualityScan] %d/%d files skipped — not in the library DB (orphan "
|
||||
"leftovers in transfer/downloads; disable 'library_tracks_only' to "
|
||||
"include them)", not_in_library, total)
|
||||
logger.info("Quality upgrade scan: %d checked, %d below profile, %d skipped",
|
||||
result.scanned, result.findings_created, result.skipped)
|
||||
return result
|
||||
|
||||
def _get_settings(self, context: JobContext) -> dict:
|
||||
merged = dict(self.default_settings)
|
||||
if context.config_manager:
|
||||
try:
|
||||
cfg = context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {})
|
||||
if isinstance(cfg, dict):
|
||||
merged.update(cfg)
|
||||
except Exception as e:
|
||||
logger.debug("settings read failed: %s", e)
|
||||
for key in ('library_tracks_only', 'deep_audio_verify', 'require_top_target'):
|
||||
val = merged.get(key)
|
||||
if not isinstance(val, bool):
|
||||
merged[key] = str(val).lower() == 'true' if val is not None else False
|
||||
return merged
|
||||
|
||||
def _collect_music_dirs(self, context: JobContext) -> list:
|
||||
"""The music-library directories to walk, as absolute paths (dedup).
|
||||
|
||||
Only the user's MUSIC LIBRARY is scanned — that's the "Output Folder
|
||||
(Music Library)" setting (soulseek.transfer_path) plus any custom
|
||||
library paths (library.music_paths, for media-server setups). The
|
||||
download/staging folders are deliberately NOT walked: they hold raw,
|
||||
pre-import downloads and leftovers, not the finished library, and the
|
||||
user expects quality checks to run on their library only. Whatever
|
||||
custom path the user configured for the output folder is respected,
|
||||
because it's read live from config here.
|
||||
"""
|
||||
cm = context.config_manager
|
||||
raw = [context.transfer_folder]
|
||||
if cm:
|
||||
try:
|
||||
raw.append(cm.get('soulseek.transfer_path', './Transfer'))
|
||||
mp = cm.get('library.music_paths', []) or []
|
||||
if isinstance(mp, list):
|
||||
raw.extend([p for p in mp if isinstance(p, str) and p.strip()])
|
||||
except Exception as e:
|
||||
logger.debug("music dir config read failed: %s", e)
|
||||
out, seen = [], set()
|
||||
for d in raw:
|
||||
if not d:
|
||||
continue
|
||||
ad = os.path.abspath(d)
|
||||
if ad in seen:
|
||||
continue
|
||||
seen.add(ad)
|
||||
if os.path.isdir(ad):
|
||||
out.append(ad)
|
||||
return out
|
||||
|
||||
def _build_db_suffix_index(self, context: JobContext) -> dict:
|
||||
"""Map normalized path suffixes (last 1-3 components, lowercased) →
|
||||
track metadata, so a walked absolute file can be matched to its DB row
|
||||
even when the DB stores a different (relative) path prefix."""
|
||||
index = {}
|
||||
conn = None
|
||||
try:
|
||||
conn = context.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT t.id, t.title,
|
||||
COALESCE(NULLIF(t.track_artist, ''), ar.name) AS artist,
|
||||
t.file_path, t.track_number,
|
||||
al.title AS album_title, al.thumb_url, ar.thumb_url
|
||||
FROM tracks t
|
||||
LEFT JOIN artists ar ON ar.id = t.artist_id
|
||||
LEFT JOIN albums al ON al.id = t.album_id
|
||||
WHERE t.file_path IS NOT NULL AND t.file_path != ''
|
||||
""")
|
||||
for row in cursor.fetchall():
|
||||
fp = (row[3] or '').replace('\\', '/')
|
||||
if not fp:
|
||||
continue
|
||||
parts = fp.split('/')
|
||||
meta = {
|
||||
'track_id': row[0],
|
||||
'title': row[1] or '',
|
||||
'artist': row[2] or '',
|
||||
'track_number': row[4],
|
||||
'album': row[5] or '',
|
||||
'album_thumb_url': row[6] or None,
|
||||
'artist_thumb_url': row[7] or None,
|
||||
}
|
||||
for depth in range(1, min(4, len(parts) + 1)):
|
||||
suffix = '/'.join(parts[-depth:]).lower()
|
||||
index.setdefault(suffix, meta)
|
||||
except Exception as e:
|
||||
logger.error("Error building DB suffix index: %s", e)
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
return index
|
||||
|
||||
def _match_db(self, fpath: str, db_index: dict):
|
||||
"""Match a walked file to a DB track via path suffix. Returns the track
|
||||
meta dict, or None when the file isn't part of the library."""
|
||||
parts = fpath.replace('\\', '/').split('/')
|
||||
for depth in range(min(3, len(parts)), 0, -1):
|
||||
suffix = '/'.join(parts[-depth:]).lower()
|
||||
hit = db_index.get(suffix)
|
||||
if hit:
|
||||
return hit
|
||||
return None
|
||||
|
||||
def _read_file_tags(self, fpath: str) -> dict:
|
||||
"""Read title/artist/album from the file's own tags (for loose files
|
||||
when library_tracks_only is off)."""
|
||||
meta = {'track_id': None}
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(fpath, easy=True)
|
||||
if audio:
|
||||
meta['title'] = (audio.get('title') or [None])[0] or ''
|
||||
meta['artist'] = (audio.get('artist') or audio.get('albumartist') or [None])[0] or ''
|
||||
meta['album'] = (audio.get('album') or [None])[0] or ''
|
||||
except Exception as e:
|
||||
logger.debug("tag read failed for %s: %s", os.path.basename(fpath), e)
|
||||
return meta
|
||||
|
||||
def estimate_scope(self, context: JobContext) -> int:
|
||||
count = 0
|
||||
for base in self._collect_music_dirs(context):
|
||||
for _root, _dirs, files in os.walk(base):
|
||||
for fname in files:
|
||||
if os.path.splitext(fname)[1].lower() in AUDIO_EXTENSIONS:
|
||||
count += 1
|
||||
return count
|
||||
267
core/repair_jobs/short_preview_track.py
Normal file
267
core/repair_jobs/short_preview_track.py
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
"""Repair job: detect ~30s PREVIEW clips and re-fetch the full track.
|
||||
|
||||
The HiFi endpoint (and occasionally others) sometimes deliver a ~30-second preview/sample
|
||||
instead of the full song. Those land in the library looking like real tracks. This job
|
||||
finds short tracks, looks up the EXPECTED length from the track's metadata source, and —
|
||||
when the source says the real track is meaningfully longer than the file — flags it as a
|
||||
preview clip.
|
||||
|
||||
Approving the finding (in repair_worker._fix_short_preview_track) deletes the preview file,
|
||||
drops the DB row so the track goes missing again, and re-adds it to the wishlist with a full
|
||||
payload so the real version gets downloaded. The scan itself ONLY creates findings — nothing
|
||||
is deleted, removed, or wishlisted without the user approving, exactly like the other tools.
|
||||
|
||||
Conservative by design (it deletes a file): a track is only flagged when the source confirms
|
||||
it should be much longer. Genuine short tracks (intros, skits, interludes — where the source
|
||||
agrees the track is short) are left alone, and tracks whose length can't be verified from a
|
||||
source are skipped, never flagged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from core.repair_jobs import register_job
|
||||
from core.repair_jobs.base import JobContext, JobResult, RepairJob
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("repair_jobs.short_preview")
|
||||
|
||||
|
||||
def _art_from_details(details: Dict[str, Any]) -> Optional[str]:
|
||||
"""Pull a renderable album-art URL out of a get_track_details() response. The cleaned dict
|
||||
doesn't carry images, but raw_data does: Spotify → raw_data.album.images[0].url, iTunes →
|
||||
raw_data.artworkUrl100 (upscaled). Returns None if neither is present."""
|
||||
raw = (details or {}).get("raw_data") or {}
|
||||
album = raw.get("album")
|
||||
if isinstance(album, dict):
|
||||
images = album.get("images")
|
||||
if isinstance(images, list) and images and isinstance(images[0], dict) and images[0].get("url"):
|
||||
return images[0]["url"]
|
||||
art = raw.get("artworkUrl100") or raw.get("artworkUrl60") or raw.get("artworkUrl30")
|
||||
if art:
|
||||
# iTunes serves tiny thumbnails by default; bump to a usable size.
|
||||
for small in ("100x100bb", "60x60bb", "30x30bb"):
|
||||
art = art.replace(small, "600x600bb")
|
||||
return art
|
||||
return None
|
||||
|
||||
|
||||
@register_job
|
||||
class ShortPreviewTrackJob(RepairJob):
|
||||
job_id = "short_preview_track"
|
||||
display_name = "Preview Clip Cleanup"
|
||||
description = (
|
||||
"Finds ~30s preview clips that slipped in instead of the full song (common from the "
|
||||
"HiFi endpoint) and re-fetches the real track."
|
||||
)
|
||||
help_text = (
|
||||
"Some downloads — especially via the HiFi source — deliver a ~30-second preview clip "
|
||||
"instead of the full song. They look like normal tracks in your library. This job scans "
|
||||
"for short tracks, checks how long the track ACTUALLY is from its metadata source "
|
||||
"(Spotify / iTunes / MusicBrainz), and flags any whose real length is much greater than "
|
||||
"the file — i.e. a preview.\n\n"
|
||||
"Approving a finding deletes the preview file, removes the track from the database (so it "
|
||||
"shows as missing), and re-adds it to your Wishlist so the full version downloads.\n\n"
|
||||
"It's conservative: genuine short tracks (intros, skits) where the source agrees the track "
|
||||
"is short are left alone, and tracks whose length can't be verified are skipped.\n\n"
|
||||
"Settings:\n"
|
||||
" - max_duration_seconds: only tracks at or below this length are considered (default 30).\n"
|
||||
" - min_expected_drift_seconds: the source must say the real track is at least this many "
|
||||
"seconds longer than the file before it's flagged (default 30)."
|
||||
)
|
||||
icon = "scissors"
|
||||
default_enabled = False
|
||||
default_interval_hours = 168 # weekly
|
||||
default_settings = {
|
||||
"max_duration_seconds": 30,
|
||||
"min_expected_drift_seconds": 30,
|
||||
}
|
||||
setting_options: Dict[str, list] = {}
|
||||
auto_fix = False
|
||||
|
||||
def _setting_int(self, context: JobContext, key: str, default: int) -> int:
|
||||
cm = getattr(context, "config_manager", None)
|
||||
if cm is None:
|
||||
return default
|
||||
try:
|
||||
return int(cm.get(self.get_config_key(key), default) or default)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
def scan(self, context: JobContext) -> JobResult:
|
||||
result = JobResult()
|
||||
max_dur_s = self._setting_int(context, "max_duration_seconds", 30)
|
||||
min_drift_s = self._setting_int(context, "min_expected_drift_seconds", 30)
|
||||
max_dur_ms = max_dur_s * 1000
|
||||
|
||||
conn = context.db._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT t.id, t.title, t.duration, t.file_path,
|
||||
t.spotify_track_id, t.itunes_track_id, t.musicbrainz_recording_id,
|
||||
ar.name AS artist_name, ar.thumb_url AS artist_thumb,
|
||||
al.title AS album_title, al.thumb_url AS album_thumb
|
||||
FROM tracks t
|
||||
LEFT JOIN artists ar ON ar.id = t.artist_id
|
||||
LEFT JOIN albums al ON al.id = t.album_id
|
||||
WHERE t.duration IS NOT NULL AND t.duration > 0 AND t.duration <= ?
|
||||
AND t.file_path IS NOT NULL AND t.file_path != ''
|
||||
""",
|
||||
(max_dur_ms,),
|
||||
)
|
||||
rows = [dict(r) for r in cursor.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
total = len(rows)
|
||||
if context.report_progress:
|
||||
try:
|
||||
context.report_progress(phase=f"Checking {total} short tracks for previews…", total=total)
|
||||
except Exception: # noqa: S110 — progress is best-effort
|
||||
pass
|
||||
|
||||
for i, row in enumerate(rows):
|
||||
if context.check_stop() or context.wait_if_paused():
|
||||
break
|
||||
result.scanned += 1
|
||||
|
||||
title = row["title"] or "Unknown"
|
||||
artist = row["artist_name"] or "Unknown"
|
||||
# Live progress EVERY track — the source lookup below is a network call, so without
|
||||
# per-track reporting the UI looks frozen at "Starting…" (the #937-follow-up report).
|
||||
if context.update_progress:
|
||||
try:
|
||||
context.update_progress(i + 1, total)
|
||||
except Exception: # noqa: S110 — best-effort
|
||||
pass
|
||||
if context.report_progress:
|
||||
try:
|
||||
context.report_progress(
|
||||
phase=f"Checking {i + 1}/{total} short tracks for previews…",
|
||||
log_line=f"{artist} — {title}", scanned=i + 1, total=total,
|
||||
)
|
||||
except Exception: # noqa: S110 — best-effort
|
||||
pass
|
||||
|
||||
file_dur_s = (row["duration"] or 0) / 1000.0
|
||||
source = self._lookup_source(context, row)
|
||||
|
||||
# Can't verify the real length → never flag (a delete must be backed by evidence).
|
||||
if source is None:
|
||||
result.skipped += 1
|
||||
continue
|
||||
expected_dur_s = source["duration_s"]
|
||||
# Prefer the source's album art (a renderable CDN url) over the library thumb, which
|
||||
# is often empty/non-renderable for un-enriched HiFi previews → art-less wishlist orb.
|
||||
album_image = source.get("album_image") or row["album_thumb"]
|
||||
|
||||
# Source agrees the track is short (genuine intro/skit) → leave it alone. Only a
|
||||
# source that says the real track is MUCH longer than the file marks a preview.
|
||||
if (expected_dur_s - file_dur_s) < min_drift_s:
|
||||
result.skipped += 1
|
||||
continue
|
||||
|
||||
if context.create_finding:
|
||||
title = row["title"] or "Unknown"
|
||||
artist = row["artist_name"] or "Unknown"
|
||||
try:
|
||||
inserted = context.create_finding(
|
||||
job_id=self.job_id,
|
||||
finding_type="short_preview_track",
|
||||
severity="warning",
|
||||
entity_type="track",
|
||||
entity_id=str(row["id"]),
|
||||
file_path=row["file_path"],
|
||||
title=f"Preview clip: {artist} - {title}",
|
||||
description=(
|
||||
f'File is {file_dur_s:.0f}s but "{title}" by {artist} is '
|
||||
f"{expected_dur_s:.0f}s at the source — looks like a preview clip. "
|
||||
"Approve to delete it and re-download the full version."
|
||||
),
|
||||
details={
|
||||
"track_id": row["id"],
|
||||
"title": row["title"],
|
||||
"artist": row["artist_name"],
|
||||
"album": row["album_title"],
|
||||
"album_thumb_url": album_image,
|
||||
"artist_thumb_url": row["artist_thumb"],
|
||||
"file_duration_s": round(file_dur_s, 1),
|
||||
"expected_duration_s": round(expected_dur_s, 1),
|
||||
"original_path": row["file_path"],
|
||||
},
|
||||
)
|
||||
if inserted:
|
||||
result.findings_created += 1
|
||||
else:
|
||||
result.findings_skipped_dedup += 1
|
||||
except Exception as exc:
|
||||
logger.debug("create_finding failed for track %s: %s", row["id"], exc)
|
||||
result.errors += 1
|
||||
|
||||
return result
|
||||
|
||||
def _lookup_source(self, context: JobContext, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Look the track up at its metadata source: returns {'duration_s', 'album_image'} or
|
||||
None when no source id is usable / the lookup fails. The SAME lookup that confirms the
|
||||
real length also carries the album art (in raw_data), which we capture so the re-wishlist
|
||||
isn't art-less when the library album thumb is missing (the #937-follow-up: HiFi previews
|
||||
on un-enriched albums). Every metadata client exposes get_track_details(id)."""
|
||||
|
||||
def _build(details) -> Optional[Dict[str, Any]]:
|
||||
ms = (details or {}).get("duration_ms")
|
||||
if not ms or ms <= 0:
|
||||
return None
|
||||
return {"duration_s": ms / 1000.0, "album_image": _art_from_details(details)}
|
||||
|
||||
# Spotify — pass allow_fallback=False. The default fallback scrapes the configured
|
||||
# metadata source, which is slow and can BLOCK a scan loop indefinitely when the
|
||||
# official API isn't authed (the #937-follow-up hang). Official-only is fast and
|
||||
# returns None cleanly when unavailable, so we just move to the next source.
|
||||
sp_id = row.get("spotify_track_id")
|
||||
if sp_id and context.spotify_client and not context.is_spotify_rate_limited():
|
||||
try:
|
||||
r = _build(context.spotify_client.get_track_details(str(sp_id), allow_fallback=False))
|
||||
if r:
|
||||
return r
|
||||
except TypeError:
|
||||
pass # older client without the flag — skip, don't risk the slow path
|
||||
except Exception as exc:
|
||||
logger.debug("spotify lookup failed for %s: %s", sp_id, exc)
|
||||
|
||||
# iTunes (public API, no auth, fast) then MusicBrainz.
|
||||
for source_id, client in (
|
||||
(row.get("itunes_track_id"), context.itunes_client),
|
||||
(row.get("musicbrainz_recording_id"), context.mb_client),
|
||||
):
|
||||
if not source_id or client is None:
|
||||
continue
|
||||
getter = getattr(client, "get_track_details", None)
|
||||
if getter is None:
|
||||
continue
|
||||
try:
|
||||
r = _build(getter(str(source_id)))
|
||||
if r:
|
||||
return r
|
||||
except Exception as exc:
|
||||
logger.debug("lookup failed for %s: %s", source_id, exc)
|
||||
return None
|
||||
|
||||
def estimate_scope(self, context: JobContext) -> int:
|
||||
try:
|
||||
max_dur_ms = self._setting_int(context, "max_duration_seconds", 30) * 1000
|
||||
conn = context.db._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT COUNT(*) FROM tracks WHERE duration > 0 AND duration <= ? "
|
||||
"AND file_path IS NOT NULL AND file_path != ''",
|
||||
(max_dur_ms,),
|
||||
)
|
||||
return (cursor.fetchone() or [0])[0]
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
return 0
|
||||
|
|
@ -994,9 +994,10 @@ class RepairWorker:
|
|||
'unwanted_content': self._fix_unwanted_content,
|
||||
'unknown_artist': self._fix_unknown_artist,
|
||||
'acoustid_mismatch': self._fix_acoustid_mismatch,
|
||||
'quality_upgrade': self._fix_quality_upgrade,
|
||||
'missing_discography_track': self._fix_discography_backfill,
|
||||
'library_retag': self._fix_library_retag,
|
||||
'quality_upgrade': self._fix_quality_upgrade,
|
||||
'short_preview_track': self._fix_short_preview_track,
|
||||
}
|
||||
handler = handlers.get(finding_type)
|
||||
if not handler:
|
||||
|
|
@ -1024,9 +1025,43 @@ class RepairWorker:
|
|||
return {'success': False, 'error': str(e)}
|
||||
|
||||
def _fix_quality_upgrade(self, entity_type, entity_id, file_path, details):
|
||||
"""Add the matched higher-quality version to the wishlist (with album
|
||||
context). Applying a Quality Upgrade finding is the user-approved step
|
||||
that the old auto-acting Quality Scanner did without review."""
|
||||
"""Apply a Quality Upgrade finding (user-approved; the old Quality
|
||||
Scanner did this without review). Action via ``details['_fix_action']``:
|
||||
|
||||
'redownload' (default): add the matched higher-quality version to the
|
||||
wishlist (with album context) for a profile-gated re-download.
|
||||
The low-quality file stays in place — it's replaced only after the
|
||||
better version actually imports (safe pattern; auto-delete-on-
|
||||
import is handled separately).
|
||||
'delete': remove the low-quality file + its DB row outright.
|
||||
'ignore' is handled in the UI by dismissing the finding — never here.
|
||||
"""
|
||||
fix_action = details.get('_fix_action', 'redownload')
|
||||
|
||||
if fix_action == 'delete':
|
||||
if file_path:
|
||||
resolved = _resolve_file_path(
|
||||
file_path, self.transfer_folder,
|
||||
config_manager=self._config_manager)
|
||||
if resolved and os.path.exists(resolved):
|
||||
try:
|
||||
os.remove(resolved)
|
||||
self._cleanup_empty_parents(resolved)
|
||||
except Exception as e:
|
||||
logger.warning("Could not delete low-quality file %s: %s",
|
||||
resolved, e)
|
||||
if entity_id:
|
||||
try:
|
||||
conn = self.db._get_connection()
|
||||
conn.cursor().execute("DELETE FROM tracks WHERE id = ?", (entity_id,))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': f'DB delete failed: {e}'}
|
||||
return {'success': True, 'action': 'deleted_file',
|
||||
'message': f'Deleted low-quality file: '
|
||||
f'{os.path.basename(file_path or "")}'}
|
||||
|
||||
track_data = details.get('matched_track_data')
|
||||
if not track_data:
|
||||
return {'success': False, 'error': 'No matched track in finding'}
|
||||
|
|
@ -1177,6 +1212,118 @@ class RepairWorker:
|
|||
if conn:
|
||||
conn.close()
|
||||
|
||||
def _fix_short_preview_track(self, entity_type, entity_id, file_path, details):
|
||||
"""Approve a preview-clip finding: delete the ~30s preview file, drop its DB row, and
|
||||
re-add the track to the wishlist (full payload) so the real version downloads. Mirrors
|
||||
the dead-file 'redownload' payload + the acoustid-mismatch file delete. (Tools #937-adj)
|
||||
"""
|
||||
if not entity_id:
|
||||
return {'success': False, 'error': 'No track ID associated with this finding'}
|
||||
conn = None
|
||||
try:
|
||||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT t.id, t.title, t.track_number, t.duration, t.bitrate,
|
||||
t.spotify_track_id, t.itunes_track_id, t.deezer_id, t.isrc,
|
||||
ar.name AS artist_name, ar.spotify_artist_id,
|
||||
al.title AS album_title, al.spotify_album_id,
|
||||
al.record_type, al.track_count, al.year, al.thumb_url AS album_thumb
|
||||
FROM tracks t
|
||||
LEFT JOIN artists ar ON ar.id = t.artist_id
|
||||
LEFT JOIN albums al ON al.id = t.album_id
|
||||
WHERE t.id = ?
|
||||
""", (entity_id,))
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return {'success': False, 'error': 'Track not found in database'}
|
||||
|
||||
track_name = row['title'] or details.get('title', 'Unknown')
|
||||
artist_name = row['artist_name'] or details.get('artist', 'Unknown Artist')
|
||||
album_title = row['album_title'] or details.get('album', '')
|
||||
|
||||
wishlist_id = (row['spotify_track_id']
|
||||
or row['itunes_track_id']
|
||||
or row['deezer_id']
|
||||
or f"preview_redl_{entity_id}")
|
||||
|
||||
# Prefer the finding's stored art (the scan captures the metadata source's CDN image)
|
||||
# over the library album thumb, which is often empty for un-enriched HiFi previews.
|
||||
album_images = []
|
||||
album_thumb = details.get('album_thumb_url') or row['album_thumb']
|
||||
if album_thumb:
|
||||
album_images = [{'url': album_thumb}]
|
||||
|
||||
spotify_track_data = {
|
||||
'id': wishlist_id,
|
||||
'name': track_name,
|
||||
'artists': [{'name': artist_name}],
|
||||
'album': {
|
||||
'name': album_title or track_name,
|
||||
'id': row['spotify_album_id'] or '',
|
||||
'release_date': str(row['year']) if row['year'] else '',
|
||||
'images': album_images,
|
||||
'album_type': row['record_type'] or 'album',
|
||||
'total_tracks': row['track_count'] or 0,
|
||||
'artists': [{'name': artist_name}],
|
||||
},
|
||||
'duration_ms': int((details.get('expected_duration_s') or 0) * 1000) or (row['duration'] or 0),
|
||||
'track_number': row['track_number'] or 1,
|
||||
'disc_number': 1,
|
||||
'explicit': False,
|
||||
'external_urls': {},
|
||||
'popularity': 0,
|
||||
'preview_url': None,
|
||||
'uri': f"spotify:track:{row['spotify_track_id']}" if row['spotify_track_id'] else '',
|
||||
'is_local': False,
|
||||
}
|
||||
|
||||
source_info = {
|
||||
'original_path': file_path or details.get('original_path', ''),
|
||||
'album_title': album_title,
|
||||
'artist': artist_name,
|
||||
'reason': 'preview_clip_redownload',
|
||||
}
|
||||
|
||||
added = self.db.add_to_wishlist(
|
||||
spotify_track_data,
|
||||
failure_reason='Preview clip — re-downloading full track',
|
||||
source_type='redownload',
|
||||
source_info=source_info,
|
||||
)
|
||||
if not added:
|
||||
return {'success': False, 'error': 'Failed to add to wishlist (may already exist or be blocklisted)'}
|
||||
|
||||
# Delete the preview file (path resolved like the other delete tools).
|
||||
deleted_file = False
|
||||
target_path = file_path or details.get('original_path')
|
||||
if target_path:
|
||||
download_folder = self._config_manager.get('soulseek.download_path', '') if self._config_manager else None
|
||||
resolved = _resolve_file_path(target_path, self.transfer_folder,
|
||||
download_folder=download_folder,
|
||||
config_manager=self._config_manager)
|
||||
if resolved and os.path.exists(resolved):
|
||||
try:
|
||||
os.remove(resolved)
|
||||
deleted_file = True
|
||||
except Exception as e:
|
||||
logger.warning("Could not delete preview file %s: %s", resolved, e)
|
||||
|
||||
# Drop the DB row so the track shows as missing.
|
||||
cursor.execute("DELETE FROM tracks WHERE id = ?", (entity_id,))
|
||||
conn.commit()
|
||||
|
||||
return {'success': True, 'action': 'added_to_wishlist',
|
||||
'message': (f'Deleted preview clip and re-wishlisted "{track_name}" for full download'
|
||||
if deleted_file else
|
||||
f'Re-wishlisted "{track_name}" (preview file already gone)')}
|
||||
except Exception as e:
|
||||
logger.error("Preview-clip fix failed for track %s: %s", entity_id, e)
|
||||
return {'success': False, 'error': str(e)}
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
def _fix_orphan_file(self, entity_type, entity_id, file_path, details):
|
||||
"""Handle an orphan file — move to staging or delete based on user choice.
|
||||
|
||||
|
|
@ -3226,6 +3373,14 @@ class RepairWorker:
|
|||
return {'success': False, 'error': f'Source file not found: {file_path}'}
|
||||
|
||||
out_path = os.path.splitext(resolved)[0] + out_ext
|
||||
# Safety invariant: ffmpeg runs with -y, so refuse to convert a file onto
|
||||
# itself (an .m4a ALAC source + AAC target shares the .m4a path) — that
|
||||
# would destroy the original lossless file (#941).
|
||||
from core.quality.lossless import lossy_output_would_overwrite_source
|
||||
if lossy_output_would_overwrite_source(resolved, out_path):
|
||||
return {'success': False,
|
||||
'error': f'{codec.upper()} output would overwrite the source file; '
|
||||
f'choose a different lossy codec'}
|
||||
if os.path.exists(out_path):
|
||||
return {'success': True, 'action': 'already_exists',
|
||||
'message': f'{quality_label} copy already exists'}
|
||||
|
|
@ -3384,7 +3539,8 @@ class RepairWorker:
|
|||
'album_tag_inconsistency',
|
||||
'incomplete_album', 'path_mismatch',
|
||||
'missing_lossy_copy', 'missing_replaygain', 'empty_folder',
|
||||
'missing_discography_track', 'acoustid_mismatch')
|
||||
'missing_discography_track', 'acoustid_mismatch',
|
||||
'quality_upgrade', 'short_preview_track')
|
||||
placeholders = ','.join(['?'] * len(fixable_types))
|
||||
where_parts = [f"finding_type IN ({placeholders})", "status = 'pending'"]
|
||||
params = list(fixable_types)
|
||||
|
|
|
|||
|
|
@ -66,6 +66,39 @@ def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True):
|
|||
return activity_item
|
||||
|
||||
|
||||
def claim_for_post_processing(task_id: str) -> bool:
|
||||
"""Atomically claim a download task for post-processing.
|
||||
|
||||
The browser-poll status endpoint AND the background download monitor both
|
||||
watch the same slskd/streaming transfers and each tries to post-process a
|
||||
completed file. Without a single claim, BOTH run the verification pipeline
|
||||
on the same download — double imports, and a nasty race where one path
|
||||
quarantines + requeues the next-best candidate (clearing the source identity
|
||||
and resetting status to ``searching``) while the other, mid-flight, then
|
||||
reports a bogus "missing file or source information" failure that clobbers
|
||||
the in-flight retry.
|
||||
|
||||
The claim is the ``downloading``/``queued`` -> ``post_processing`` status
|
||||
transition, done under ``tasks_lock``. Exactly one caller wins:
|
||||
|
||||
- Returns ``True`` (and flips the status) for the caller that claimed it.
|
||||
- Returns ``False`` if the task is gone, already ``post_processing`` (owned
|
||||
by the other path), requeued (``searching``), or terminal — the caller
|
||||
must then NOT process the file.
|
||||
|
||||
Acquires ``tasks_lock`` itself, so callers must NOT already hold it.
|
||||
"""
|
||||
with tasks_lock:
|
||||
task = download_tasks.get(task_id)
|
||||
if not task:
|
||||
return False
|
||||
if task.get("status") in ("downloading", "queued"):
|
||||
task["status"] = "post_processing"
|
||||
task["status_change_time"] = time.time()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@caller_must_hold_tasks_lock
|
||||
def mark_task_completed(task_id: str, track_info: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""Mark a download task as completed.
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ from core.download_plugins.album_bundle import (
|
|||
get_poll_timeout,
|
||||
)
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
from core.quality.model import QualityTarget, filter_and_rank, v2_qualities_to_ranked_targets
|
||||
from core.quality.source_map import AUDIO_EXTENSIONS, format_from_extension
|
||||
from utils.async_helpers import run_async
|
||||
|
||||
logger = get_logger("soulseek_client")
|
||||
|
|
@ -409,7 +411,7 @@ class SoulseekClient(DownloadSourcePlugin):
|
|||
|
||||
|
||||
# Audio file extensions to filter for
|
||||
audio_extensions = {'.mp3', '.flac', '.ogg', '.aac', '.wma', '.wav', '.m4a'}
|
||||
audio_extensions = AUDIO_EXTENSIONS
|
||||
|
||||
for response_data in responses_data:
|
||||
username = response_data.get('username', '')
|
||||
|
|
@ -426,26 +428,28 @@ class SoulseekClient(DownloadSourcePlugin):
|
|||
if f'.{file_ext}' not in audio_extensions:
|
||||
continue
|
||||
|
||||
# .m4a is the usual AAC container — bucket it as 'aac' (the
|
||||
# quality filter treats AAC as an opt-in tier; off by default).
|
||||
quality = 'aac' if file_ext == 'm4a' else (
|
||||
file_ext if file_ext in ['flac', 'mp3', 'ogg', 'aac', 'wma'] else 'unknown')
|
||||
# Source-agnostic extension → format (shared with every other
|
||||
# extension-based source). Ranked targets do the rest.
|
||||
quality = format_from_extension(file_ext)
|
||||
|
||||
# Create TrackResult
|
||||
# Convert duration from seconds to milliseconds (slskd returns seconds, Spotify uses ms)
|
||||
raw_duration = file_data.get('length')
|
||||
duration_ms = raw_duration * 1000 if raw_duration else None
|
||||
|
||||
slskd_attrs = {a['type']: a['value'] for a in file_data.get('attributes', [])}
|
||||
track = TrackResult(
|
||||
username=username,
|
||||
filename=filename,
|
||||
size=size,
|
||||
bitrate=file_data.get('bitRate'),
|
||||
bitrate=file_data.get('bitRate') or slskd_attrs.get(0),
|
||||
duration=duration_ms,
|
||||
quality=quality,
|
||||
free_upload_slots=response_data.get('freeUploadSlots', 0),
|
||||
upload_speed=response_data.get('uploadSpeed', 0),
|
||||
queue_length=response_data.get('queueLength', 0)
|
||||
queue_length=response_data.get('queueLength', 0),
|
||||
sample_rate=slskd_attrs.get(4),
|
||||
bit_depth=slskd_attrs.get(5),
|
||||
)
|
||||
|
||||
all_tracks.append(track)
|
||||
|
|
@ -1136,7 +1140,7 @@ class SoulseekClient(DownloadSourcePlugin):
|
|||
Returns:
|
||||
List of TrackResult objects for audio files
|
||||
"""
|
||||
audio_extensions = {'.mp3', '.flac', '.ogg', '.aac', '.wma', '.wav', '.m4a'}
|
||||
audio_extensions = AUDIO_EXTENSIONS
|
||||
results = []
|
||||
if files:
|
||||
logger.debug(f"Browse raw file sample: {files[0]}")
|
||||
|
|
@ -1150,15 +1154,17 @@ class SoulseekClient(DownloadSourcePlugin):
|
|||
ext = Path(filename).suffix.lower()
|
||||
if ext not in audio_extensions:
|
||||
continue
|
||||
_qext = ext.lstrip('.')
|
||||
quality = 'aac' if _qext == 'm4a' else (
|
||||
_qext if _qext in ['flac', 'mp3', 'ogg', 'aac', 'wma'] else 'unknown')
|
||||
quality = format_from_extension(ext)
|
||||
raw_duration = file_data.get('length')
|
||||
duration_ms = raw_duration * 1000 if raw_duration else None
|
||||
slskd_attrs = {a['type']: a['value'] for a in file_data.get('attributes', [])}
|
||||
results.append(TrackResult(
|
||||
username=username, filename=filename, size=file_data.get('size', 0),
|
||||
bitrate=file_data.get('bitRate'), duration=duration_ms, quality=quality,
|
||||
free_upload_slots=free_slots, upload_speed=upload_speed, queue_length=queue_length
|
||||
bitrate=file_data.get('bitRate') or slskd_attrs.get(0),
|
||||
duration=duration_ms, quality=quality,
|
||||
free_upload_slots=free_slots, upload_speed=upload_speed, queue_length=queue_length,
|
||||
sample_rate=slskd_attrs.get(4),
|
||||
bit_depth=slskd_attrs.get(5),
|
||||
))
|
||||
return results
|
||||
|
||||
|
|
@ -2008,189 +2014,57 @@ class SoulseekClient(DownloadSourcePlugin):
|
|||
return kept
|
||||
|
||||
def filter_results_by_quality_preference(self, results: List[TrackResult]) -> List[TrackResult]:
|
||||
"""
|
||||
Filter candidates based on user's quality profile with bitrate density constraints.
|
||||
Uses priority waterfall logic: tries highest priority quality first, falls back to lower priorities.
|
||||
Returns candidates matching quality profile constraints, sorted by confidence and effective bitrate.
|
||||
"""Filter and rank candidates using the global quality target list.
|
||||
|
||||
Issue #652: also drops candidates whose `(username, filename)`
|
||||
matches a previously-quarantined download. Without this pre-filter
|
||||
the auto-wishlist processor's ranking is deterministic — the same
|
||||
`(uploader, file)` keeps winning the quality picker, downloading,
|
||||
failing AcoustID, quarantining, and re-queueing in an infinite
|
||||
loop. Users wake up to hundreds of duplicate `.quarantined` files
|
||||
for the same source URL.
|
||||
Replaces the old bucket+heuristic approach with ``core.quality.model``
|
||||
so every download source shares the same ranking logic.
|
||||
|
||||
Issue #652: also drops candidates whose ``(username, filename)``
|
||||
matches a previously-quarantined download to break infinite retry loops.
|
||||
"""
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
if not results:
|
||||
return []
|
||||
|
||||
# Drop sources already quarantined — bypass the quality picker
|
||||
# entirely so the same bad upload doesn't get re-selected on the
|
||||
# next wishlist cycle. Filesystem read is bounded (~few hundred
|
||||
# sidecars in practical use × <1ms each).
|
||||
# Issue #652: drop candidates on the quarantine record BEFORE ranking,
|
||||
# so a previously-quarantined source can't win the quality picker by
|
||||
# superior bitrate and re-trigger the same failed download in a loop.
|
||||
results = self._drop_quarantined_sources(results)
|
||||
if not results:
|
||||
return []
|
||||
|
||||
# Get quality profile from database
|
||||
db = MusicDatabase()
|
||||
profile = db.get_quality_profile()
|
||||
|
||||
logger.debug(f"Quality Filter: Using profile preset '{profile.get('preset', 'custom')}', filtering {len(results)} candidates")
|
||||
# Build ranked target list — v3 profiles carry it directly;
|
||||
# v2 profiles are converted on the fly (no DB write needed here).
|
||||
raw_targets = profile.get('ranked_targets')
|
||||
if not raw_targets and 'qualities' in profile:
|
||||
raw_targets = v2_qualities_to_ranked_targets(profile['qualities'])
|
||||
|
||||
# Categorize candidates by quality with bitrate density constraints
|
||||
quality_buckets = {
|
||||
'flac': [],
|
||||
'mp3_320': [],
|
||||
'mp3_256': [],
|
||||
'mp3_192': [],
|
||||
'aac': [],
|
||||
'other': []
|
||||
}
|
||||
targets = [QualityTarget.from_dict(t) for t in (raw_targets or [])]
|
||||
fallback_enabled = profile.get('fallback_enabled', True)
|
||||
|
||||
# Track all candidates that pass checks (for fallback)
|
||||
density_filtered_all = []
|
||||
# Every format (AAC included) follows the SAME universal rule: a
|
||||
# candidate passes only if it matches a ranked target; if nothing
|
||||
# matches, the fallback toggle decides. No per-format special-casing.
|
||||
|
||||
for candidate in results:
|
||||
if not candidate.quality:
|
||||
quality_buckets['other'].append(candidate)
|
||||
continue
|
||||
logger.debug(
|
||||
"Quality Filter: profile='%s', %d targets, %d candidates",
|
||||
profile.get('preset', 'custom'), len(targets), len(results),
|
||||
)
|
||||
|
||||
track_format = candidate.quality.lower()
|
||||
track_bitrate = candidate.bitrate or 0
|
||||
ranked = filter_and_rank(results, targets, fallback_enabled=fallback_enabled)
|
||||
|
||||
# Determine quality key
|
||||
if track_format == 'flac':
|
||||
quality_key = 'flac'
|
||||
elif track_format == 'mp3':
|
||||
if track_bitrate >= 320:
|
||||
quality_key = 'mp3_320'
|
||||
elif track_bitrate >= 256:
|
||||
quality_key = 'mp3_256'
|
||||
elif track_bitrate >= 192:
|
||||
quality_key = 'mp3_192'
|
||||
else:
|
||||
quality_buckets['other'].append(candidate)
|
||||
continue
|
||||
elif track_format in ('aac', 'm4a'):
|
||||
# Opt-in AAC tier. ADDITIVE: when AAC isn't enabled in the
|
||||
# profile (the default, and every profile that predates this),
|
||||
# route exactly where AAC went before — the 'other' bucket — so
|
||||
# behaviour is byte-identical. Only a user who turns AAC on lets
|
||||
# it become a first-class, selectable tier.
|
||||
aac_cfg = profile['qualities'].get('aac')
|
||||
if not (aac_cfg and aac_cfg.get('enabled')):
|
||||
quality_buckets['other'].append(candidate)
|
||||
continue
|
||||
quality_key = 'aac'
|
||||
else:
|
||||
quality_buckets['other'].append(candidate)
|
||||
continue
|
||||
|
||||
quality_config = profile['qualities'].get(quality_key, {})
|
||||
min_kbps = quality_config.get('min_kbps', 0)
|
||||
max_kbps = quality_config.get('max_kbps', 99999)
|
||||
|
||||
effective_kbps = self._calculate_effective_kbps(candidate.size, candidate.duration)
|
||||
|
||||
if effective_kbps is not None:
|
||||
# Primary: bitrate density check
|
||||
if min_kbps <= effective_kbps <= max_kbps:
|
||||
if quality_config.get('enabled', False):
|
||||
quality_buckets[quality_key].append(candidate)
|
||||
density_filtered_all.append(candidate)
|
||||
else:
|
||||
logger.debug(f"Quality Filter: {quality_key} rejected - {effective_kbps:.0f} kbps outside {min_kbps}-{max_kbps} kbps range")
|
||||
else:
|
||||
# Fallback: duration unavailable, use generous raw-size sanity check
|
||||
file_size_mb = candidate.size / (1024 * 1024)
|
||||
size_min, size_max = self._FALLBACK_SIZE_LIMITS.get(quality_key, (0, 500))
|
||||
if size_min <= file_size_mb <= size_max:
|
||||
if quality_config.get('enabled', False):
|
||||
quality_buckets[quality_key].append(candidate)
|
||||
density_filtered_all.append(candidate)
|
||||
logger.debug(f"Quality Filter: {quality_key} accepted via size fallback ({file_size_mb:.1f} MB, no duration available)")
|
||||
else:
|
||||
logger.debug(f"Quality Filter: {quality_key} rejected via size fallback - {file_size_mb:.1f} MB outside {size_min}-{size_max} MB safety limits")
|
||||
|
||||
# Sort each bucket: effective bitrate first (prefer highest audio quality),
|
||||
# then peer quality score as tiebreaker (prefer fastest peer at same quality)
|
||||
for bucket in quality_buckets.values():
|
||||
bucket.sort(key=lambda x: (self._calculate_effective_kbps(x.size, x.duration) or 0, x.quality_score), reverse=True)
|
||||
|
||||
# Enforce FLAC bit depth preference from quality profile
|
||||
flac_config = profile['qualities'].get('flac', {})
|
||||
bit_depth_pref = flac_config.get('bit_depth', 'any')
|
||||
bit_depth_fallback = flac_config.get('bit_depth_fallback', True)
|
||||
|
||||
if bit_depth_pref != 'any' and quality_buckets['flac']:
|
||||
# 16-bit/44.1kHz FLAC theoretical max is 1411 kbps; 24-bit starts at ~2116 kbps
|
||||
# Real-world compressed: 16-bit = 800-1400 kbps, 24-bit = 1500+ kbps
|
||||
DEPTH_THRESHOLD = 1450
|
||||
|
||||
if bit_depth_pref == '24':
|
||||
hi_res = [c for c in quality_buckets['flac']
|
||||
if (self._calculate_effective_kbps(c.size, c.duration) or 0) > DEPTH_THRESHOLD]
|
||||
if hi_res:
|
||||
logger.info(f"Quality Filter: Bit depth 24-bit preference — {len(hi_res)}/{len(quality_buckets['flac'])} FLAC candidates are hi-res")
|
||||
quality_buckets['flac'] = hi_res
|
||||
elif not bit_depth_fallback:
|
||||
logger.info("Quality Filter: No 24-bit FLAC found and fallback disabled — rejecting all FLAC")
|
||||
quality_buckets['flac'] = []
|
||||
else:
|
||||
logger.info("Quality Filter: No 24-bit FLAC found — falling back to 16-bit")
|
||||
|
||||
elif bit_depth_pref == '16':
|
||||
lo_res = [c for c in quality_buckets['flac']
|
||||
if (self._calculate_effective_kbps(c.size, c.duration) or 0) <= DEPTH_THRESHOLD]
|
||||
if lo_res:
|
||||
logger.info(f"Quality Filter: Bit depth 16-bit preference — {len(lo_res)}/{len(quality_buckets['flac'])} FLAC candidates are standard")
|
||||
quality_buckets['flac'] = lo_res
|
||||
elif not bit_depth_fallback:
|
||||
logger.info("Quality Filter: No 16-bit FLAC found and fallback disabled — rejecting all FLAC")
|
||||
quality_buckets['flac'] = []
|
||||
else:
|
||||
logger.info("Quality Filter: No 16-bit FLAC found — falling back to 24-bit")
|
||||
|
||||
# Debug logging
|
||||
for quality, bucket in quality_buckets.items():
|
||||
if bucket:
|
||||
logger.debug(f"Quality Filter: Found {len(bucket)} '{quality}' candidates (after bitrate + bit depth filtering)")
|
||||
|
||||
# Waterfall priority logic: try qualities in priority order
|
||||
# Build priority list from enabled qualities
|
||||
quality_priorities = []
|
||||
for quality_name, quality_config in profile['qualities'].items():
|
||||
if quality_config.get('enabled', False):
|
||||
priority = quality_config.get('priority', 999)
|
||||
quality_priorities.append((priority, quality_name))
|
||||
|
||||
# Sort by priority (lower number = higher priority)
|
||||
quality_priorities.sort()
|
||||
|
||||
# Try each quality in priority order
|
||||
for priority, quality_name in quality_priorities:
|
||||
candidates_for_quality = quality_buckets.get(quality_name, [])
|
||||
if candidates_for_quality:
|
||||
logger.info(f"Quality Filter: Returning {len(candidates_for_quality)} '{quality_name}' candidates (priority {priority})")
|
||||
return candidates_for_quality
|
||||
|
||||
# If no enabled qualities matched, check if fallback is enabled
|
||||
if profile.get('fallback_enabled', True):
|
||||
logger.warning("Quality Filter: No enabled qualities matched, falling back to density-filtered candidates")
|
||||
if density_filtered_all:
|
||||
density_filtered_all.sort(key=lambda x: (x.quality_score, self._calculate_effective_kbps(x.size, x.duration) or 0), reverse=True)
|
||||
logger.info(f"Quality Filter: Returning {len(density_filtered_all)} fallback candidates (bitrate-filtered, any quality)")
|
||||
return density_filtered_all
|
||||
else:
|
||||
logger.warning("Quality Filter: All candidates failed bitrate checks, returning empty (respecting constraints)")
|
||||
return []
|
||||
if ranked:
|
||||
best_label = ranked[0].audio_quality.label()
|
||||
logger.info("Quality Filter: returning %d candidate(s), best=%s", len(ranked), best_label)
|
||||
else:
|
||||
logger.warning("Quality Filter: No enabled qualities matched and fallback is disabled, returning empty")
|
||||
return []
|
||||
|
||||
logger.warning("Quality Filter: no candidates passed quality constraints")
|
||||
|
||||
return ranked
|
||||
|
||||
async def get_session_info(self) -> Optional[Dict[str, Any]]:
|
||||
"""Get slskd session information including version"""
|
||||
if not self.base_url:
|
||||
|
|
|
|||
|
|
@ -128,7 +128,14 @@ class SoundcloudClient(DownloadSourcePlugin):
|
|||
download_path = config_manager.get('soulseek.download_path', './downloads')
|
||||
|
||||
self.download_path = Path(download_path)
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
# Don't crash construction if the path isn't creatable yet (e.g. an
|
||||
# unmounted/misconfigured volume) — the registry would null the whole
|
||||
# client and the source vanishes. Warn and continue, same as
|
||||
# SoulseekClient; the dir is (re)created lazily at download time.
|
||||
try:
|
||||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
logger.warning(f"Could not verify SoundCloud download path {self.download_path}: {e}")
|
||||
|
||||
logger.info(f"SoundCloud client using download path: {self.download_path}")
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,61 @@ from core.metadata.cache import get_metadata_cache
|
|||
|
||||
logger = get_logger("spotify_client")
|
||||
|
||||
|
||||
# Single source of truth for the Spotify OAuth scope. Used by EVERY SpotifyOAuth
|
||||
# construction (the client, the per-profile registry, and all web_server callbacks) so
|
||||
# the authorize URL and token exchange can never request different scopes — a mismatch
|
||||
# silently re-prompts or denies.
|
||||
#
|
||||
# IMPORTANT — do NOT add scopes here lightly. Spotipy's validate_token treats a cached
|
||||
# token as invalid the moment the requested scope is no longer a subset of the token's
|
||||
# granted scope, so GROWING this string invalidates EVERY existing user's token and forces
|
||||
# a re-auth on upgrade. `playlist-modify-*` (for exporting a playlist back to Spotify, #945)
|
||||
# was pulled back out for exactly that reason — it broke all Spotify users on upgrade. The
|
||||
# Spotify export must request write access on-demand (incremental auth) instead.
|
||||
SPOTIFY_OAUTH_SCOPE = (
|
||||
"user-library-read user-read-private playlist-read-private "
|
||||
"playlist-read-collaborative user-read-email user-follow-read"
|
||||
)
|
||||
|
||||
# The export scope = the normal login scope PLUS playlist write. Requested ONLY by the
|
||||
# on-demand export-auth route (/auth/spotify/export) when a user chooses to export a playlist
|
||||
# to Spotify — NEVER by the normal login. That's the whole safety property: the global scope
|
||||
# above is unchanged, so no existing token is invalidated. The token Spotify returns from the
|
||||
# export flow is a SUPERSET of the read scope, so it still passes the normal auth check
|
||||
# (read ⊆ read+write) — one account, one token, just with write added for the opt-in user.
|
||||
SPOTIFY_EXPORT_SCOPE = (
|
||||
SPOTIFY_OAUTH_SCOPE + " playlist-modify-public playlist-modify-private"
|
||||
)
|
||||
|
||||
|
||||
def normalize_spotify_oauth_config(config: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""Normalize Spotify OAuth config before building an auth manager.
|
||||
|
||||
Spotify rejects values that include surrounding whitespace or quotes, and the
|
||||
settings UI can paste values carrying such formatting, so we trim those — they
|
||||
can never be part of a real credential.
|
||||
|
||||
We deliberately do NOT strip a trailing slash from ``redirect_uri``: Spotify
|
||||
matches the redirect URI EXACTLY against the app's dashboard registration, and
|
||||
a trailing slash is a legitimate part of a URI. Stripping it would silently
|
||||
break anyone who registered ``…/callback/`` (we'd send ``…/callback`` →
|
||||
"INVALID_CLIENT: Invalid redirect URI"). So the value is preserved verbatim
|
||||
apart from the unambiguous whitespace/quote garbage (#942 follow-up).
|
||||
"""
|
||||
if not isinstance(config, dict):
|
||||
return {}
|
||||
|
||||
normalized = {}
|
||||
for key in ("client_id", "client_secret", "redirect_uri"):
|
||||
value = config.get(key, "")
|
||||
if isinstance(value, str):
|
||||
normalized[key] = value.strip().strip('"').strip("'")
|
||||
else:
|
||||
normalized[key] = value
|
||||
return normalized
|
||||
|
||||
|
||||
def _upgrade_spotify_image_url(url: str) -> str:
|
||||
"""Upgrade a Spotify CDN image URL to the highest available resolution.
|
||||
|
||||
|
|
@ -697,7 +752,7 @@ class SpotifyClient:
|
|||
self._setup_client()
|
||||
|
||||
def _setup_client(self):
|
||||
config = config_manager.get_spotify_config()
|
||||
config = normalize_spotify_oauth_config(config_manager.get_spotify_config())
|
||||
|
||||
if not config.get('client_id') or not config.get('client_secret'):
|
||||
logger.warning("Spotify credentials not configured")
|
||||
|
|
@ -716,7 +771,7 @@ class SpotifyClient:
|
|||
client_id=config['client_id'],
|
||||
client_secret=config['client_secret'],
|
||||
redirect_uri=config.get('redirect_uri', "http://127.0.0.1:8888/callback"),
|
||||
scope="user-library-read user-read-private playlist-read-private playlist-read-collaborative user-read-email user-follow-read",
|
||||
scope=SPOTIFY_OAUTH_SCOPE,
|
||||
cache_handler=DatabaseTokenCache(config_manager)
|
||||
)
|
||||
|
||||
|
|
@ -751,6 +806,16 @@ class SpotifyClient:
|
|||
self._auth_cached_result = None
|
||||
self._auth_cache_time = 0
|
||||
|
||||
def _has_cached_oauth_token(self) -> bool:
|
||||
"""Return True when a persisted OAuth token exists (no network I/O)."""
|
||||
if self.sp is None:
|
||||
return False
|
||||
try:
|
||||
cache_handler = getattr(self.sp.auth_manager, 'cache_handler', None)
|
||||
return bool(cache_handler and cache_handler.get_cached_token() is not None)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def is_spotify_authenticated(self) -> bool:
|
||||
"""Check if Spotify client is specifically authenticated (not just iTunes fallback).
|
||||
Results are cached for 60 seconds to avoid excessive API calls.
|
||||
|
|
@ -826,6 +891,26 @@ class SpotifyClient:
|
|||
logger.debug("publish_spotify_status cache hit: %s", e)
|
||||
return self._auth_cached_result
|
||||
|
||||
from core.boot_phase import is_boot_phase
|
||||
if is_boot_phase():
|
||||
result = self._has_cached_oauth_token()
|
||||
with self._auth_cache_lock:
|
||||
self._auth_cached_result = result
|
||||
self._auth_cache_time = time.time()
|
||||
try:
|
||||
from core.metadata.status import publish_spotify_status
|
||||
|
||||
publish_spotify_status(
|
||||
connected=result,
|
||||
authenticated=result,
|
||||
rate_limited=False,
|
||||
rate_limit=None,
|
||||
post_ban_cooldown=None,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("publish_spotify_status boot-phase: %s", e)
|
||||
return result
|
||||
|
||||
# Cache miss — make API call outside the lock.
|
||||
# Safety: if there's no cached token, return False immediately.
|
||||
# Without this guard, spotipy's auth_manager will try to start an interactive
|
||||
|
|
@ -857,7 +942,8 @@ class SpotifyClient:
|
|||
# Use a dedicated probe client (retries=0) so a 429 here propagates
|
||||
# immediately and we can detect long Retry-After bans.
|
||||
try:
|
||||
probe = spotipy.Spotify(auth_manager=self.sp.auth_manager, retries=0)
|
||||
probe = spotipy.Spotify(
|
||||
auth_manager=self.sp.auth_manager, retries=0, requests_timeout=15)
|
||||
probe.current_user()
|
||||
result = True
|
||||
except Exception as e:
|
||||
|
|
@ -1108,6 +1194,69 @@ class SpotifyClient:
|
|||
return playlists
|
||||
return []
|
||||
|
||||
def has_write_scope(self) -> bool:
|
||||
"""True when the cached Spotify token carries playlist-modify (the export write scope).
|
||||
The export endpoint uses this to decide whether to run, or to first send the user
|
||||
through the on-demand export-auth flow. Fail-safe: any error → False (not authorized)."""
|
||||
if self.sp is None:
|
||||
return False
|
||||
try:
|
||||
cache_handler = getattr(self.sp.auth_manager, "cache_handler", None)
|
||||
token = cache_handler.get_cached_token() if cache_handler else None
|
||||
return "playlist-modify" in ((token or {}).get("scope") or "")
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def create_or_update_playlist(self, name, track_ids, *, existing_id=None,
|
||||
public=False, description=""):
|
||||
"""Create a Spotify playlist owned by the authed user (or replace an existing
|
||||
one's tracks in place), for exporting a mirrored playlist back to Spotify (#945).
|
||||
|
||||
``track_ids`` are Spotify track IDs (the stored ``spotify_track_id`` per library
|
||||
track). ``existing_id`` set → replace that playlist's contents (idempotent
|
||||
re-export); unset → create a new playlist. Requires the ``playlist-modify-*``
|
||||
scope — a token issued before that scope was added gets a clear "reconnect"
|
||||
error rather than a raw 403. Returns
|
||||
``{success, playlist_id, url, added, error}``.
|
||||
"""
|
||||
if not self.is_spotify_authenticated():
|
||||
return {"success": False, "error": "Spotify is not connected"}
|
||||
uris = [f"spotify:track:{t}" for t in (track_ids or []) if t]
|
||||
if not uris:
|
||||
return {"success": False, "error": "No matching Spotify tracks to export"}
|
||||
try:
|
||||
playlist_id = existing_id
|
||||
if playlist_id:
|
||||
# Replace contents (re-export). replace_items caps at 100 — seed with the
|
||||
# first 100, then append the rest in 100-track chunks.
|
||||
self.sp.playlist_replace_items(playlist_id, uris[:100])
|
||||
for i in range(100, len(uris), 100):
|
||||
self.sp.playlist_add_items(playlist_id, uris[i:i + 100])
|
||||
else:
|
||||
user_id = (self.sp.current_user() or {}).get("id")
|
||||
created = self.sp.user_playlist_create(
|
||||
user_id, name, public=public, description=description,
|
||||
)
|
||||
playlist_id = (created or {}).get("id")
|
||||
if not playlist_id:
|
||||
return {"success": False, "error": "Spotify did not return a playlist id"}
|
||||
for i in range(0, len(uris), 100):
|
||||
self.sp.playlist_add_items(playlist_id, uris[i:i + 100])
|
||||
return {
|
||||
"success": True,
|
||||
"playlist_id": playlist_id,
|
||||
"url": f"https://open.spotify.com/playlist/{playlist_id}",
|
||||
"added": len(uris),
|
||||
}
|
||||
except Exception as e:
|
||||
msg = str(e)
|
||||
if "403" in msg or "scope" in msg.lower() or "insufficient" in msg.lower():
|
||||
return {"success": False,
|
||||
"error": "Reconnect Spotify to grant playlist write access "
|
||||
"(Settings → reconnect Spotify)."}
|
||||
_detect_and_set_rate_limit(e, "create_or_update_playlist")
|
||||
return {"success": False, "error": msg}
|
||||
|
||||
@rate_limited
|
||||
def get_saved_tracks_count(self) -> int:
|
||||
"""Get the total count of user's saved/liked songs without fetching all tracks"""
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue