commit
f4dbaea68b
125 changed files with 10997 additions and 463 deletions
4
.github/workflows/docker-publish.yml
vendored
4
.github/workflows/docker-publish.yml
vendored
|
|
@ -9,9 +9,9 @@ on:
|
|||
workflow_dispatch:
|
||||
inputs:
|
||||
version_tag:
|
||||
description: 'Version tag (e.g. 2.6.7)'
|
||||
description: 'Version tag (e.g. 2.6.8)'
|
||||
required: true
|
||||
default: '2.6.7'
|
||||
default: '2.6.8'
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
|
|
|
|||
23
Dockerfile
23
Dockerfile
|
|
@ -29,6 +29,16 @@ COPY requirements.txt .
|
|||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# yt-dlp must track YouTube faster than its stable channel ships — stable can
|
||||
# lag months behind a breaking YouTube change while extraction is broken
|
||||
# ("Requested format is not available"). Build images with the NIGHTLY channel.
|
||||
# COMMIT_SHA is referenced in the RUN so CI's layer cache (cache-from: gha)
|
||||
# busts on every new commit — otherwise this layer could pin a stale "nightly"
|
||||
# for months, silently defeating its purpose.
|
||||
ARG COMMIT_SHA=""
|
||||
RUN echo "yt-dlp nightly for build ${COMMIT_SHA}" && \
|
||||
pip install --no-cache-dir -U --pre "yt-dlp[default]"
|
||||
|
||||
# Stage 2: Runtime — only runtime dependencies, no build tools
|
||||
FROM python:3.11-slim
|
||||
|
||||
|
|
@ -44,14 +54,25 @@ ENV PATH="/opt/venv/bin:$PATH"
|
|||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Install runtime-only system dependencies (no gcc/build tools)
|
||||
# Install runtime-only system dependencies (no gcc/build tools).
|
||||
# unzip is needed by the Deno installer below.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
gosu \
|
||||
ffmpeg \
|
||||
libchromaprint-tools \
|
||||
unzip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Deno — JavaScript runtime for yt-dlp. YouTube gates its downloadable formats
|
||||
# behind JS challenges (nsig); without a JS runtime, yt-dlp's extraction is
|
||||
# deprecated and streams / music-video downloads fail with "Requested format
|
||||
# is not available". Deno is yt-dlp's default-enabled runtime; the official
|
||||
# installer auto-detects amd64/arm64. `deno --version` fails the build early
|
||||
# if the install ever breaks.
|
||||
RUN curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh && \
|
||||
deno --version
|
||||
|
||||
# Create non-root user for security
|
||||
RUN useradd --create-home --shell /bin/bash --uid 1000 soulsync
|
||||
|
||||
|
|
|
|||
|
|
@ -317,6 +317,11 @@ cd ..
|
|||
|
||||
If `webui/static/dist/.vite/manifest.json` is missing or stale, React-owned routes and route handoffs may not load correctly.
|
||||
|
||||
**YouTube streaming / music videos** need two extra things on bare-metal installs (Docker bundles both):
|
||||
|
||||
- **Deno** — yt-dlp now requires a JavaScript runtime to unlock YouTube formats. Without it, streams and music-video downloads fail with `Requested format is not available`. Install: `winget install DenoLand.Deno` (Windows) or see [deno.com](https://docs.deno.com/runtime/), then restart SoulSync.
|
||||
- **yt-dlp nightly** — the stable release can lag months behind YouTube changes. If YouTube breaks, update with: `python -m pip install -U --pre "yt-dlp[default]"`
|
||||
|
||||
### Local Development
|
||||
|
||||
This is only for contributors working on the WebUI with hot reload. Normal Python/no-Docker installs should build once with `npm run build` as shown above, then run only Gunicorn.
|
||||
|
|
@ -355,6 +360,7 @@ on any OS. `./dev.sh` remains available as a Unix shell wrapper.
|
|||
- **slskd** running and accessible ([Download](https://github.com/slskd/slskd/releases)) — required for Soulseek downloads
|
||||
- **Spotify API** credentials ([Dashboard](https://developer.spotify.com/dashboard)) — optional but recommended for discovery
|
||||
- **Media Server** (optional): Plex, Jellyfin, or Navidrome
|
||||
- **Deno** (Python/no-Docker installs only): JavaScript runtime required by yt-dlp for YouTube streaming/music videos — `winget install DenoLand.Deno` or [deno.com](https://docs.deno.com/runtime/). Docker images bundle it.
|
||||
- **Deezer ARL token** (optional): For Deezer downloads — get from browser cookies after logging into deezer.com
|
||||
- **Tidal account** (optional): For Tidal downloads — authenticate via device flow in Settings
|
||||
- **Qobuz account** (optional): For Qobuz downloads — email/password login in Settings
|
||||
|
|
|
|||
|
|
@ -493,6 +493,32 @@ class ConfigManager:
|
|||
# editing source.
|
||||
"album_bundle_poll_interval_seconds": 2.0,
|
||||
"album_bundle_timeout_seconds": 6 * 60 * 60, # 6 hours
|
||||
# Stalled-torrent handling (noldevin): abandon a torrent that
|
||||
# makes zero download progress for this long (dead magnet
|
||||
# stuck on "downloading metadata", no seeders) instead of
|
||||
# holding the worker for the full album timeout. 0 disables.
|
||||
"torrent_stall_timeout_seconds": 10 * 60, # 10 minutes
|
||||
# What to do when a torrent stalls: "abandon" (remove it +
|
||||
# its partial data, fail the download so the next source can
|
||||
# try) or "pause" (pause in the client, leave for the user).
|
||||
"torrent_stall_action": "abandon",
|
||||
},
|
||||
"post_processing": {
|
||||
# When a download is quarantined (AcoustID mismatch, integrity /
|
||||
# duration failure), retry the next-best candidate instead of
|
||||
# failing outright. Default ON (PR #801's documented default —
|
||||
# the monitor reads this with inline default True; this template
|
||||
# said False, so fresh installs silently shipped with the retry
|
||||
# engine off while existing configs got it on. CI caught the
|
||||
# split: its fresh default config failed all 7 requeue tests).
|
||||
"retry_next_candidate_on_mismatch": True,
|
||||
# Opt-in exhaustive retry: budget retries PER SOURCE so every
|
||||
# source (Soulseek, then HiFi/Tidal/…) gets its own attempts
|
||||
# before the track gives up. Default off (single global cap).
|
||||
"retry_exhaustive": False,
|
||||
# Retries per search query per source in exhaustive mode. The
|
||||
# per-source budget is query_count × this value.
|
||||
"retries_per_query": 5,
|
||||
},
|
||||
"tidal_download": {
|
||||
"quality": "lossless", # Options: "low", "high", "lossless", "hires"
|
||||
|
|
|
|||
37
core/blocklist/__init__.py
Normal file
37
core/blocklist/__init__.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
"""Artist / album / track blocklist (the "proper" blacklist).
|
||||
|
||||
Distinct from ``download_blacklist`` (which skips one bad source file from one
|
||||
Soulseek peer — untouched here). This blocklist bans an ARTIST, ALBUM, or
|
||||
TRACK from being acquired, keyed by metadata-source IDs (Spotify / iTunes /
|
||||
Deezer / MusicBrainz) so a ban survives a source switch.
|
||||
|
||||
Phase 1 enforces at the single ``add_to_wishlist`` chokepoint: every
|
||||
auto-acquisition path (watchlist, discography backfill, repair, manual
|
||||
wishlist add) funnels through it, so one guard covers them all.
|
||||
|
||||
- ``matching`` — the pure decision core (no DB, no I/O): build an index from
|
||||
blocklist rows, ask whether a candidate is blocked, with artist→album→track
|
||||
cascade.
|
||||
"""
|
||||
|
||||
from core.blocklist.matching import (
|
||||
ENTITY_ALBUM,
|
||||
ENTITY_ARTIST,
|
||||
ENTITY_TRACK,
|
||||
ENTITY_TYPES,
|
||||
SOURCE_ID_FIELDS,
|
||||
BlocklistIndex,
|
||||
build_index,
|
||||
candidate_block_reason,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ENTITY_ARTIST",
|
||||
"ENTITY_ALBUM",
|
||||
"ENTITY_TRACK",
|
||||
"ENTITY_TYPES",
|
||||
"SOURCE_ID_FIELDS",
|
||||
"BlocklistIndex",
|
||||
"build_index",
|
||||
"candidate_block_reason",
|
||||
]
|
||||
50
core/blocklist/backfill.py
Normal file
50
core/blocklist/backfill.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
"""Cross-source ID backfill for blocklist entries.
|
||||
|
||||
When a user blocks an item, the modal gives us the ID for ONE source (the one
|
||||
they searched). For the ban to survive a source switch, we resolve the OTHER
|
||||
sources' IDs too — matching the blocked artist/album/track by name on each
|
||||
source and taking a confident hit.
|
||||
|
||||
The resolution is kept pure + injected so it tests without a network: callers
|
||||
pass a ``resolvers`` map ``{source: fn(entity_type, name, parent_name) -> id |
|
||||
None}``. ``core/blocklist/runtime.py`` wires the real metadata clients.
|
||||
|
||||
Honest about fragility (acknowledged in design): artist matching is reliable,
|
||||
album/track cross-source matching is best-effort (editions, common titles), so
|
||||
a resolver returning None just leaves that source unmatched — the artist
|
||||
name-fallback in matching.py covers artist gaps; album/track gaps mean that
|
||||
ban only applies on sources where an ID resolved.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from core.blocklist.matching import SOURCE_ID_FIELDS
|
||||
|
||||
|
||||
def resolve_missing_ids(
|
||||
entry: Dict[str, Any],
|
||||
resolvers: Dict[str, Callable[..., Optional[str]]],
|
||||
) -> Dict[str, str]:
|
||||
"""Return ``{id_column: resolved_id}`` for the sources currently missing an
|
||||
ID on ``entry``. Never raises — a resolver that errors is skipped."""
|
||||
out: Dict[str, str] = {}
|
||||
entity_type = entry.get("entity_type")
|
||||
name = entry.get("name")
|
||||
parent = entry.get("parent_name")
|
||||
if not entity_type or not name:
|
||||
return out
|
||||
for source, col in SOURCE_ID_FIELDS.items():
|
||||
if entry.get(col):
|
||||
continue # already known
|
||||
fn = resolvers.get(source)
|
||||
if not fn:
|
||||
continue
|
||||
try:
|
||||
rid = fn(entity_type, name, parent)
|
||||
except Exception:
|
||||
rid = None
|
||||
if rid:
|
||||
out[col] = str(rid)
|
||||
return out
|
||||
128
core/blocklist/matching.py
Normal file
128
core/blocklist/matching.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
"""Pure blocklist matching — no DB, no I/O, fully unit-testable.
|
||||
|
||||
The brain of the blocklist: given the stored blocklist rows and a candidate
|
||||
track being considered for the wishlist, decide whether it's blocked.
|
||||
|
||||
Design decisions (per Boulder):
|
||||
- **ID-keyed.** Each row carries the candidate's IDs in up to four metadata
|
||||
sources. A candidate is matched against the SAME source it came in on
|
||||
(the wishlist payload carries active-source IDs), so a Deezer-numeric id
|
||||
can't collide with an iTunes-numeric id of a different entity.
|
||||
- **Cascade.** Blocking an artist blocks their albums + tracks; blocking an
|
||||
album blocks its tracks. The candidate carries its own artist/album/track
|
||||
IDs, so the check walks track → album → artist and blocks on the first hit.
|
||||
- **Name fallback for ARTISTS only.** A blocked artist also matches by
|
||||
case-folded name — this covers the window before the background ID-backfill
|
||||
has resolved the active source's id. Albums/tracks do NOT fall back to name
|
||||
(common titles like "Greatest Hits" would false-positive across artists);
|
||||
they rely on IDs, which backfill fills in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
|
||||
|
||||
ENTITY_ARTIST = "artist"
|
||||
ENTITY_ALBUM = "album"
|
||||
ENTITY_TRACK = "track"
|
||||
ENTITY_TYPES = (ENTITY_ARTIST, ENTITY_ALBUM, ENTITY_TRACK)
|
||||
|
||||
# Blocklist-row column → the metadata source it belongs to.
|
||||
SOURCE_ID_FIELDS = {
|
||||
"spotify": "spotify_id",
|
||||
"itunes": "itunes_id",
|
||||
"deezer": "deezer_id",
|
||||
"musicbrainz": "musicbrainz_id",
|
||||
}
|
||||
|
||||
|
||||
def _norm(text: Any) -> str:
|
||||
return str(text or "").strip().casefold()
|
||||
|
||||
|
||||
@dataclass
|
||||
class _TypeIndex:
|
||||
# per-source set of blocked ids, plus a case-folded name set (artists only)
|
||||
ids: Dict[str, Set[str]] = field(default_factory=lambda: {s: set() for s in SOURCE_ID_FIELDS})
|
||||
names: Set[str] = field(default_factory=set)
|
||||
|
||||
def hit(self, source: Optional[str], entity_id: Any, name: Any, use_name: bool) -> bool:
|
||||
if entity_id and source in self.ids and str(entity_id) in self.ids[source]:
|
||||
return True
|
||||
if use_name and name and _norm(name) in self.names:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@dataclass
|
||||
class BlocklistIndex:
|
||||
"""Membership index built once per scan from the blocklist rows."""
|
||||
artists: _TypeIndex = field(default_factory=_TypeIndex)
|
||||
albums: _TypeIndex = field(default_factory=_TypeIndex)
|
||||
tracks: _TypeIndex = field(default_factory=_TypeIndex)
|
||||
|
||||
@property
|
||||
def is_empty(self) -> bool:
|
||||
for ti in (self.artists, self.albums, self.tracks):
|
||||
if ti.names or any(ti.ids.values()):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def build_index(rows: Iterable[Dict[str, Any]]) -> BlocklistIndex:
|
||||
"""Build a BlocklistIndex from blocklist DB rows.
|
||||
|
||||
Each row needs ``entity_type``, ``name``, and the source id columns
|
||||
(``spotify_id`` / ``itunes_id`` / ``deezer_id`` / ``musicbrainz_id``).
|
||||
Unknown entity types are ignored."""
|
||||
idx = BlocklistIndex()
|
||||
by_type = {ENTITY_ARTIST: idx.artists, ENTITY_ALBUM: idx.albums, ENTITY_TRACK: idx.tracks}
|
||||
for row in rows or []:
|
||||
ti = by_type.get((row.get("entity_type") or "").strip().lower())
|
||||
if ti is None:
|
||||
continue
|
||||
for source, col in SOURCE_ID_FIELDS.items():
|
||||
val = row.get(col)
|
||||
if val:
|
||||
ti.ids[source].add(str(val))
|
||||
name = row.get("name")
|
||||
if name:
|
||||
ti.names.add(_norm(name))
|
||||
return idx
|
||||
|
||||
|
||||
def candidate_block_reason(
|
||||
index: BlocklistIndex,
|
||||
*,
|
||||
source: Optional[str],
|
||||
track_id: Any = None,
|
||||
track_name: Any = None,
|
||||
album_id: Any = None,
|
||||
album_name: Any = None,
|
||||
artists: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> Optional[Tuple[str, str]]:
|
||||
"""Return ``(entity_type, label)`` for the first cascade hit, else None.
|
||||
|
||||
``source`` is the metadata source the candidate IDs came from (the wishlist
|
||||
payload's provider). ``artists`` is a list of ``{'id', 'name'}`` dicts.
|
||||
Order matters only for the returned reason — any hit blocks."""
|
||||
if index.is_empty:
|
||||
return None
|
||||
|
||||
# Track level — id only (names too ambiguous to ban across artists).
|
||||
if index.tracks.hit(source, track_id, track_name, use_name=False):
|
||||
return (ENTITY_TRACK, str(track_name or track_id or "track"))
|
||||
|
||||
# Album level — id only.
|
||||
if index.albums.hit(source, album_id, album_name, use_name=False):
|
||||
return (ENTITY_ALBUM, str(album_name or album_id or "album"))
|
||||
|
||||
# Artist level — id OR case-folded name (safe + covers the backfill window).
|
||||
for artist in artists or []:
|
||||
a_id = artist.get("id") if isinstance(artist, dict) else None
|
||||
a_name = artist.get("name") if isinstance(artist, dict) else artist
|
||||
if index.artists.hit(source, a_id, a_name, use_name=True):
|
||||
return (ENTITY_ARTIST, str(a_name or a_id or "artist"))
|
||||
|
||||
return None
|
||||
82
core/blocklist/runtime.py
Normal file
82
core/blocklist/runtime.py
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
"""Wire real metadata clients to the blocklist backfill resolvers.
|
||||
|
||||
Resolves a blocked item's ID on each metadata source by searching that source
|
||||
for the name and taking a confidently name-matched hit. Confidence = exact
|
||||
significant-token match (drops articles/punctuation) so we never hang a wrong
|
||||
ID on an entry. Albums/tracks additionally require the parent artist to match
|
||||
when both sides expose one.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("blocklist.runtime")
|
||||
|
||||
_STOP = {"the", "a", "an", "feat", "ft", "featuring", "with"}
|
||||
|
||||
|
||||
def _tokens(text: Any) -> frozenset:
|
||||
words = re.sub(r"[^a-z0-9]+", " ", str(text or "").lower()).split()
|
||||
return frozenset(w for w in words if w not in _STOP)
|
||||
|
||||
|
||||
def _name_of(obj: Any) -> str:
|
||||
if isinstance(obj, dict):
|
||||
return str(obj.get("name") or obj.get("title") or "")
|
||||
return str(getattr(obj, "name", None) or getattr(obj, "title", None) or "")
|
||||
|
||||
|
||||
def _id_of(obj: Any) -> Optional[str]:
|
||||
val = obj.get("id") if isinstance(obj, dict) else getattr(obj, "id", None)
|
||||
return str(val) if val else None
|
||||
|
||||
|
||||
def _confident(result_name: str, want_name: str) -> bool:
|
||||
rt, wt = _tokens(result_name), _tokens(want_name)
|
||||
return bool(rt) and rt == wt
|
||||
|
||||
|
||||
def _make_resolver(source: str) -> Callable[..., Optional[str]]:
|
||||
def resolve(entity_type: str, name: str, parent_name: Optional[str] = None) -> Optional[str]:
|
||||
from core.metadata.registry import get_client_for_source
|
||||
client = get_client_for_source(source)
|
||||
if not client:
|
||||
return None
|
||||
method = {
|
||||
"artist": "search_artists",
|
||||
"album": "search_albums",
|
||||
"track": "search_tracks",
|
||||
}.get(entity_type)
|
||||
fn = getattr(client, method, None) if method else None
|
||||
if not fn:
|
||||
return None
|
||||
try:
|
||||
results = fn(name, limit=5) or []
|
||||
except Exception as e:
|
||||
logger.debug("%s %s search failed for %r: %s", source, entity_type, name, e)
|
||||
return None
|
||||
for r in results:
|
||||
if not _confident(_name_of(r), name):
|
||||
continue
|
||||
# For album/track, also require the artist to line up when known.
|
||||
if entity_type in ("album", "track") and parent_name:
|
||||
artists = (r.get("artists") if isinstance(r, dict) else getattr(r, "artists", None)) or []
|
||||
cand_artists = " ".join(
|
||||
a.get("name", "") if isinstance(a, dict) else str(a) for a in artists)
|
||||
if _tokens(parent_name) and not (_tokens(parent_name) & _tokens(cand_artists)):
|
||||
continue
|
||||
rid = _id_of(r)
|
||||
if rid:
|
||||
return rid
|
||||
return None
|
||||
|
||||
return resolve
|
||||
|
||||
|
||||
def build_resolvers() -> Dict[str, Callable[..., Optional[str]]]:
|
||||
"""Source→resolver map for core.blocklist.backfill.resolve_missing_ids."""
|
||||
return {s: _make_resolver(s) for s in ("spotify", "itunes", "deezer", "musicbrainz")}
|
||||
0
core/diagnostics/__init__.py
Normal file
0
core/diagnostics/__init__.py
Normal file
136
core/diagnostics/memory_tracker.py
Normal file
136
core/diagnostics/memory_tracker.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
"""On-demand memory-growth diagnostic (issue #802: ~0.7 MiB/s RSS growth).
|
||||
|
||||
Wraps ``tracemalloc`` so a user seeing runaway memory can capture WHERE the
|
||||
allocations come from instead of us guessing:
|
||||
|
||||
1. start_tracking() — begins tracing + stores a baseline snapshot
|
||||
2. ...reproduce the growth for a few minutes...
|
||||
3. report() — top allocation sites, with the DELTA since baseline
|
||||
(the delta is the leak; absolute sizes are mostly
|
||||
startup noise)
|
||||
4. stop_tracking() — ends tracing, frees trace memory
|
||||
|
||||
Opt-in by design: tracemalloc costs CPU and memory while active (it shadows
|
||||
every allocation), so it must never run by default. The Flask endpoints that
|
||||
expose this live in web_server (GET /api/debug/memory/...) so a user can drive
|
||||
the whole flow from a browser.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
import tracemalloc
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("diagnostics.memory")
|
||||
|
||||
_baseline: Optional[tracemalloc.Snapshot] = None
|
||||
_started_at: Optional[float] = None
|
||||
|
||||
# Allocation-site traces this deep give useful "who called it" context without
|
||||
# pathological overhead.
|
||||
_TRACE_FRAMES = 15
|
||||
|
||||
|
||||
def is_tracking() -> bool:
|
||||
return tracemalloc.is_tracing()
|
||||
|
||||
|
||||
def start_tracking() -> Dict[str, Any]:
|
||||
"""Begin tracing and store the baseline snapshot. Idempotent."""
|
||||
global _baseline, _started_at
|
||||
if tracemalloc.is_tracing():
|
||||
return {"tracking": True, "already_running": True, "started_at": _started_at}
|
||||
tracemalloc.start(_TRACE_FRAMES)
|
||||
_baseline = tracemalloc.take_snapshot()
|
||||
_started_at = time.time()
|
||||
logger.info("Memory tracking started (tracemalloc, %d frames)", _TRACE_FRAMES)
|
||||
return {"tracking": True, "already_running": False, "started_at": _started_at}
|
||||
|
||||
|
||||
def stop_tracking() -> Dict[str, Any]:
|
||||
"""End tracing and free the trace bookkeeping."""
|
||||
global _baseline, _started_at
|
||||
was = tracemalloc.is_tracing()
|
||||
if was:
|
||||
tracemalloc.stop()
|
||||
logger.info("Memory tracking stopped")
|
||||
_baseline = None
|
||||
_started_at = None
|
||||
return {"tracking": False, "was_tracking": was}
|
||||
|
||||
|
||||
def _rss_mb() -> Optional[float]:
|
||||
"""Process RSS in MiB, best-effort (psutil, then /proc fallback)."""
|
||||
try:
|
||||
import psutil
|
||||
return round(psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024), 1)
|
||||
except Exception: # noqa: S110 — RSS is optional context; fall through to /proc
|
||||
pass
|
||||
try:
|
||||
with open("/proc/self/status", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
if line.startswith("VmRSS:"):
|
||||
return round(int(line.split()[1]) / 1024, 1)
|
||||
except Exception: # noqa: S110 — no /proc on this platform; RSS stays None
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def format_stat(stat: Any) -> Dict[str, Any]:
|
||||
"""Project one tracemalloc StatisticDiff/Statistic into a plain dict.
|
||||
Duck-typed (reads size/count/size_diff/count_diff/traceback) so it's
|
||||
unit-testable without real snapshots."""
|
||||
tb = getattr(stat, "traceback", None)
|
||||
frames: List[str] = []
|
||||
if tb:
|
||||
# Most-recent-call-last reads naturally top-down in a report.
|
||||
for frame in list(tb)[-3:]:
|
||||
frames.append(f"{frame.filename}:{frame.lineno}")
|
||||
return {
|
||||
"location": frames[-1] if frames else "?",
|
||||
"trace": frames,
|
||||
"size_mb": round(getattr(stat, "size", 0) / (1024 * 1024), 3),
|
||||
"size_diff_mb": round(getattr(stat, "size_diff", 0) / (1024 * 1024), 3),
|
||||
"count": getattr(stat, "count", 0),
|
||||
"count_diff": getattr(stat, "count_diff", 0),
|
||||
}
|
||||
|
||||
|
||||
def report(top: int = 25) -> Dict[str, Any]:
|
||||
"""Current snapshot vs the start_tracking() baseline: the top allocation
|
||||
sites by GROWTH (size_diff). Includes traced totals + process RSS so the
|
||||
user can see how much of the real growth tracemalloc accounts for."""
|
||||
if not tracemalloc.is_tracing():
|
||||
return {
|
||||
"tracking": False,
|
||||
"rss_mb": _rss_mb(),
|
||||
"hint": "Start with /api/debug/memory/start, reproduce the growth "
|
||||
"for a few minutes, then call this again.",
|
||||
}
|
||||
snapshot = tracemalloc.take_snapshot()
|
||||
# Filter the tracer's own bookkeeping out of the picture.
|
||||
snapshot = snapshot.filter_traces((
|
||||
tracemalloc.Filter(False, tracemalloc.__file__),
|
||||
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
|
||||
))
|
||||
current, peak = tracemalloc.get_traced_memory()
|
||||
|
||||
if _baseline is not None:
|
||||
stats = snapshot.compare_to(_baseline, "traceback")
|
||||
stats.sort(key=lambda s: s.size_diff, reverse=True)
|
||||
else:
|
||||
stats = snapshot.statistics("traceback")
|
||||
|
||||
return {
|
||||
"tracking": True,
|
||||
"started_at": _started_at,
|
||||
"elapsed_seconds": round(time.time() - _started_at, 1) if _started_at else None,
|
||||
"traced_current_mb": round(current / (1024 * 1024), 1),
|
||||
"traced_peak_mb": round(peak / (1024 * 1024), 1),
|
||||
"rss_mb": _rss_mb(),
|
||||
"top_growth": [format_stat(s) for s in stats[:top]],
|
||||
}
|
||||
|
|
@ -524,12 +524,12 @@ def run_sync_task(
|
|||
# don't want persisted to app.log.
|
||||
_synced = getattr(result, 'synced_tracks', 0)
|
||||
logger.info(f"[PLAYLIST IMAGE] has_image={bool(playlist_image_url)}, synced_tracks={_synced}")
|
||||
# In reconcile mode the whole point (#792) is to NOT touch the playlist's
|
||||
# existing image — pushing the source image here would re-clobber a
|
||||
# user's custom poster every sync, the exact bug reconcile fixes. So skip
|
||||
# the auto image push for reconcile (the playlist keeps its own art).
|
||||
if sync_mode == 'reconcile':
|
||||
logger.info("[PLAYLIST IMAGE] reconcile mode — preserving existing playlist image")
|
||||
# Modes that edit a playlist in place (reconcile #792, append #811) must
|
||||
# NOT push the source image — doing so re-clobbers a user's custom poster
|
||||
# every sync, the exact bug these modes exist to avoid. Only the
|
||||
# destructive 'replace' (recreate-from-scratch) pushes the image.
|
||||
if sync_mode in ('reconcile', 'append'):
|
||||
logger.info(f"[PLAYLIST IMAGE] {sync_mode} mode — preserving existing playlist image")
|
||||
elif playlist_image_url and _synced > 0:
|
||||
try:
|
||||
active_server = deps.config_manager.get_active_media_server()
|
||||
|
|
|
|||
|
|
@ -68,6 +68,11 @@ from core.download_plugins.album_bundle import (
|
|||
resolve_reported_save_path,
|
||||
)
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
from core.download_plugins.torrent_stall import (
|
||||
StallTracker,
|
||||
get_stall_action,
|
||||
get_stall_timeout,
|
||||
)
|
||||
from core.download_plugins.types import AlbumResult, DownloadStatus, TrackResult
|
||||
from core.prowlarr_client import (
|
||||
DEFAULT_MUSIC_CATEGORIES,
|
||||
|
|
@ -305,6 +310,11 @@ class TorrentDownloadPlugin(DownloadSourcePlugin):
|
|||
# but the same tolerance keeps a one-off connection failure
|
||||
# from killing an otherwise-healthy download.
|
||||
misses = TransientMissCounter()
|
||||
# Stalled-torrent handling (noldevin): give up early on a torrent
|
||||
# making zero progress (dead magnet stuck on metadata, no seeders)
|
||||
# instead of holding this worker for the full album deadline. Read
|
||||
# per-download so a settings change applies to in-flight torrents.
|
||||
stall = StallTracker(get_stall_timeout())
|
||||
while time.monotonic() < deadline:
|
||||
if self.shutdown_check and self.shutdown_check():
|
||||
return
|
||||
|
|
@ -345,10 +355,37 @@ class TorrentDownloadPlugin(DownloadSourcePlugin):
|
|||
self._mark_error(download_id, status.error or "Torrent client reported error")
|
||||
return
|
||||
|
||||
if stall.is_stalled(status.downloaded, status.state, time.monotonic()):
|
||||
self._handle_stalled(download_id, torrent_hash, get_stall_action())
|
||||
return
|
||||
|
||||
time.sleep(_POLL_INTERVAL_SECONDS)
|
||||
|
||||
self._mark_error(download_id, "Torrent download timed out")
|
||||
|
||||
def _handle_stalled(self, download_id: str, torrent_hash: str, action: str) -> None:
|
||||
"""A torrent made no progress past the stall timeout. Abandon it
|
||||
(remove from client + delete its partial data) or pause it for the
|
||||
user, then fail the download so the worker frees up."""
|
||||
adapter = get_active_torrent_adapter()
|
||||
timeout_min = round(get_stall_timeout() / 60, 1)
|
||||
if adapter is not None:
|
||||
try:
|
||||
if action == "pause":
|
||||
run_async(adapter.pause(torrent_hash))
|
||||
else:
|
||||
# delete_files: a stalled torrent's partial data is junk
|
||||
# (often just a metadata stub) — don't leave it on disk.
|
||||
run_async(adapter.remove(torrent_hash, delete_files=True))
|
||||
except Exception as e:
|
||||
logger.warning("Stalled-torrent %s on %s failed: %s",
|
||||
action, torrent_hash[:8] if torrent_hash else "?", e)
|
||||
verb = "paused" if action == "pause" else "removed"
|
||||
self._mark_error(
|
||||
download_id,
|
||||
f"Torrent stalled (no progress for {timeout_min} min) — {verb}",
|
||||
)
|
||||
|
||||
def _finalize_download(self, download_id: str, save_path: Optional[str]) -> None:
|
||||
"""Adapter said complete. Walk the directory + pick the
|
||||
first audio file as the canonical ``file_path``."""
|
||||
|
|
|
|||
107
core/download_plugins/torrent_stall.py
Normal file
107
core/download_plugins/torrent_stall.py
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
"""Stalled-torrent detection + policy (noldevin's request).
|
||||
|
||||
A torrent can sit forever making zero progress — most commonly stuck
|
||||
"downloading metadata" on a magnet with no peers, but also a dead swarm
|
||||
mid-download. The torrent poll loop would just burn the full 6-hour album
|
||||
timeout on it. This module decides, from the live status stream, when a
|
||||
torrent has been stalled too long, and what to do about it.
|
||||
|
||||
Design split, kept testable:
|
||||
- ``StallTracker`` is the pure decision core — feed it each poll's
|
||||
``(downloaded, state, now)`` and it answers "stalled too long?" using a
|
||||
monotonic clock passed in (no time import, no I/O). Progress = bytes
|
||||
moved since the last poll; any forward movement resets the stall clock.
|
||||
Terminal/healthy-but-idle states (seeding, completed, paused) never count
|
||||
as stalled — only states where the torrent is *supposed* to be working.
|
||||
- ``get_stall_timeout`` / ``get_stall_action`` read the two settings.
|
||||
|
||||
A timeout of 0 disables stall handling entirely (back to the old behavior:
|
||||
ride the full poll deadline).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from config.settings import config_manager
|
||||
|
||||
# 0 = disabled. 10 minutes is long enough to ride out a slow metadata fetch
|
||||
# or a brief peer drought, short enough to give up on a truly dead magnet
|
||||
# instead of holding a worker for 6 hours.
|
||||
DEFAULT_STALL_TIMEOUT_SECONDS = 10 * 60
|
||||
|
||||
# What to do when a torrent stalls past the timeout:
|
||||
# 'abandon' — remove it from the client (and its partial data) + fail the
|
||||
# download so the worker is freed and the next source can try.
|
||||
# 'pause' — pause it in the client + fail the download, leaving the
|
||||
# torrent for the user to inspect/resume manually.
|
||||
_VALID_ACTIONS = ("abandon", "pause")
|
||||
DEFAULT_STALL_ACTION = "abandon"
|
||||
|
||||
# States where the torrent is meant to be making download progress, so a
|
||||
# lack of it counts toward the stall clock. Mirrors the adapter-uniform set
|
||||
# in core/torrent_clients/base.py. Notably EXCLUDES seeding/completed (done)
|
||||
# and paused (the user's own choice) — neither is a stall.
|
||||
STALLABLE_STATES = frozenset(("queued", "downloading", "stalled", "error"))
|
||||
|
||||
|
||||
def get_stall_timeout() -> float:
|
||||
"""Seconds of zero progress before a torrent is considered stalled.
|
||||
0 (or invalid/negative) disables stall handling."""
|
||||
raw = config_manager.get("download_source.torrent_stall_timeout_seconds",
|
||||
DEFAULT_STALL_TIMEOUT_SECONDS)
|
||||
try:
|
||||
value = float(raw)
|
||||
if value >= 0:
|
||||
return value
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return DEFAULT_STALL_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def get_stall_action() -> str:
|
||||
"""What to do with a stalled torrent: 'abandon' (default) or 'pause'."""
|
||||
raw = config_manager.get("download_source.torrent_stall_action",
|
||||
DEFAULT_STALL_ACTION)
|
||||
action = str(raw or "").strip().lower()
|
||||
return action if action in _VALID_ACTIONS else DEFAULT_STALL_ACTION
|
||||
|
||||
|
||||
class StallTracker:
|
||||
"""Tracks one torrent's forward progress across polls.
|
||||
|
||||
Pure + clock-injected so it tests without sleeping. ``timeout`` <= 0
|
||||
disables it (``is_stalled`` always returns False)."""
|
||||
|
||||
def __init__(self, timeout_seconds: float):
|
||||
self.timeout = float(timeout_seconds or 0)
|
||||
self._last_downloaded = -1 # -1 = first observation
|
||||
self._progress_since = None # monotonic time of last forward movement
|
||||
|
||||
def is_stalled(self, downloaded: int, state: str, now: float) -> bool:
|
||||
"""Record this poll's observation; return True iff the torrent has
|
||||
gone ``timeout`` seconds with no byte progress while in a state
|
||||
that's supposed to be downloading.
|
||||
|
||||
``downloaded`` is cumulative bytes; ``state`` is the adapter-uniform
|
||||
state; ``now`` is a monotonic timestamp (seconds)."""
|
||||
if self.timeout <= 0:
|
||||
return False
|
||||
|
||||
downloaded = int(downloaded or 0)
|
||||
|
||||
# Forward progress (or first sighting) resets the stall clock.
|
||||
if self._last_downloaded < 0 or downloaded > self._last_downloaded:
|
||||
self._last_downloaded = downloaded
|
||||
self._progress_since = now
|
||||
return False
|
||||
self._last_downloaded = downloaded
|
||||
|
||||
# Not in a working state → not a stall (seeding/paused/completed).
|
||||
if state not in STALLABLE_STATES:
|
||||
self._progress_since = now # don't accrue stall time while idle-by-design
|
||||
return False
|
||||
|
||||
if self._progress_since is None:
|
||||
self._progress_since = now
|
||||
return False
|
||||
|
||||
return (now - self._progress_since) >= self.timeout
|
||||
|
|
@ -608,6 +608,34 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma
|
|||
if skipped > 0:
|
||||
logger.warning(f"[Content Filter] Filtered out {skipped} explicit track(s) from download queue")
|
||||
|
||||
# Blocklist (Phase 2a): drop banned artists/albums/tracks before queueing,
|
||||
# so a blocked item can't slip in via playlist sync / album download /
|
||||
# discography. Same ID-cascade brain as the wishlist guard (Phase 1) —
|
||||
# the only other auto-acquisition path. Skipped when the user confirmed
|
||||
# "download anyway" at the modal (Phase 2b override).
|
||||
_ignore_blocklist = False
|
||||
with tasks_lock:
|
||||
if batch_id in download_batches:
|
||||
_ignore_blocklist = download_batches[batch_id].get('ignore_blocklist', False)
|
||||
if not _ignore_blocklist:
|
||||
try:
|
||||
_bl_before = len(missing_tracks)
|
||||
_bl_kept = []
|
||||
for res in missing_tracks:
|
||||
reason = db.blocklist_reason_for_track(
|
||||
batch_profile_id, res.get('track', {}), source=batch_source)
|
||||
if reason:
|
||||
logger.info("[Blocklist] Skipping %s '%s' from download queue (%s blocked)",
|
||||
reason[0], res.get('track', {}).get('name', '?'), reason[0])
|
||||
else:
|
||||
_bl_kept.append(res)
|
||||
if len(_bl_kept) != _bl_before:
|
||||
logger.info("[Blocklist] Filtered out %d blocklisted track(s) from download queue",
|
||||
_bl_before - len(_bl_kept))
|
||||
missing_tracks = _bl_kept
|
||||
except Exception as _bl_err:
|
||||
logger.debug("blocklist queue filter skipped: %s", _bl_err)
|
||||
|
||||
with tasks_lock:
|
||||
if batch_id in download_batches:
|
||||
download_batches[batch_id]['analysis_results'] = analysis_results
|
||||
|
|
@ -1089,6 +1117,26 @@ def run_full_missing_tracks_process(batch_id, playlist_id, tracks_json, deps: Ma
|
|||
f"{track_info.get('name')}"
|
||||
)
|
||||
|
||||
# Download-origin provenance: stamp what TRIGGERED this download
|
||||
# so the history chokepoint can record it (origin-history modal).
|
||||
# Wishlist rows already ride their source_info in track_info
|
||||
# (watchlist_artist_name / playlist_name — the deriver reads
|
||||
# those directly); this stamp covers DIRECT playlist batches,
|
||||
# where the playlist context otherwise only survives in
|
||||
# folder mode.
|
||||
if '_dl_origin' not in track_info and batch_source_playlist_ref and batch_playlist_name:
|
||||
_prov_si = track_info.get('source_info') or {}
|
||||
if isinstance(_prov_si, str):
|
||||
try:
|
||||
_prov_si = json.loads(_prov_si)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
_prov_si = {}
|
||||
if not _prov_si.get('watchlist_artist_name'):
|
||||
track_info['_dl_origin'] = 'playlist'
|
||||
track_info['_dl_origin_context'] = (
|
||||
_prov_si.get('playlist_name') or batch_playlist_name
|
||||
)
|
||||
|
||||
download_tasks[task_id] = {
|
||||
'status': 'pending', 'track_info': track_info,
|
||||
'playlist_id': playlist_id, 'batch_id': batch_id,
|
||||
|
|
|
|||
|
|
@ -37,11 +37,228 @@ missing_download_executor = None
|
|||
download_orchestrator = None
|
||||
_RELEASE_SOURCE_NAMES = frozenset(('torrent', 'usenet'))
|
||||
|
||||
# Hard ceiling on automatic next-candidate retries after a download was
|
||||
# quarantined (AcoustID mismatch / integrity / duration). The natural
|
||||
# terminator is used_sources exhaustion — once every candidate the worker can
|
||||
# find has been tried, attempt_download_with_candidates returns False and the
|
||||
# worker reports a clean failure. This cap is a safety net against a pathological
|
||||
# quarantine→retry→quarantine loop (e.g. a source that keeps returning fresh
|
||||
# wrong files).
|
||||
#
|
||||
# Default (non-exhaustive) mode uses this single global cap. The opt-in
|
||||
# exhaustive mode (post_processing.retry_exhaustive) instead budgets retries
|
||||
# PER SOURCE — see requeue_quarantined_task_for_retry.
|
||||
MAX_QUARANTINE_RETRIES = 5
|
||||
|
||||
# Absolute runaway guard for exhaustive mode. Per-source budgets are already
|
||||
# finite (query_count × retries_per_query, and Soulseek peers all collapse to
|
||||
# one 'soulseek' bucket), but this ceiling caps the TOTAL retries across every
|
||||
# source so a misbehaving source-resolution can never loop forever.
|
||||
MAX_TOTAL_QUARANTINE_RETRIES = 100
|
||||
|
||||
# Streaming plugins report their source name as the download's "username"
|
||||
# (see download_orchestrator._streaming_sources). Soulseek uses the peer name
|
||||
# instead, so anything not in this set is bucketed under 'soulseek' for the
|
||||
# per-source retry budget.
|
||||
_STREAMING_SOURCE_NAMES = frozenset((
|
||||
'youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr', 'soundcloud', 'amazon',
|
||||
))
|
||||
|
||||
|
||||
def _resolve_download_source(username):
|
||||
"""Map a download's username to its logical source for per-source budgeting.
|
||||
|
||||
Streaming sources use the source name as username; Soulseek uses the peer
|
||||
name, so every Soulseek peer collapses to a single 'soulseek' bucket.
|
||||
"""
|
||||
if username and username in _STREAMING_SOURCE_NAMES:
|
||||
return username
|
||||
return 'soulseek'
|
||||
|
||||
|
||||
def _remaining_fallback_sources(exhausted):
|
||||
"""Sources in the configured hybrid chain that haven't exhausted their
|
||||
per-source budget yet.
|
||||
|
||||
When a source spends its whole budget (exhaustive mode), the task switches
|
||||
to the next source instead of failing — but only if there *is* another
|
||||
source. Single-source mode has nothing to fall back to, so this returns
|
||||
empty there (and when the orchestrator isn't wired). The returned list
|
||||
drives both the give-up decision here and the worker's search-exclusion on
|
||||
the next attempt (see task_worker: exhausted_download_sources).
|
||||
"""
|
||||
orch = download_orchestrator
|
||||
if orch is None or getattr(orch, 'mode', None) != 'hybrid':
|
||||
return []
|
||||
chain = getattr(orch, 'hybrid_order', None) or []
|
||||
blocked = {str(s).lower() for s in exhausted}
|
||||
return [s for s in chain if str(s).lower() not in blocked]
|
||||
|
||||
|
||||
def _download_id_key(download_id):
|
||||
return f"download_id::{download_id}" if download_id else None
|
||||
|
||||
|
||||
def requeue_quarantined_task_for_retry(task_id, batch_id, trigger):
|
||||
"""Re-queue a task whose download was just quarantined so the worker tries
|
||||
the NEXT best candidate instead of failing outright.
|
||||
|
||||
Called from the post-processing verification wrapper when AcoustID
|
||||
verification or the integrity/duration check quarantines a file. It mirrors
|
||||
the monitor's transfer-error retry path: mark the bad source as used, clear
|
||||
the stale download identity, reset the task to ``searching`` and resubmit
|
||||
the download worker. Because ``used_sources`` is preserved across the
|
||||
re-run, the worker skips the quarantined source and picks the next-best
|
||||
candidate (see ``attempt_download_with_candidates``).
|
||||
|
||||
Returns True if a retry was queued — the caller must then NOT mark the task
|
||||
failed or notify batch completion, since the task is going around again.
|
||||
Returns False when no retry is possible (retry engine unwired, manual pick,
|
||||
cancelled, or retry budget exhausted); the caller falls through to its
|
||||
existing failure handling.
|
||||
"""
|
||||
# Opt-out escape hatch — default on. Lets users restore the old
|
||||
# quarantine-and-fail behaviour without a code change.
|
||||
if not config_manager.get('post_processing.retry_next_candidate_on_mismatch', True):
|
||||
return False
|
||||
|
||||
# Retry engine not wired (e.g. manual-import path that never started a
|
||||
# download worker). Nothing to re-run.
|
||||
if missing_download_executor is None or _download_track_worker is None:
|
||||
return False
|
||||
|
||||
with tasks_lock:
|
||||
task = download_tasks.get(task_id)
|
||||
if not task:
|
||||
return False
|
||||
# The user explicitly picked this candidate via the candidates modal —
|
||||
# honour their choice rather than silently swapping in another file.
|
||||
# (Matches the monitor's transfer-retry guards.)
|
||||
if task.get('_user_manual_pick'):
|
||||
return False
|
||||
if task.get('status') == 'cancelled':
|
||||
return False
|
||||
|
||||
username = task.get('username')
|
||||
filename = task.get('filename')
|
||||
# No source identity means this wasn't a worker-dispatched download we
|
||||
# can retry — without the "{username}_{filename}" key we can't flag the
|
||||
# bad source as used, so a re-run could re-pick the same file and loop.
|
||||
# Bail and let the caller fail it normally.
|
||||
if not username or not filename:
|
||||
return False
|
||||
|
||||
total_count = task.get('quarantine_retry_count', 0)
|
||||
|
||||
if config_manager.get('post_processing.retry_exhaustive', False):
|
||||
# Exhaustive mode: a SEPARATE budget per source. The budget scales
|
||||
# with the track's own query count (the worker generates a variable
|
||||
# number of search queries per track) × the configured retries per
|
||||
# query. Soulseek candidates are walked first (one per retry), then
|
||||
# the worker's hybrid fallback moves to the next source — each source
|
||||
# spending its own budget. The natural terminator (used_sources
|
||||
# exhaustion → worker clean-fail) still ends most tracks well before
|
||||
# any budget is reached; the budget is the per-source safety ceiling.
|
||||
source = _resolve_download_source(username)
|
||||
retries_per_query = config_manager.get('post_processing.retries_per_query', 5)
|
||||
try:
|
||||
retries_per_query = int(retries_per_query)
|
||||
except (TypeError, ValueError):
|
||||
retries_per_query = 5
|
||||
if retries_per_query < 1:
|
||||
retries_per_query = 1
|
||||
|
||||
query_count = task.get('query_count') or 1
|
||||
if query_count < 1:
|
||||
query_count = 1
|
||||
budget = query_count * retries_per_query
|
||||
|
||||
counts = task.get('quarantine_retry_counts_by_source')
|
||||
if not isinstance(counts, dict):
|
||||
counts = {}
|
||||
source_count = counts.get(source, 0)
|
||||
|
||||
if source_count >= budget:
|
||||
# This source spent its whole budget. Rather than fail the
|
||||
# track outright, mark the source exhausted and fall through to
|
||||
# the next source in the hybrid chain (the worker excludes
|
||||
# exhausted sources from its next search). Only give up once no
|
||||
# fallback source remains — or the absolute ceiling trips.
|
||||
exhausted = set(task.get('exhausted_download_sources') or ())
|
||||
exhausted.add(source)
|
||||
remaining = _remaining_fallback_sources(exhausted)
|
||||
if not remaining:
|
||||
logger.warning(
|
||||
f"[Retry:{trigger}] Task {task_id} exhausted its retry "
|
||||
f"budget for source '{source}' ({source_count}/{budget}) "
|
||||
f"and no fallback source remains — giving up, marking failed"
|
||||
)
|
||||
return False
|
||||
if total_count >= MAX_TOTAL_QUARANTINE_RETRIES:
|
||||
logger.warning(
|
||||
f"[Retry:{trigger}] Task {task_id} hit the absolute retry "
|
||||
f"ceiling ({MAX_TOTAL_QUARANTINE_RETRIES}) — giving up, "
|
||||
f"marking failed"
|
||||
)
|
||||
return False
|
||||
task['exhausted_download_sources'] = exhausted
|
||||
# Don't push this source's counter past its budget — it's done.
|
||||
# The next source starts spending its own fresh budget when its
|
||||
# first candidate fails verification.
|
||||
attempt_desc = (
|
||||
f"source '{source}' budget spent ({source_count}/{budget}) "
|
||||
f"— switching sources (remaining: {', '.join(remaining)})"
|
||||
)
|
||||
else:
|
||||
if total_count >= MAX_TOTAL_QUARANTINE_RETRIES:
|
||||
logger.warning(
|
||||
f"[Retry:{trigger}] Task {task_id} hit the absolute retry "
|
||||
f"ceiling ({MAX_TOTAL_QUARANTINE_RETRIES}) — giving up, "
|
||||
f"marking failed"
|
||||
)
|
||||
return False
|
||||
counts[source] = source_count + 1
|
||||
task['quarantine_retry_counts_by_source'] = counts
|
||||
attempt_desc = f"source '{source}' {source_count + 1}/{budget}"
|
||||
else:
|
||||
# Default mode: a single global cap, conservative and predictable.
|
||||
if total_count >= MAX_QUARANTINE_RETRIES:
|
||||
logger.warning(
|
||||
f"[Retry:{trigger}] Task {task_id} hit the quarantine-retry cap "
|
||||
f"({MAX_QUARANTINE_RETRIES}) — giving up, marking failed"
|
||||
)
|
||||
return False
|
||||
attempt_desc = f"{total_count + 1}/{MAX_QUARANTINE_RETRIES}"
|
||||
|
||||
# Mark the quarantined source as used so the re-run won't pick it again.
|
||||
# Uses the same "{username}_{filename}" key the worker dedups against.
|
||||
used_sources = task.get('used_sources', set())
|
||||
used_sources.add(f"{username}_{filename}")
|
||||
task['used_sources'] = used_sources
|
||||
|
||||
task['quarantine_retry_count'] = total_count + 1
|
||||
# Flag the re-run as a quarantine retry so the worker walks the
|
||||
# already-found candidates (cached-first) before re-searching — the
|
||||
# connection was fine, the content was just wrong. Dead-connection /
|
||||
# stuck retries (handled elsewhere in the monitor) deliberately do NOT
|
||||
# set this, so they re-search fresh.
|
||||
task['_quarantine_retry'] = True
|
||||
# Drop the stale download identity + the prior attempt's quarantine link.
|
||||
task.pop('download_id', None)
|
||||
task.pop('username', None)
|
||||
task.pop('filename', None)
|
||||
task.pop('quarantine_entry_id', None)
|
||||
task['status'] = 'searching'
|
||||
task['status_change_time'] = time.time()
|
||||
|
||||
logger.info(
|
||||
f"[Retry:{trigger}] Re-queuing task {task_id} for next-best candidate "
|
||||
f"(attempt {attempt_desc})"
|
||||
)
|
||||
missing_download_executor.submit(_download_track_worker, task_id, batch_id)
|
||||
return True
|
||||
|
||||
|
||||
def _is_release_task(task):
|
||||
ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {}
|
||||
username = task.get('username') or ti.get('username')
|
||||
|
|
|
|||
71
core/downloads/origin.py
Normal file
71
core/downloads/origin.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
"""Download-origin provenance: what TRIGGERED a download.
|
||||
|
||||
The library history records which SERVICE a file came from (Soulseek,
|
||||
YouTube, ...) but not WHY it was downloaded — a watchlist scan, a playlist
|
||||
sync, or a manual click. The origin-history modal (watchlist page / sync
|
||||
page) answers that, so the trigger must be derived once, at the history
|
||||
chokepoint (``record_library_history_download``), from the post-process
|
||||
context.
|
||||
|
||||
Signals, in priority order:
|
||||
1. explicit ``track_info._dl_origin`` / ``_dl_origin_context`` stamps
|
||||
(set at batch-task creation in core/downloads/master.py)
|
||||
2. wishlist provenance riding in ``track_info.source_info`` — watchlist
|
||||
items carry ``watchlist_artist_name``, playlist items ``playlist_name``
|
||||
3. the playlist-folder-mode ``_playlist_name`` thread
|
||||
|
||||
Anything unmatched derives ``(None, '')`` — manual/other downloads are
|
||||
intentionally not classified.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
ORIGIN_WATCHLIST = "watchlist"
|
||||
ORIGIN_PLAYLIST = "playlist"
|
||||
VALID_ORIGINS = (ORIGIN_WATCHLIST, ORIGIN_PLAYLIST)
|
||||
|
||||
|
||||
def _parse_source_info(raw: Any) -> Dict[str, Any]:
|
||||
if isinstance(raw, dict):
|
||||
return raw
|
||||
if isinstance(raw, str) and raw:
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return {}
|
||||
return {}
|
||||
|
||||
|
||||
def derive_download_origin(context: Dict[str, Any]) -> Tuple[Optional[str], str]:
|
||||
"""Return ``(origin, origin_context)`` for a completed download.
|
||||
|
||||
``origin`` is 'watchlist' / 'playlist' / None; ``origin_context`` is the
|
||||
human label (watchlist artist name / playlist name). Never raises."""
|
||||
try:
|
||||
ti = context.get("track_info") or {}
|
||||
if not isinstance(ti, dict):
|
||||
return None, ""
|
||||
si = _parse_source_info(ti.get("source_info"))
|
||||
|
||||
# 1. Explicit stamp wins.
|
||||
origin = ti.get("_dl_origin")
|
||||
if origin in VALID_ORIGINS:
|
||||
return origin, str(ti.get("_dl_origin_context") or "")
|
||||
|
||||
# 2. Wishlist provenance riding in source_info.
|
||||
if si.get("watchlist_artist_name"):
|
||||
return ORIGIN_WATCHLIST, str(si["watchlist_artist_name"])
|
||||
if si.get("playlist_name"):
|
||||
return ORIGIN_PLAYLIST, str(si["playlist_name"])
|
||||
|
||||
# 3. Playlist-folder-mode thread.
|
||||
if ti.get("_playlist_name"):
|
||||
return ORIGIN_PLAYLIST, str(ti["_playlist_name"])
|
||||
|
||||
return None, ""
|
||||
except Exception:
|
||||
return None, ""
|
||||
|
|
@ -19,7 +19,6 @@ a large web_server.py helper that will get its own lift in subsequent PRs.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import traceback
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -27,8 +26,72 @@ from typing import Any, Callable, Optional
|
|||
|
||||
from core.runtime_state import download_batches, download_tasks, tasks_lock
|
||||
from core.spotify_client import Track as SpotifyTrack
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
# Must live under the soulsync.* namespace — handlers only attach there. The
|
||||
# old bare getLogger(__name__) ("core.downloads.task_worker") had no handler,
|
||||
# so the entire [Modal Worker] story — search queries, retry walks, candidate
|
||||
# decisions — never reached app.log.
|
||||
logger = get_logger("downloads.task_worker")
|
||||
|
||||
|
||||
def _resolve_worker_source(username):
|
||||
"""Logical source bucket for a candidate's username (Soulseek peers all
|
||||
collapse to 'soulseek'; streaming sources keep their name). Mirrors the
|
||||
monitor's resolver — imported lazily to avoid an import cycle."""
|
||||
try:
|
||||
from core.downloads.monitor import _resolve_download_source
|
||||
return _resolve_download_source(username)
|
||||
except Exception:
|
||||
return 'soulseek'
|
||||
|
||||
|
||||
def _cand_user_file(candidate):
|
||||
"""Read (username, filename) from a candidate that may be a TrackResult
|
||||
object or a plain dict (tests / cached raw rows)."""
|
||||
if isinstance(candidate, dict):
|
||||
return candidate.get('username'), candidate.get('filename')
|
||||
return getattr(candidate, 'username', None), getattr(candidate, 'filename', None)
|
||||
|
||||
|
||||
def _try_cached_candidates(task_id, batch_id, track, deps):
|
||||
"""Quarantine-retry fast path: attempt the already-found candidates before
|
||||
re-searching anything.
|
||||
|
||||
When a verified-bad file is re-queued, the connection was fine (the file
|
||||
downloaded, it was just the wrong/broken content) — so the next-best pick is
|
||||
almost always already sitting in ``cached_candidates``. Walk those (skipping
|
||||
sources already tried or budget-exhausted) and hand them to the normal
|
||||
download path. Returns True if a download was started; False to fall through
|
||||
to a fresh search (which only happens for a not-yet-searched source).
|
||||
"""
|
||||
with tasks_lock:
|
||||
task = download_tasks.get(task_id)
|
||||
if not task:
|
||||
return False
|
||||
cached = list(task.get('cached_candidates') or [])
|
||||
used = set(task.get('used_sources') or ())
|
||||
exhausted = {str(s).lower() for s in (task.get('exhausted_download_sources') or ())}
|
||||
|
||||
remaining = []
|
||||
for c in cached:
|
||||
uname, fname = _cand_user_file(c)
|
||||
if not uname or not fname:
|
||||
continue
|
||||
if f"{uname}_{fname}" in used:
|
||||
continue
|
||||
if _resolve_worker_source(uname).lower() in exhausted:
|
||||
continue
|
||||
remaining.append(c)
|
||||
|
||||
if not remaining:
|
||||
return False
|
||||
|
||||
logger.info(
|
||||
f"[Modal Worker] Quarantine retry: trying {len(remaining)} cached "
|
||||
f"candidate(s) before re-searching (task {task_id})"
|
||||
)
|
||||
return deps.attempt_download_with_candidates(task_id, remaining, track, batch_id)
|
||||
|
||||
|
||||
def _private_album_bundle_staging_miss_reason(batch_id: Optional[str], deps: Any) -> Optional[str]:
|
||||
|
|
@ -92,6 +155,7 @@ class TaskWorkerDeps:
|
|||
attempt_download_with_candidates: Callable # (task_id, candidates, track, batch_id) -> bool
|
||||
on_download_completed: Callable # (batch_id, task_id, success) -> None
|
||||
recover_worker_slot: Callable # (batch_id, task_id) -> None
|
||||
try_version_mismatch_fallback: Optional[Callable] = None # (title, artist, task_id, batch_id) -> bool
|
||||
|
||||
|
||||
def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorkerDeps) -> None:
|
||||
|
|
@ -206,6 +270,26 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
download_tasks[task_id]['used_sources'] = set()
|
||||
# Else: keep existing used_sources to avoid retrying same failed hosts
|
||||
|
||||
# Cached-first quarantine retry. The monitor sets ``_quarantine_retry``
|
||||
# when a verified-bad file is re-queued; in that case we walk the
|
||||
# already-found candidates before re-searching (the connection was fine,
|
||||
# just the content was wrong). A NON-quarantine entry (fresh download, or
|
||||
# the monitor's dead-connection/stuck retry) instead starts a new search
|
||||
# generation: clear the searched-source memory so each source can be
|
||||
# searched fresh again.
|
||||
with tasks_lock:
|
||||
_t = download_tasks.get(task_id, {})
|
||||
is_quarantine_retry = bool(_t.pop('_quarantine_retry', False))
|
||||
if not is_quarantine_retry:
|
||||
_t.pop('searched_queries', None)
|
||||
if is_quarantine_retry and _try_cached_candidates(task_id, batch_id, track, deps):
|
||||
with tasks_lock:
|
||||
used_filename = download_tasks.get(task_id, {}).get('filename')
|
||||
used_username = download_tasks.get(task_id, {}).get('username')
|
||||
if used_filename and used_username:
|
||||
deps.store_batch_source(batch_id, used_username, used_filename)
|
||||
return
|
||||
|
||||
# 1. Generate multiple search queries (like GUI's generate_smart_search_queries)
|
||||
artist_name = track.artists[0] if track.artists else None
|
||||
track_name = track.name
|
||||
|
|
@ -277,12 +361,40 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
seen.add(query.lower())
|
||||
|
||||
search_queries = unique_queries
|
||||
# Expose the query count so the quarantine-retry budget (exhaustive mode)
|
||||
# can size each source's budget as query_count × retries_per_query.
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['query_count'] = len(search_queries)
|
||||
logger.info(f"[Modal Worker] Generated {len(search_queries)} smart search queries for '{track.name}': {search_queries}")
|
||||
logger.info(f"[Modal Worker] About to start search loop for task {task_id} (track: '{track.name}')")
|
||||
|
||||
# 2. Sequential Query Search (matches GUI's start_search_worker_parallel logic)
|
||||
search_diagnostics = [] # Track what happened per query for detailed error messages
|
||||
all_raw_results = [] # Collect raw results across queries for candidate review modal
|
||||
# Sources whose per-source quarantine-retry budget is spent (exhaustive
|
||||
# mode). The monitor sets this when a source gives up; we exclude those
|
||||
# sources from the hybrid search so the chain falls through to the next
|
||||
# source instead of re-fetching the same exhausted one (e.g. Soulseek
|
||||
# keeps returning fresh wrong peers — once its budget is gone, switch to
|
||||
# HiFi/Tidal/…). See monitor.requeue_quarantined_task_for_retry.
|
||||
#
|
||||
# On a quarantine retry we do NOT exclude a source just because it was
|
||||
# searched once: the first run only ran ONE query before starting a
|
||||
# download, so the later queries (e.g. "artist + album") have never hit
|
||||
# that source yet and may surface the correct upload. Instead we remember
|
||||
# which QUERIES already ran (``searched_queries``) and skip re-running
|
||||
# only those — their candidates are walked via the cached-first path
|
||||
# above. The not-yet-searched queries still search the same source, so
|
||||
# every query is exhausted per source before the chain switches sources.
|
||||
# Fresh / dead-connection runs cleared searched_queries above, so they
|
||||
# search everything again.
|
||||
with tasks_lock:
|
||||
_t = download_tasks.get(task_id, {})
|
||||
_exhausted_sources = [str(s) for s in (_t.get('exhausted_download_sources') or ())]
|
||||
_searched_queries = (
|
||||
set(_t.get('searched_queries') or ()) if is_quarantine_retry else set()
|
||||
)
|
||||
for query_index, query in enumerate(search_queries):
|
||||
# Cancellation check before each query
|
||||
with tasks_lock:
|
||||
|
|
@ -295,6 +407,17 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
return
|
||||
download_tasks[task_id]['current_query_index'] = query_index
|
||||
|
||||
# Cached-first: a query already run last generation has its candidates
|
||||
# sitting in cache (walked above) — re-searching it is the wasteful
|
||||
# repeat the cached-first design removes. Skip it; the not-yet-run
|
||||
# queries below still search this source.
|
||||
if is_quarantine_retry and query in _searched_queries:
|
||||
logger.debug(
|
||||
f"[Modal Worker] Skipping already-searched query '{query}' "
|
||||
f"(candidates served from cache) for task {task_id}"
|
||||
)
|
||||
continue
|
||||
|
||||
logger.debug(f"[Modal Worker] Query {query_index + 1}/{len(search_queries)}: '{query}'")
|
||||
logger.debug(f"About to call soulseek search for task {task_id}")
|
||||
|
||||
|
|
@ -319,9 +442,13 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
_exclude_for_hybrid_album = ['torrent', 'usenet']
|
||||
except Exception as _exc_filter_err:
|
||||
logger.debug("[Modal Worker] album-source-exclusion check failed: %s", _exc_filter_err)
|
||||
# Fold in budget-exhausted sources (per-source quarantine retry).
|
||||
_exclude_sources = list(_exhausted_sources)
|
||||
if _exclude_for_hybrid_album:
|
||||
_exclude_sources.extend(_exclude_for_hybrid_album)
|
||||
# Perform search with timeout
|
||||
tracks_result, _ = deps.run_async(deps.download_orchestrator.search(
|
||||
query, timeout=30, exclude_sources=_exclude_for_hybrid_album,
|
||||
query, timeout=30, exclude_sources=_exclude_sources or None,
|
||||
))
|
||||
logger.debug(f"Search completed for task {task_id}, got {len(tracks_result) if tracks_result else 0} results")
|
||||
|
||||
|
|
@ -330,6 +457,16 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
if task_id not in download_tasks:
|
||||
logger.info(f"[Modal Worker] Task {task_id} was deleted after search returned")
|
||||
return
|
||||
# Remember this query ran so a later quarantine retry skips
|
||||
# re-searching it (its candidates are walked via cached-first).
|
||||
# Recorded regardless of result count: re-running a query is
|
||||
# deterministic, so a query that returned nothing won't return
|
||||
# anything new next time either.
|
||||
_sq = download_tasks[task_id].get('searched_queries')
|
||||
if not isinstance(_sq, set):
|
||||
_sq = set()
|
||||
_sq.add(query)
|
||||
download_tasks[task_id]['searched_queries'] = _sq
|
||||
if download_tasks[task_id]['status'] == 'cancelled':
|
||||
logger.warning(f"[Modal Worker] Task {task_id} cancelled after search returned - ignoring results")
|
||||
# Don't call _on_download_completed for cancelled tasks as it can stop monitoring
|
||||
|
|
@ -352,7 +489,9 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
logger.warning(f"[Modal Worker] Task {task_id} cancelled before processing candidates")
|
||||
# Don't call _on_download_completed for cancelled tasks as it can stop monitoring
|
||||
return
|
||||
# Store candidates for retry fallback (like GUI)
|
||||
# Store candidates for retry fallback (like GUI). A
|
||||
# later quarantine retry walks these via cached-first
|
||||
# and skips re-searching this query (searched_queries).
|
||||
download_tasks[task_id]['cached_candidates'] = candidates
|
||||
|
||||
# Try to download with these candidates
|
||||
|
|
@ -414,7 +553,12 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
# (which was definitely tried). If the first was skipped (unconfigured),
|
||||
# the orchestrator would have tried the second — but trying it again is
|
||||
# harmless (streaming sources return fast).
|
||||
remaining_sources = [s for s in hybrid_order[1:] if s in source_clients and source_clients[s]]
|
||||
_exhausted_lower = {s.lower() for s in _exhausted_sources}
|
||||
remaining_sources = [
|
||||
s for s in hybrid_order[1:]
|
||||
if s in source_clients and source_clients[s]
|
||||
and s.lower() not in _exhausted_lower
|
||||
]
|
||||
if remaining_sources:
|
||||
logger.warning(f"[Hybrid Fallback] Primary source had no valid matches. Trying fallback sources: {remaining_sources}")
|
||||
|
||||
|
|
@ -433,6 +577,9 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
fb_candidates = deps.get_valid_candidates(fb_results, track, fb_query)
|
||||
if fb_candidates:
|
||||
logger.warning(f"[Hybrid Fallback] {fallback_source} found {len(fb_candidates)} valid candidates!")
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['cached_candidates'] = fb_candidates
|
||||
success = deps.attempt_download_with_candidates(task_id, fb_candidates, track, batch_id)
|
||||
if success:
|
||||
return
|
||||
|
|
@ -447,6 +594,15 @@ def download_track_worker(task_id: str, batch_id: Optional[str], deps: TaskWorke
|
|||
|
||||
# If we get here, all search queries and hybrid fallbacks failed
|
||||
logger.warning(f"[Modal Worker] No valid candidates found for '{track.name}' after trying all {len(search_queries)} queries.")
|
||||
|
||||
# Last-resort: quarantine retry with no new candidates — the retry search
|
||||
# exhausted all sources. If the setting is enabled, accept the best
|
||||
# already-quarantined candidate rather than leaving the track missing.
|
||||
if is_quarantine_retry and deps.try_version_mismatch_fallback:
|
||||
_fallback_artist = track.artists[0] if track.artists else ''
|
||||
if deps.try_version_mismatch_fallback(track.name, _fallback_artist, task_id, batch_id):
|
||||
return # fallback re-dispatched; batch completion handled by reprocess thread
|
||||
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'not_found'
|
||||
|
|
|
|||
57
core/enrichment/yield_policy.py
Normal file
57
core/enrichment/yield_policy.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
"""Enrichment-worker yield policy: who pauses while the user's foreground
|
||||
work is running.
|
||||
|
||||
Background enrichment workers share external API budgets with the foreground
|
||||
pipelines — most painfully MusicBrainz (~1 req/s per IP), where a worker
|
||||
grinding through the library can starve the import pipeline's per-track
|
||||
lookups into multi-minute crawls (measured: ~4m15s/track vs the normal ~20s).
|
||||
|
||||
Policy (set with Boulder, 2026-06-06):
|
||||
- downloads active -> EVERYTHING yields (post-processing touches every
|
||||
metadata source: MusicBrainz, Spotify, iTunes,
|
||||
Deezer, Discogs, Last.fm, Genius, ...)
|
||||
- discovery active -> the API-contention five yield (discovery hammers
|
||||
the track-matching sources only)
|
||||
Workers the user explicitly resumed mid-yield are honored upstream (the
|
||||
override set lives in web_server's loop, as does the user-paused bookkeeping).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
# Everything that yields during active downloads. listening-stats (talks only
|
||||
# to the local media server) and repair (user-scheduled job runner, not a
|
||||
# background API drip) intentionally keep running.
|
||||
ALL_YIELD_WORKERS = (
|
||||
'musicbrainz', 'audiodb', 'discogs', 'deezer',
|
||||
'spotify-enrichment', 'itunes-enrichment', 'lastfm-enrichment',
|
||||
'genius-enrichment', 'tidal-enrichment', 'qobuz-enrichment',
|
||||
'amazon-enrichment', 'similar_artists', 'hydrabase', 'soulid',
|
||||
)
|
||||
|
||||
# The sources discovery contends with (track matching APIs).
|
||||
API_CONTENTION_WORKERS = frozenset({
|
||||
'spotify-enrichment', 'itunes-enrichment', 'deezer', 'discogs', 'hydrabase',
|
||||
})
|
||||
|
||||
# Discovery state phases that mean "nothing running" (idle or terminal).
|
||||
_INACTIVE_PHASES = frozenset({'', 'idle', 'discovered', 'error', 'failed', 'cancelled'})
|
||||
|
||||
|
||||
def worker_yield_reason(name: str, downloading: bool, discovering: bool) -> Optional[str]:
|
||||
"""Why ``name`` should be paused right now, or None to run.
|
||||
Downloads outrank discovery so the label reflects the stronger cause."""
|
||||
if name not in ALL_YIELD_WORKERS:
|
||||
return None
|
||||
if downloading:
|
||||
return 'downloads'
|
||||
if discovering and name in API_CONTENTION_WORKERS:
|
||||
return 'discovery'
|
||||
return None
|
||||
|
||||
|
||||
def discovery_state_active(state: dict) -> bool:
|
||||
"""True when a per-playlist discovery state dict represents live work."""
|
||||
phase = str((state or {}).get('phase', '') or '').lower()
|
||||
return phase not in _INACTIVE_PHASES
|
||||
|
|
@ -16,8 +16,24 @@ _rate_limit_backoff = 0 # Extra backoff seconds after 429
|
|||
_rate_limit_until = 0 # Timestamp until which all calls should wait
|
||||
|
||||
|
||||
class GeniusRateLimitedError(requests.exceptions.RequestException):
|
||||
"""Raised IMMEDIATELY while Genius is inside a 429 backoff window.
|
||||
|
||||
Subclasses RequestException so every existing caller (the import
|
||||
pipeline's source lookups, the enrichment worker's per-item guards)
|
||||
already treats it as a plain network failure: log one line, skip
|
||||
Genius, move on. Lyrics/metadata garnish — nothing is allowed to WAIT
|
||||
for it."""
|
||||
|
||||
|
||||
def rate_limited(func):
|
||||
"""Decorator to enforce rate limiting on Genius API calls with exponential backoff on 429"""
|
||||
"""Decorator to enforce rate limiting on Genius API calls.
|
||||
|
||||
The 429 backoff is a fail-fast GATE, not a sleep. The old version
|
||||
slept the backoff in the calling thread — while HOLDING the API lock,
|
||||
so every other Genius caller queued behind it — and then re-raised
|
||||
anyway. The import pipeline measurably napped 2x120s per track
|
||||
("Genius track lookup took 242.4s") for lookups that still failed."""
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
global _last_api_call_time, _rate_limit_backoff, _rate_limit_until
|
||||
|
|
@ -25,11 +41,12 @@ def rate_limited(func):
|
|||
with _api_call_lock:
|
||||
current_time = time.time()
|
||||
|
||||
# If in backoff period from a previous 429, wait it out
|
||||
# Inside a backoff window: fail fast, never wait.
|
||||
if current_time < _rate_limit_until:
|
||||
wait = _rate_limit_until - current_time
|
||||
logger.debug(f"Genius rate limit backoff: waiting {wait:.1f}s")
|
||||
time.sleep(wait)
|
||||
remaining = _rate_limit_until - current_time
|
||||
raise GeniusRateLimitedError(
|
||||
f"Genius in 429 backoff for another {remaining:.0f}s — skipping"
|
||||
)
|
||||
|
||||
time_since_last_call = time.time() - _last_api_call_time
|
||||
if time_since_last_call < MIN_API_INTERVAL:
|
||||
|
|
@ -48,11 +65,11 @@ def rate_limited(func):
|
|||
return result
|
||||
except Exception as e:
|
||||
if "429" in str(e) or "rate limit" in str(e).lower():
|
||||
# Exponential backoff: 30s → 60s → 120s (cap at 120s)
|
||||
# Open the gate: 30s → 60s → 120s (cap). Callers fail fast
|
||||
# against it instead of sleeping here.
|
||||
_rate_limit_backoff = min(120, max(30, _rate_limit_backoff * 2) if _rate_limit_backoff else 30)
|
||||
_rate_limit_until = time.time() + _rate_limit_backoff
|
||||
logger.warning(f"Genius 429 rate limit — backing off {_rate_limit_backoff}s")
|
||||
time.sleep(_rate_limit_backoff)
|
||||
logger.warning(f"Genius 429 rate limit — gating calls for {_rate_limit_backoff}s")
|
||||
raise e
|
||||
return wrapper
|
||||
|
||||
|
|
|
|||
|
|
@ -84,6 +84,26 @@ def resolve_duration_tolerance(value: Any) -> Optional[float]:
|
|||
return parsed
|
||||
|
||||
|
||||
def expected_duration_for_check(expected_ms: Any, is_local_import: bool) -> Optional[int]:
|
||||
"""The expected duration (ms) to run the duration-agreement leg against,
|
||||
or None to skip that leg.
|
||||
|
||||
The duration check exists to catch BROKEN slskd TRANSFERS (truncated /
|
||||
wrong-file downloads). A local/manual import is the user's own already-
|
||||
tagged file being sorted, not a transfer — duration-agreeing it against a
|
||||
re-resolved release is meaningless and produces false quarantines (#804:
|
||||
Coldplay "Yellow" album file, 269s, false-rejected against a *single*
|
||||
edition's 266s). So for local imports we skip the duration leg; the
|
||||
size + mutagen-parse legs still run and catch genuinely broken files.
|
||||
"""
|
||||
if is_local_import:
|
||||
return None
|
||||
try:
|
||||
return int(expected_ms) or None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class IntegrityResult:
|
||||
"""Outcome of an integrity check.
|
||||
|
|
|
|||
|
|
@ -32,10 +32,15 @@ from core.imports.context import (
|
|||
get_import_track_info,
|
||||
normalize_import_context,
|
||||
)
|
||||
from core.imports.file_integrity import check_audio_integrity, resolve_duration_tolerance
|
||||
from core.imports.file_integrity import check_audio_integrity, expected_duration_for_check, resolve_duration_tolerance
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
from core.imports.guards import check_flac_bit_depth, move_to_quarantine
|
||||
from core.imports.quarantine import entry_id_from_quarantined_filename
|
||||
from core.imports.quarantine import (
|
||||
approve_quarantine_entry,
|
||||
entry_id_from_quarantined_filename,
|
||||
list_quarantine_entries,
|
||||
)
|
||||
from core.imports.version_mismatch_fallback import try_accept_version_mismatch_fallback
|
||||
from core.imports.side_effects import (
|
||||
emit_track_downloaded,
|
||||
record_download_provenance,
|
||||
|
|
@ -57,6 +62,7 @@ from core.runtime_state import (
|
|||
)
|
||||
from core.metadata.artwork import download_cover_art
|
||||
from core.metadata.common import wipe_source_tags
|
||||
from core.imports.tag_policy import should_wipe_tags_on_enhancement_failure
|
||||
from core.metadata.enrichment import enhance_file_metadata
|
||||
from core.imports.paths import (
|
||||
build_final_path_for_track,
|
||||
|
|
@ -109,6 +115,30 @@ def _mark_task_quarantined(context: dict, quarantine_path: str | None) -> None:
|
|||
download_tasks[task_id]['quarantine_entry_id'] = entry_id
|
||||
|
||||
|
||||
def _requeue_quarantined_task_for_retry(task_id, batch_id, trigger) -> bool:
|
||||
"""Ask the download monitor to re-run this task on its next-best candidate.
|
||||
|
||||
Thin lazy-import wrapper around
|
||||
``core.downloads.monitor.requeue_quarantined_task_for_retry``. Imported
|
||||
lazily (and defensively) so the post-processing pipeline stays importable on
|
||||
its own — the monitor's retry globals are wired by web_server at startup, and
|
||||
manual-import callers that never started a download worker simply get False.
|
||||
Returns True when a retry was queued (caller must not mark the task failed).
|
||||
"""
|
||||
if not task_id:
|
||||
return False
|
||||
try:
|
||||
from core.downloads.monitor import requeue_quarantined_task_for_retry
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug(f"next-candidate retry unavailable ({trigger}): {exc}")
|
||||
return False
|
||||
try:
|
||||
return requeue_quarantined_task_for_retry(task_id, batch_id, trigger)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.error(f"next-candidate retry failed ({trigger}): {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def import_rejection_reason(context: dict) -> str | None:
|
||||
"""Human-readable reason if post-processing terminally rejected the file
|
||||
(quarantine or race-guard), else ``None`` for a clean import.
|
||||
|
|
@ -178,6 +208,19 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
f"{os.path.basename(existing_final)}"
|
||||
)
|
||||
return
|
||||
# File was intentionally moved to quarantine by a concurrent/earlier
|
||||
# post-process call — this is a stale duplicate dispatch, not a race.
|
||||
# _mark_task_quarantined sets _quarantine_entry_id for every quarantine
|
||||
# trigger (AcoustID, integrity, bit-depth). The quarantine entry and
|
||||
# its retry are already in flight; don't overwrite the task state with
|
||||
# a spurious race-guard failure.
|
||||
if context.get('_quarantine_entry_id'):
|
||||
logger.debug(
|
||||
f"[Race Guard] Source gone but already quarantined (entry %s) — stale duplicate call, ignoring: "
|
||||
f"{os.path.basename(file_path)}",
|
||||
context['_quarantine_entry_id'],
|
||||
)
|
||||
return
|
||||
logger.error(
|
||||
f"[Race Guard] Source file gone and no known destination — marking as failed: "
|
||||
f"{os.path.basename(file_path)}"
|
||||
|
|
@ -211,6 +254,13 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
_expected_duration_ms = int(_duration_track.get("duration_ms", 0) or 0) or None
|
||||
except Exception:
|
||||
_expected_duration_ms = None
|
||||
# Local/manual imports are the user's own files, not slskd transfers —
|
||||
# skip the duration-agreement leg (it would false-quarantine a file that
|
||||
# drifts from a re-resolved release; #804). Size + parse legs still run.
|
||||
_is_local_import = bool(context.get('is_local_import')) if isinstance(context, dict) else False
|
||||
_expected_duration_ms = expected_duration_for_check(_expected_duration_ms, _is_local_import)
|
||||
if _is_local_import and _expected_duration_ms is None:
|
||||
logger.debug("[Integrity] Local import — duration-agreement leg skipped for %s", _basename)
|
||||
|
||||
# User-configurable tolerance override. None = use built-in
|
||||
# auto-scaled defaults (3s normal / 5s for tracks >10min). Set
|
||||
|
|
@ -548,7 +598,13 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
except Exception as meta_err:
|
||||
import traceback
|
||||
pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")
|
||||
wipe_source_tags(file_path)
|
||||
if should_wipe_tags_on_enhancement_failure(has_clean_metadata):
|
||||
wipe_source_tags(file_path)
|
||||
else:
|
||||
logger.warning(
|
||||
"[Metadata] Enhancement failed but import has clean/matched metadata — "
|
||||
"preserving the file's existing tags (not wiping): %s",
|
||||
os.path.basename(file_path))
|
||||
|
||||
logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'")
|
||||
safe_move_file(file_path, final_path)
|
||||
|
|
@ -734,11 +790,22 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
)
|
||||
else:
|
||||
logger.info("[Metadata Input] album_info: None (single track)")
|
||||
_enhance_started = time.time()
|
||||
enhance_file_metadata(file_path, context, artist_context, album_info, runtime=metadata_runtime)
|
||||
# The enhancement block is the pipeline's biggest variable cost
|
||||
# (external source lookups) and used to be a silent multi-minute
|
||||
# gap in the log — always say how long it took.
|
||||
logger.info(f"Metadata enhancement took {time.time() - _enhance_started:.1f}s")
|
||||
except Exception as meta_err:
|
||||
import traceback
|
||||
pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")
|
||||
wipe_source_tags(file_path)
|
||||
if should_wipe_tags_on_enhancement_failure(has_clean_metadata):
|
||||
wipe_source_tags(file_path)
|
||||
else:
|
||||
logger.warning(
|
||||
"[Metadata] Enhancement failed but import has clean/matched metadata — "
|
||||
"preserving the file's existing tags (not wiping): %s",
|
||||
os.path.basename(file_path))
|
||||
|
||||
_enhance_source_info = context.get('track_info', {}).get('source_info') or {}
|
||||
if isinstance(_enhance_source_info, str):
|
||||
|
|
@ -961,6 +1028,53 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta
|
|||
post_process_locks.pop(context_key, None)
|
||||
|
||||
|
||||
def _attempt_version_mismatch_fallback(context, task_id, batch_id, runtime, metadata_runtime):
|
||||
"""Opt-in last resort once AcoustID retries are exhausted: accept the best
|
||||
quarantined version-mismatch candidate for this track instead of failing.
|
||||
|
||||
Delegates the decision + safety rules to
|
||||
``core.imports.version_mismatch_fallback`` (version-mismatch only, all the
|
||||
same matched version, >= min_count, AcoustID-only bypass). Returns True when
|
||||
a candidate was accepted and re-dispatched — the caller then skips marking
|
||||
the task failed.
|
||||
"""
|
||||
try:
|
||||
download_path = docker_resolve_path(
|
||||
config_manager.get('soulseek.download_path', './downloads')
|
||||
)
|
||||
quarantine_dir = os.path.join(download_path, 'ss_quarantine')
|
||||
restore_dir = os.path.join(download_path, 'Transfer')
|
||||
expected_title = get_import_clean_title(context, default='')
|
||||
expected_artist = get_import_clean_artist(context, default='')
|
||||
if not expected_title or not expected_artist:
|
||||
return False
|
||||
|
||||
def _reprocess(restored_path, ctx, tid, bid):
|
||||
new_key = f"vmfallback_{tid}_{int(time.time())}"
|
||||
threading.Thread(
|
||||
target=lambda: post_process_matched_download_with_verification(
|
||||
new_key, ctx, restored_path, tid, bid, runtime, metadata_runtime
|
||||
),
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
return try_accept_version_mismatch_fallback(
|
||||
quarantine_dir=quarantine_dir,
|
||||
restore_dir=restore_dir,
|
||||
expected_title=expected_title,
|
||||
expected_artist=expected_artist,
|
||||
task_id=task_id,
|
||||
batch_id=batch_id,
|
||||
config_get=config_manager.get,
|
||||
list_entries=list_quarantine_entries,
|
||||
approve_entry=approve_quarantine_entry,
|
||||
reprocess=_reprocess,
|
||||
)
|
||||
except Exception as exc:
|
||||
pp_logger.debug("[Version-Mismatch Fallback] skipped due to error: %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime, metadata_runtime=None):
|
||||
on_download_completed = getattr(runtime, "on_download_completed", None)
|
||||
|
||||
|
|
@ -992,6 +1106,21 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
|
||||
if context.get('_acoustid_quarantined'):
|
||||
failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed')
|
||||
# Before failing outright, try the next-best candidate. The wrong
|
||||
# file was just quarantined; re-running the worker (with the bad
|
||||
# source flagged used) picks the runner-up match instead.
|
||||
with matched_context_lock:
|
||||
matched_downloads_context.pop(context_key, None)
|
||||
if _requeue_quarantined_task_for_retry(task_id, batch_id, 'acoustid'):
|
||||
logger.info(
|
||||
f"AcoustID mismatch for task {task_id} — retrying next-best candidate: {failure_msg}"
|
||||
)
|
||||
return
|
||||
# Retries exhausted. Opt-in last resort: if every quarantined
|
||||
# candidate for this track failed the SAME version mismatch (e.g. all
|
||||
# instrumental), accept the best one rather than leaving it missing.
|
||||
if _attempt_version_mismatch_fallback(context, task_id, batch_id, runtime, metadata_runtime):
|
||||
return
|
||||
logger.info(f"File was quarantined by AcoustID verification (task={task_id}): {failure_msg}")
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
|
|
@ -1000,9 +1129,6 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
_eid = context.get('_quarantine_entry_id')
|
||||
if _eid:
|
||||
download_tasks[task_id]['quarantine_entry_id'] = _eid
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
|
|
@ -1044,6 +1170,16 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
# source files failed integrity and were quarantined.
|
||||
if context.get('_integrity_failure_msg'):
|
||||
failure_msg = context.get('_integrity_failure_msg', 'unknown')
|
||||
# Integrity/duration mismatch (truncated transfer, wrong-length cut,
|
||||
# etc). Same treatment as an AcoustID mismatch: quarantine the bad
|
||||
# file and retry the next-best candidate before failing.
|
||||
with matched_context_lock:
|
||||
matched_downloads_context.pop(context_key, None)
|
||||
if _requeue_quarantined_task_for_retry(task_id, batch_id, 'integrity'):
|
||||
logger.info(
|
||||
f"Integrity check failed for task {task_id} — retrying next-best candidate: {failure_msg}"
|
||||
)
|
||||
return
|
||||
logger.error(
|
||||
f"Task {task_id} failed integrity check — marking failed: {failure_msg}"
|
||||
)
|
||||
|
|
@ -1056,9 +1192,6 @@ def post_process_matched_download_with_verification(context_key, context, file_p
|
|||
_eid = context.get('_quarantine_entry_id')
|
||||
if _eid:
|
||||
download_tasks[task_id]['quarantine_entry_id'] = _eid
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -329,6 +329,8 @@ def album_process(runtime: ImportRouteRuntime, data: Dict[str, Any]) -> tuple[Di
|
|||
total_discs=total_discs,
|
||||
source=source,
|
||||
)
|
||||
if isinstance(context, dict):
|
||||
context['is_local_import'] = True # user's own file, not an slskd transfer (#804)
|
||||
|
||||
try:
|
||||
runtime.post_process_matched_download(context_key, context, file_path)
|
||||
|
|
@ -425,6 +427,7 @@ def process_single_import_file(runtime: ImportRouteRuntime, file_info: Dict[str,
|
|||
override_source=manual_match_source,
|
||||
)
|
||||
context = runtime.normalize_import_context(resolved["context"])
|
||||
context['is_local_import'] = True # user's own file, not an slskd transfer (#804)
|
||||
artist_data = runtime.get_import_context_artist(context)
|
||||
track_data = runtime.get_import_track_info(context)
|
||||
final_title = track_data.get("name", title)
|
||||
|
|
|
|||
|
|
@ -246,6 +246,11 @@ def record_library_history_download(context: Dict[str, Any]) -> None:
|
|||
|
||||
acoustid_result = context.get("_acoustid_result", "")
|
||||
|
||||
# What TRIGGERED this download (watchlist scan / playlist sync) —
|
||||
# feeds the origin-history modal. None for manual/unclassified.
|
||||
from core.downloads.origin import derive_download_origin
|
||||
origin, origin_context = derive_download_origin(context)
|
||||
|
||||
db = get_database()
|
||||
db.add_library_history_entry(
|
||||
event_type="download",
|
||||
|
|
@ -261,6 +266,8 @@ def record_library_history_download(context: Dict[str, Any]) -> None:
|
|||
source_filename=source_filename,
|
||||
acoustid_result=acoustid_result,
|
||||
source_artist=source_artist,
|
||||
origin=origin,
|
||||
origin_context=origin_context,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("library history record failed: %s", e)
|
||||
|
|
|
|||
23
core/imports/tag_policy.py
Normal file
23
core/imports/tag_policy.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
"""Tag-preservation policy for the import pipeline.
|
||||
|
||||
Tiny, pure, and deliberately its own seam: it encodes one rule that, when it
|
||||
was wrong, silently destroyed users' metadata (#804). Keeping it here with a
|
||||
regression test stops anyone from re-introducing the unconditional wipe.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def should_wipe_tags_on_enhancement_failure(has_clean_metadata: bool) -> bool:
|
||||
"""Whether to strip the file's tags after metadata enhancement raised.
|
||||
|
||||
Enhancement throwing means NO new tags were written, so wiping just
|
||||
destroys whatever the file already had. For a clean/matched import that's
|
||||
catastrophic — #804: already-tagged files (Bruno Mars, Coldplay) got
|
||||
blanked into an "Unknown Artist" folder by a transient enhancement error.
|
||||
|
||||
So: only wipe for UNMATCHED downloads (no clean/matched metadata), where
|
||||
the tags are likely source junk anyway. NEVER wipe a clean/matched import —
|
||||
preserve the user's existing tags.
|
||||
"""
|
||||
return not bool(has_clean_metadata)
|
||||
190
core/imports/version_mismatch_fallback.py
Normal file
190
core/imports/version_mismatch_fallback.py
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
"""Last-resort acceptance of a version-mismatched download.
|
||||
|
||||
Some tracks simply don't exist on the configured sources in the wanted cut —
|
||||
every copy is, say, the instrumental. The retry engine correctly rejects each
|
||||
one (version mismatch) and eventually gives up, leaving the track missing.
|
||||
|
||||
This module provides an OPT-IN fallback: once a track's retries are fully
|
||||
exhausted, if every quarantined candidate for it failed the *same* way (same
|
||||
matched version, e.g. all ``instrumental``) and there are at least ``min_count``
|
||||
of them, accept the best (first-tried) one rather than failing outright.
|
||||
|
||||
Hard safety rules:
|
||||
- Only ``Version mismatch`` quarantines qualify. Audio/artist mismatches
|
||||
(a genuinely different recording) and integrity/duration failures
|
||||
(truncated or wrong file) never participate.
|
||||
- All qualifying entries must share the same matched version. A mix
|
||||
(instrumental + live) is ambiguous → no acceptance.
|
||||
- The chosen candidate is re-imported with only the AcoustID gate bypassed;
|
||||
the integrity / duration / bit-depth gates still run, so a truncated or
|
||||
corrupt file is never let through by this path.
|
||||
|
||||
``select_version_mismatch_fallback`` is the pure decision core (no I/O) so it
|
||||
can be tested directly. ``try_accept_version_mismatch_fallback`` wires it to the
|
||||
quarantine store + re-import dispatch via injected callables.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
# Must live under the soulsync.* namespace — handlers only attach there, so a
|
||||
# bare getLogger(__name__) sent every line (including the critical "accepting
|
||||
# best quarantined candidate as last resort" warning) into the void instead of
|
||||
# app.log. Same bug class as the prepare.py fix.
|
||||
logger = get_logger("imports.version_fallback")
|
||||
|
||||
# Matches the reason string written by acoustid_verification's version gate:
|
||||
# "Version mismatch: expected '<title>' (<exp>) but file is '<title>' (<got>)"
|
||||
# We only need the matched (<got>) version to test cross-entry consistency.
|
||||
_VERSION_MISMATCH_RE = re.compile(
|
||||
r"^Version mismatch:.*\bbut file is\b.*\(([^()]+)\)\s*$"
|
||||
)
|
||||
|
||||
|
||||
def _norm(text: Optional[str]) -> str:
|
||||
return (text or "").strip().casefold()
|
||||
|
||||
|
||||
def matched_version(reason: Optional[str]) -> Optional[str]:
|
||||
"""Return the matched version token (e.g. ``'instrumental'``) for a
|
||||
Version-mismatch reason string, or None if the reason isn't a version
|
||||
mismatch / can't be parsed."""
|
||||
if not reason:
|
||||
return None
|
||||
m = _VERSION_MISMATCH_RE.match(reason.strip())
|
||||
if not m:
|
||||
return None
|
||||
return m.group(1).strip().casefold()
|
||||
|
||||
|
||||
def select_version_mismatch_fallback(
|
||||
entries: List[Dict[str, Any]],
|
||||
expected_title: str,
|
||||
expected_artist: str,
|
||||
min_count: int,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Pick the quarantine entry to accept as a last resort, or None.
|
||||
|
||||
``entries`` are dicts as produced by
|
||||
:func:`core.imports.quarantine.list_quarantine_entries` (needs ``id``,
|
||||
``reason``, ``expected_track``, ``expected_artist``, ``has_full_context``).
|
||||
|
||||
Returns the chosen entry (the first-tried = oldest = best, by ascending
|
||||
``id`` whose timestamp prefix sorts chronologically) when, for this track,
|
||||
there are at least ``min_count`` version-mismatch entries that all share the
|
||||
same matched version and carry full context. Otherwise None.
|
||||
"""
|
||||
title = _norm(expected_title)
|
||||
artist = _norm(expected_artist)
|
||||
|
||||
candidates = []
|
||||
for e in entries:
|
||||
if not e.get("has_full_context"):
|
||||
continue
|
||||
if _norm(e.get("expected_track")) != title:
|
||||
continue
|
||||
if _norm(e.get("expected_artist")) != artist:
|
||||
continue
|
||||
version = matched_version(e.get("reason"))
|
||||
if version is None:
|
||||
continue
|
||||
candidates.append((version, e))
|
||||
|
||||
if len(candidates) < max(1, int(min_count or 1)):
|
||||
return None
|
||||
|
||||
versions = {v for v, _ in candidates}
|
||||
if len(versions) != 1:
|
||||
# Inconsistent wrong versions (e.g. instrumental + live) — ambiguous,
|
||||
# don't guess which the user wants.
|
||||
return None
|
||||
|
||||
# First tried = oldest = highest-confidence (the retry walks candidates
|
||||
# best-first). The id is a "<date>_<time>_<name>" timestamp prefix, so the
|
||||
# lexicographically smallest id is the earliest attempt.
|
||||
return min((e for _, e in candidates), key=lambda e: e["id"])
|
||||
|
||||
|
||||
def try_accept_version_mismatch_fallback(
|
||||
*,
|
||||
quarantine_dir: str,
|
||||
restore_dir: str,
|
||||
expected_title: str,
|
||||
expected_artist: str,
|
||||
task_id: str,
|
||||
batch_id: Optional[str],
|
||||
config_get: Callable[[str, Any], Any],
|
||||
list_entries: Callable[[str], List[Dict[str, Any]]],
|
||||
approve_entry: Callable[..., Optional[Any]],
|
||||
reprocess: Callable[..., None],
|
||||
) -> bool:
|
||||
"""Orchestrate the last-resort acceptance. Returns True if a candidate was
|
||||
accepted and re-dispatched (caller must then NOT mark the task failed).
|
||||
|
||||
All I/O is injected so this is testable without a filesystem or the
|
||||
web_server pipeline:
|
||||
- ``config_get(key, default)`` — settings lookup.
|
||||
- ``list_entries(quarantine_dir)`` — quarantine.list_quarantine_entries.
|
||||
- ``approve_entry(quarantine_dir, entry_id, restore_dir)`` ->
|
||||
``(restored_path, context, trigger)`` or None — quarantine.approve_quarantine_entry.
|
||||
- ``reprocess(restored_path, context, task_id, batch_id)`` — re-run the
|
||||
verification pipeline on the restored file.
|
||||
"""
|
||||
if not config_get("post_processing.accept_version_mismatch_fallback", False):
|
||||
return False
|
||||
|
||||
try:
|
||||
min_count = int(config_get("post_processing.version_mismatch_min_count", 2))
|
||||
except (TypeError, ValueError):
|
||||
min_count = 2
|
||||
if min_count < 1:
|
||||
min_count = 1
|
||||
|
||||
try:
|
||||
entries = list_entries(quarantine_dir) or []
|
||||
except Exception as exc: # never let the fallback break the failure path
|
||||
logger.debug("[Version-Mismatch Fallback] listing quarantine failed: %s", exc)
|
||||
return False
|
||||
|
||||
chosen = select_version_mismatch_fallback(
|
||||
entries, expected_title, expected_artist, min_count
|
||||
)
|
||||
if not chosen:
|
||||
return False
|
||||
|
||||
version = matched_version(chosen.get("reason")) or "?"
|
||||
try:
|
||||
result = approve_entry(quarantine_dir, chosen["id"], restore_dir)
|
||||
except Exception as exc:
|
||||
logger.error("[Version-Mismatch Fallback] approve failed for %s: %s", chosen["id"], exc)
|
||||
return False
|
||||
if not result:
|
||||
return False
|
||||
|
||||
restored_path, context, _trigger = result
|
||||
if not isinstance(context, dict):
|
||||
return False
|
||||
# Bypass ONLY the AcoustID gate — integrity / duration / bit-depth still run,
|
||||
# so a truncated or genuinely wrong file is still caught.
|
||||
context["_skip_quarantine_check"] = "acoustid"
|
||||
context["_version_mismatch_fallback"] = version
|
||||
context["task_id"] = task_id
|
||||
if batch_id:
|
||||
context["batch_id"] = batch_id
|
||||
|
||||
logger.warning(
|
||||
"[Version-Mismatch Fallback] retries exhausted for '%s - %s'; accepting "
|
||||
"best quarantined candidate (%s, entry %s) as last resort",
|
||||
expected_artist, expected_title, version, chosen["id"],
|
||||
)
|
||||
|
||||
try:
|
||||
reprocess(restored_path, context, task_id, batch_id)
|
||||
except Exception as exc:
|
||||
logger.error("[Version-Mismatch Fallback] re-import dispatch failed: %s", exc)
|
||||
return False
|
||||
return True
|
||||
53
core/library/direct_id.py
Normal file
53
core/library/direct_id.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
"""Direct-ID detection for manual matching (Ashh's request).
|
||||
|
||||
The manual-match modal fuzzy-searches a service and shows the top 8 hits.
|
||||
When the right release isn't in those 8 (common name like "Idols"), the user
|
||||
is stuck. But they often already KNOW the exact ID — so let them paste it
|
||||
and match directly instead of fighting the search ranking.
|
||||
|
||||
This module is the pure detector: given a service + the text the user typed,
|
||||
return the canonical ID if the text *is* an ID (bare or pasted as a URL/URI),
|
||||
else None. No network, no I/O — the caller decides whether to do a direct
|
||||
lookup or fall through to the normal fuzzy search.
|
||||
|
||||
Conservative by design: only return an ID when the text matches that
|
||||
service's ID shape unambiguously. Anything else returns None so a normal
|
||||
text search still runs (pasting "Idols" never looks like an ID).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
# MusicBrainz MBIDs are UUIDs (8-4-4-4-12 hex). Accept a bare UUID or one
|
||||
# embedded in a musicbrainz.org URL (/artist/<id>, /release/<id>, ...).
|
||||
_UUID_RE = re.compile(
|
||||
r"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def extract_direct_id(service: str, entity_type: str, query: str) -> Optional[str]:
|
||||
"""Return the canonical service ID if ``query`` is one, else None.
|
||||
|
||||
``entity_type`` is accepted for future per-type shapes (e.g. Spotify
|
||||
track vs album URLs); MusicBrainz UUIDs are type-agnostic so it's unused
|
||||
there today."""
|
||||
if not query:
|
||||
return None
|
||||
text = query.strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
service = (service or "").strip().lower()
|
||||
|
||||
if service == "musicbrainz":
|
||||
# Bare UUID, or a MB URL — but ONLY a UUID. A search like "Idols"
|
||||
# can't match, so normal search is never hijacked.
|
||||
m = _UUID_RE.search(text)
|
||||
if m and (text == m.group(1) or "musicbrainz.org" in text.lower()):
|
||||
return m.group(1).lower()
|
||||
return None
|
||||
|
||||
return None
|
||||
98
core/library/expired_cleanup.py
Normal file
98
core/library/expired_cleanup.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
"""Pure expiry decision for the Expired Download Cleaner job.
|
||||
|
||||
Decides which origin-tracked downloads (watchlist / playlist, recorded by the
|
||||
Download Origins provenance) are past their retention window and safe to
|
||||
propose for deletion. No DB, no clock, no I/O — the job annotates each entry
|
||||
with the facts (play_count, whether it's still in an active mirror) and this
|
||||
module decides. Fully unit-testable.
|
||||
|
||||
A download is proposed for deletion ONLY when ALL hold:
|
||||
- its origin's retention is set (not 'off') and it's older than that window,
|
||||
- it's NOT protected (still in an actively-mirrored playlist / watched artist),
|
||||
- it has been played FEWER than ``min_plays`` times (default 2 → "played more
|
||||
than once is kept"; play_count is the reliable signal, last_played is not).
|
||||
|
||||
Anything failing a check is kept. Deliberately conservative — this deletes the
|
||||
user's files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
# Retention option → days. 'off' (or anything unmapped) disables that origin.
|
||||
RETENTION_DAYS = {
|
||||
"1w": 7, "2w": 14, "3w": 21, "4w": 28,
|
||||
"2mo": 60, "3mo": 90, "6mo": 180,
|
||||
}
|
||||
RETENTION_OPTIONS = ["off", "1w", "2w", "3w", "4w", "2mo", "3mo", "6mo"]
|
||||
|
||||
|
||||
def retention_cutoff(retention: Optional[str], now: datetime) -> Optional[datetime]:
|
||||
"""Datetime before which an entry of this retention is expired, or None
|
||||
when the retention is off/unknown (origin never auto-cleaned)."""
|
||||
days = RETENTION_DAYS.get((retention or "").strip().lower())
|
||||
if not days:
|
||||
return None
|
||||
return now - timedelta(days=days)
|
||||
|
||||
|
||||
def _parse_ts(value: Any) -> Optional[datetime]:
|
||||
"""Parse a SQLite CURRENT_TIMESTAMP (UTC, no zone) or ISO string."""
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
|
||||
if not value:
|
||||
return None
|
||||
text = str(value).strip().replace(" ", "T")
|
||||
try:
|
||||
dt = datetime.fromisoformat(text)
|
||||
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def is_expired(
|
||||
entry: Dict[str, Any],
|
||||
*,
|
||||
watchlist_retention: Optional[str],
|
||||
playlist_retention: Optional[str],
|
||||
min_plays: int,
|
||||
now: datetime,
|
||||
) -> bool:
|
||||
"""True if this origin entry should be proposed for deletion.
|
||||
|
||||
``entry`` needs: ``origin`` ('watchlist'|'playlist'), ``created_at``,
|
||||
``play_count`` (int, may be None), ``protected`` (bool — still in an active
|
||||
mirror/watch)."""
|
||||
if entry.get("protected"):
|
||||
return False
|
||||
if (entry.get("play_count") or 0) >= max(1, int(min_plays or 1)):
|
||||
return False # listened to enough to keep
|
||||
origin = (entry.get("origin") or "").strip().lower()
|
||||
retention = watchlist_retention if origin == "watchlist" else playlist_retention
|
||||
cutoff = retention_cutoff(retention, now)
|
||||
if cutoff is None:
|
||||
return False # this origin's auto-clean is off
|
||||
created = _parse_ts(entry.get("created_at"))
|
||||
if created is None:
|
||||
return False # unknown age → never delete
|
||||
return created < cutoff
|
||||
|
||||
|
||||
def select_expired(
|
||||
entries: Iterable[Dict[str, Any]],
|
||||
*,
|
||||
watchlist_retention: Optional[str],
|
||||
playlist_retention: Optional[str],
|
||||
min_plays: int = 2,
|
||||
now: Optional[datetime] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Return the subset of ``entries`` that are expired + safe to delete."""
|
||||
now = now or datetime.now(timezone.utc)
|
||||
return [
|
||||
e for e in (entries or [])
|
||||
if is_expired(e, watchlist_retention=watchlist_retention,
|
||||
playlist_retention=playlist_retention,
|
||||
min_plays=min_plays, now=now)
|
||||
]
|
||||
|
|
@ -59,10 +59,65 @@ def _detect_provider(items, client):
|
|||
return 'spotify'
|
||||
|
||||
|
||||
def _mb_direct_lookup(entity_type, mbid):
|
||||
"""Confirm a pasted MusicBrainz MBID by fetching that exact entity.
|
||||
Returns a one-item result list (same shape as the search path) so the
|
||||
modal shows it for confirmation, or [] if the ID doesn't resolve."""
|
||||
if not mb_worker or not mb_worker.mb_service:
|
||||
raise ValueError("MusicBrainz worker not initialized")
|
||||
mb_client = mb_worker.mb_service.mb_client
|
||||
|
||||
if entity_type == 'artist':
|
||||
a = mb_client.get_artist(mbid)
|
||||
if not a:
|
||||
return []
|
||||
extra = a.get('disambiguation') or a.get('country') or a.get('type') or ''
|
||||
return [{'id': a['id'], 'name': a.get('name', ''), 'image': None,
|
||||
'extra': f"Direct ID match{' · ' + extra if extra else ''}"}]
|
||||
|
||||
if entity_type == 'album':
|
||||
# A pasted album ID may be a release OR a release-group — try release
|
||||
# first (what the matcher stores), fall back to release-group.
|
||||
r = mb_client.get_release(mbid) or mb_client.get_release_group(mbid)
|
||||
if not r:
|
||||
return []
|
||||
artists = ', '.join(ac.get('name', '') for ac in r.get('artist-credit', []) if isinstance(ac, dict))
|
||||
cover_url = f"https://coverartarchive.org/release/{r['id']}/front-250" if r.get('id') else None
|
||||
bits = ' · '.join(b for b in (artists, r.get('date', '')) if b)
|
||||
return [{'id': r['id'], 'name': r.get('title', ''), 'image': cover_url,
|
||||
'extra': f"Direct ID match{' · ' + bits if bits else ''}"}]
|
||||
|
||||
if entity_type == 'track':
|
||||
rec = mb_client.get_recording(mbid)
|
||||
if not rec:
|
||||
return []
|
||||
artists = ', '.join(ac.get('name', '') for ac in rec.get('artist-credit', []) if isinstance(ac, dict))
|
||||
return [{'id': rec['id'], 'name': rec.get('title', ''), 'image': None,
|
||||
'extra': f"Direct ID match{' · ' + artists if artists else ''}"}]
|
||||
return []
|
||||
|
||||
|
||||
def _search_service(service, entity_type, query):
|
||||
"""Search a service and return normalized results."""
|
||||
import requests as req_lib
|
||||
|
||||
# Direct-ID fast path (Ashh): if the user pasted an exact service ID
|
||||
# (e.g. a MusicBrainz MBID for a release the top-8 fuzzy search missed),
|
||||
# confirm it by direct lookup and return just that entity. A failed
|
||||
# lookup falls through to the normal search, so a paste that merely
|
||||
# LOOKS like an ID can't dead-end the modal.
|
||||
from core.library.direct_id import extract_direct_id
|
||||
direct_id = extract_direct_id(service, entity_type, query)
|
||||
if direct_id:
|
||||
try:
|
||||
if service == 'musicbrainz':
|
||||
hit = _mb_direct_lookup(entity_type, direct_id)
|
||||
if hit:
|
||||
return hit
|
||||
except Exception as e:
|
||||
logger.debug("Direct-ID lookup failed for %s %s: %s", service, direct_id, e)
|
||||
# fall through to fuzzy search
|
||||
|
||||
if service == 'spotify':
|
||||
if not spotify_enrichment_worker or not spotify_enrichment_worker.client:
|
||||
raise ValueError("Spotify worker not initialized")
|
||||
|
|
|
|||
|
|
@ -213,7 +213,79 @@ def _unresolvable_reason(album_data: dict, primary_source: str, strict_source: b
|
|||
return "No metadata source ID for this album"
|
||||
|
||||
|
||||
def _resolve_source(album_data: dict, primary_source: str, strict_source: bool = False):
|
||||
# #767-2: a walked edition scoring below this against the on-disk files is treated
|
||||
# as the WRONG edition (e.g. a 1-track single vs the 10-track deluxe scores 0.1),
|
||||
# triggering the alternate-edition search. Matches the resolver's min_score.
|
||||
_CANONICAL_FIT_FLOOR = 0.5
|
||||
|
||||
|
||||
def _score_edition_items(file_tracks: List[dict], items: List[dict]) -> float:
|
||||
"""Score a fetched provider tracklist (raw ``items``) against the on-disk
|
||||
``file_tracks`` using the canonical scorer. Normalises the provider's varied
|
||||
shapes (``name``/``title``, ``duration_ms``/``duration`` seconds) first."""
|
||||
from core.metadata.canonical_version import score_release_against_files
|
||||
rel = []
|
||||
for it in items or []:
|
||||
dur = it.get('duration_ms')
|
||||
if dur is None:
|
||||
secs = it.get('duration')
|
||||
dur = int(secs * 1000) if isinstance(secs, (int, float)) and secs else None
|
||||
rel.append({'title': it.get('name') or it.get('title') or '', 'duration_ms': dur})
|
||||
return score_release_against_files(file_tracks, rel) if rel else 0.0
|
||||
|
||||
|
||||
def _resolve_better_edition(album_data, source_ids, file_tracks, primary_source):
|
||||
"""Misfit path: run the canonical resolver WITH alternate-edition expansion and,
|
||||
if it lands on a genuinely different edition than the linked ones, fetch it for
|
||||
organizing. Returns ``(source, album_id, api_album, items, score)`` or ``None``."""
|
||||
from core.metadata.canonical_resolver import (
|
||||
default_fetch_alternates,
|
||||
default_fetch_tracklist,
|
||||
resolve_canonical_for_album,
|
||||
)
|
||||
art_id = str(album_data.get('artist_id') or '')
|
||||
art_name = album_data.get('artist_name') or ''
|
||||
title = album_data.get('title') or ''
|
||||
|
||||
def _alts(source, aid):
|
||||
return default_fetch_alternates(
|
||||
source, aid, artist_id=art_id, artist_name=art_name, album_title=title,
|
||||
)
|
||||
|
||||
try:
|
||||
result = resolve_canonical_for_album(
|
||||
album_source_ids=source_ids,
|
||||
file_tracks=file_tracks,
|
||||
fetch_tracklist=default_fetch_tracklist,
|
||||
fetch_alternates=_alts,
|
||||
source_priority=get_source_priority(primary_source),
|
||||
primary_source=primary_source,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"[Reorganize] canonical resolve raised: {e}")
|
||||
return None
|
||||
if not result:
|
||||
return None
|
||||
linked = source_ids.get(result['source'])
|
||||
if str(result['album_id']) == str(linked or ''):
|
||||
return None # resolver chose a linked edition the walk already considered
|
||||
try:
|
||||
b_album = get_album_for_source(result['source'], result['album_id'])
|
||||
b_items = _normalize_album_tracks(
|
||||
get_album_tracks_for_source(result['source'], result['album_id'])
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"[Reorganize] alternate edition fetch raised: {e}")
|
||||
return None
|
||||
if not b_album or not b_items:
|
||||
return None
|
||||
return result['source'], result['album_id'], b_album, b_items, result.get('score') or 0.0
|
||||
|
||||
|
||||
def _resolve_source(
|
||||
album_data: dict, primary_source: str, strict_source: bool = False,
|
||||
*, file_tracks: Optional[List[dict]] = None, on_better_edition=None,
|
||||
):
|
||||
"""Walk the configured source priority looking for the first source
|
||||
we have an ID for AND that returns a usable tracklist.
|
||||
|
||||
|
|
@ -223,6 +295,11 @@ def _resolve_source(album_data: dict, primary_source: str, strict_source: bool =
|
|||
means "use Spotify or fail", not "use Spotify and silently fall
|
||||
back to Deezer".
|
||||
|
||||
When ``file_tracks`` is supplied (and not ``strict_source``), the walked
|
||||
edition is fit-scored against the on-disk files; a clear misfit triggers an
|
||||
alternate-edition search (#767-2). ``on_better_edition(source, album_id,
|
||||
score)`` is invoked to persist the pin when a better edition is chosen.
|
||||
|
||||
Returns ``(source_name, album_meta, tracks_list)`` or ``(None, None, None)``.
|
||||
"""
|
||||
source_ids = _extract_source_ids(album_data)
|
||||
|
|
@ -251,6 +328,7 @@ def _resolve_source(album_data: dict, primary_source: str, strict_source: bool =
|
|||
else:
|
||||
sources_to_try = get_source_priority(primary_source)
|
||||
|
||||
walk_source = walk_album = walk_items = None
|
||||
for source in sources_to_try:
|
||||
sid = source_ids.get(source) or ''
|
||||
if not sid:
|
||||
|
|
@ -264,8 +342,36 @@ def _resolve_source(album_data: dict, primary_source: str, strict_source: bool =
|
|||
items = _normalize_album_tracks(api_tracks)
|
||||
if not items or not api_album:
|
||||
continue
|
||||
return source, api_album, items
|
||||
walk_source, walk_album, walk_items = source, api_album, items
|
||||
break
|
||||
|
||||
# #767-2: the walk takes the first source we have an ID for, but that ID can
|
||||
# point at the WRONG edition (a single enriched against the deluxe → it'd file
|
||||
# the track as #2 of a 10-track album). With the on-disk tracklist in hand,
|
||||
# fit-score the walked edition; only a clear misfit looks for a better-fitting
|
||||
# edition. Well-fitting albums keep today's exact behavior + make no extra calls.
|
||||
if not strict_source and file_tracks:
|
||||
walk_fit = _score_edition_items(file_tracks, walk_items) if walk_items else 0.0
|
||||
if walk_fit < _CANONICAL_FIT_FLOOR:
|
||||
better = _resolve_better_edition(
|
||||
album_data, source_ids, file_tracks, primary_source,
|
||||
)
|
||||
if better is not None:
|
||||
b_source, b_id, b_album, b_items, b_score = better
|
||||
if on_better_edition:
|
||||
try:
|
||||
on_better_edition(b_source, b_id, b_score)
|
||||
except Exception as e:
|
||||
logger.warning(f"[Reorganize] canonical pin persist failed: {e}")
|
||||
logger.info(
|
||||
"[Reorganize] %s: walked edition fit %.2f below floor — using "
|
||||
"better-fit %s edition %s (fit %.2f)",
|
||||
album_data.get('title', '?'), walk_fit, b_source, b_id, b_score,
|
||||
)
|
||||
return b_source, b_album, b_items
|
||||
|
||||
if walk_source:
|
||||
return walk_source, walk_album, walk_items
|
||||
return None, None, None
|
||||
|
||||
|
||||
|
|
@ -682,6 +788,7 @@ def plan_album_reorganize(
|
|||
strict_source: bool = False,
|
||||
metadata_source: str = 'api',
|
||||
resolve_file_path_fn: Optional[Callable[[Optional[str]], Optional[str]]] = None,
|
||||
on_better_edition: Optional[Callable[[str, str, float], None]] = None,
|
||||
) -> dict:
|
||||
"""Compute the per-track plan for an album reorganize without doing
|
||||
any file IO. Both the actual reorganize orchestrator and the preview
|
||||
|
|
@ -731,8 +838,14 @@ def plan_album_reorganize(
|
|||
except Exception:
|
||||
primary_source = 'deezer'
|
||||
|
||||
# On-disk track shape for the #767-2 fit check (duration stored in ms).
|
||||
file_tracks = [
|
||||
{'duration_ms': t.get('duration') or 0, 'title': t.get('title') or ''}
|
||||
for t in tracks
|
||||
]
|
||||
source, api_album, api_tracks = _resolve_source(
|
||||
album_data, primary_source, strict_source=strict_source
|
||||
album_data, primary_source, strict_source=strict_source,
|
||||
file_tracks=file_tracks, on_better_edition=on_better_edition,
|
||||
)
|
||||
if not source:
|
||||
reason = _unresolvable_reason(album_data, primary_source, strict_source)
|
||||
|
|
@ -1475,12 +1588,23 @@ def reorganize_album(
|
|||
summary['total'] = len(tracks)
|
||||
_emit(total=len(tracks))
|
||||
|
||||
# #767-2: persist the canonical pin when the resolver lands on a better-fit
|
||||
# edition than the linked one, so Track Number Repair + future runs agree and
|
||||
# we don't re-resolve every time. Never overrides a manually-locked pin (the
|
||||
# set_album_canonical SQL guard from #758 enforces that).
|
||||
def _persist_canonical(source, alt_album_id, score):
|
||||
try:
|
||||
db.set_album_canonical(album_id, source, alt_album_id, score)
|
||||
except Exception as e:
|
||||
logger.warning("[Reorganize] set_album_canonical failed: %s", e)
|
||||
|
||||
# Build the per-track plan (same logic the preview uses).
|
||||
plan = plan_album_reorganize(
|
||||
album_data, tracks,
|
||||
primary_source=primary_source, strict_source=strict_source,
|
||||
metadata_source=metadata_source,
|
||||
resolve_file_path_fn=resolve_file_path_fn,
|
||||
on_better_edition=_persist_canonical,
|
||||
)
|
||||
if plan['status'] == 'no_source_id':
|
||||
summary['status'] = 'no_source_id'
|
||||
|
|
|
|||
|
|
@ -28,6 +28,44 @@ class LyricsClient:
|
|||
logger.error(f"Error initializing LRClib API: {e}")
|
||||
self.api = None
|
||||
|
||||
def _fetch_remote_lyrics(self, track_name: str, artist_name: str,
|
||||
album_name: str = None, duration_seconds: int = None):
|
||||
"""LRClib fetch — exact match (with duration) then search fallback.
|
||||
Returns the lyrics_data object or None. Shared by create_lrc_file and
|
||||
has_remote_lyrics so the fetch strategy lives in one place."""
|
||||
if not self.api:
|
||||
return None
|
||||
lyrics_data = None
|
||||
# Strategy 1: Exact match with duration (most accurate)
|
||||
if duration_seconds and album_name:
|
||||
try:
|
||||
lyrics_data = self.api.get_lyrics(
|
||||
track_name=track_name, artist_name=artist_name,
|
||||
album_name=album_name, duration=duration_seconds)
|
||||
except Exception as e:
|
||||
logger.debug(f"Exact match failed: {e}")
|
||||
# Strategy 2: Search without duration
|
||||
if not lyrics_data:
|
||||
try:
|
||||
search_results = self.api.search_lyrics(
|
||||
track_name=track_name, artist_name=artist_name)
|
||||
if search_results:
|
||||
lyrics_data = search_results[0]
|
||||
except Exception as e:
|
||||
logger.debug(f"Search fallback failed: {e}")
|
||||
return lyrics_data
|
||||
|
||||
def has_remote_lyrics(self, track_name: str, artist_name: str,
|
||||
album_name: str = None, duration_seconds: int = None) -> bool:
|
||||
"""True if LRClib has (synced OR plain) lyrics for this track, without
|
||||
writing anything. Powers the Missing Lyrics maintenance job's scan so
|
||||
it only surfaces tracks that are actually fixable (instrumentals return
|
||||
nothing → never flagged)."""
|
||||
data = self._fetch_remote_lyrics(track_name, artist_name, album_name, duration_seconds)
|
||||
if not data:
|
||||
return False
|
||||
return bool(getattr(data, 'synced_lyrics', None) or getattr(data, 'plain_lyrics', None))
|
||||
|
||||
def create_lrc_file(self, audio_file_path: str, track_name: str, artist_name: str,
|
||||
album_name: str = None, duration_seconds: int = None) -> bool:
|
||||
"""
|
||||
|
|
@ -79,37 +117,8 @@ class LyricsClient:
|
|||
|
||||
# Fetch lyrics from LRClib
|
||||
logger.debug(f"Fetching lyrics for: {artist_name} - {track_name}")
|
||||
|
||||
lyrics_data = None
|
||||
|
||||
# Strategy 1: Exact match with duration (most accurate)
|
||||
if duration_seconds and album_name:
|
||||
try:
|
||||
logger.debug(f"Trying exact match: {track_name} by {artist_name} from {album_name} ({duration_seconds}s)")
|
||||
lyrics_data = self.api.get_lyrics(
|
||||
track_name=track_name,
|
||||
artist_name=artist_name,
|
||||
album_name=album_name,
|
||||
duration=duration_seconds
|
||||
)
|
||||
if lyrics_data:
|
||||
logger.debug("Exact match found!")
|
||||
except Exception as e:
|
||||
logger.debug(f"Exact match failed: {e}")
|
||||
|
||||
# Strategy 2: Search without duration
|
||||
if not lyrics_data:
|
||||
try:
|
||||
logger.debug(f"Trying search: {track_name} by {artist_name}")
|
||||
search_results = self.api.search_lyrics(
|
||||
track_name=track_name,
|
||||
artist_name=artist_name
|
||||
)
|
||||
if search_results:
|
||||
lyrics_data = search_results[0] # Take first result
|
||||
logger.debug(f"Search found {len(search_results)} results, using first")
|
||||
except Exception as e:
|
||||
logger.debug(f"Search fallback failed: {e}")
|
||||
lyrics_data = self._fetch_remote_lyrics(
|
||||
track_name, artist_name, album_name, duration_seconds)
|
||||
|
||||
# No lyrics found
|
||||
if not lyrics_data:
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ Two jobs, both reusing the post-processing standard so the user's
|
|||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import errno
|
||||
import os
|
||||
from typing import Iterable
|
||||
|
||||
|
|
@ -111,8 +112,19 @@ def apply_art_to_album_files(
|
|||
musicbrainz_release_id). Existing tags are preserved — only art is added.
|
||||
|
||||
Returns counts; never raises (unwritable/read-only files are skipped).
|
||||
``read_only_fs`` is True when the target filesystem itself rejects writes
|
||||
(a real EROFS from an actual write — a ':ro' volume, a read-only host/NFS/
|
||||
SMB mount, or a read-only underlying fs) so callers can tell the user the
|
||||
real cause instead of a generic failure.
|
||||
|
||||
NOTE: read-only is detected from an ACTUAL write raising EROFS, never from
|
||||
statvfs/mount flags — union/FUSE/network filesystems (mergerfs, rclone,
|
||||
NFS) common in self-hosted setups misreport those flags, which would
|
||||
false-block a perfectly writable library (Sokhi: read-only error with no
|
||||
':ro' in compose). The write itself is the only honest test.
|
||||
"""
|
||||
result = {"embedded": 0, "failed": 0, "skipped": 0, "cover_written": False}
|
||||
result = {"embedded": 0, "failed": 0, "skipped": 0, "cover_written": False,
|
||||
"read_only_fs": False}
|
||||
symbols = get_mutagen_symbols()
|
||||
paths = [p for p in (file_paths or []) if p]
|
||||
if not symbols:
|
||||
|
|
@ -144,6 +156,13 @@ def apply_art_to_album_files(
|
|||
result["failed"] += 1
|
||||
except Exception as exc:
|
||||
# Read-only mounts / permission errors land here — skip, don't crash.
|
||||
# A real EROFS = the mount is read-only; flag it and stop trying the
|
||||
# rest (fast-fail without the unreliable statvfs guess).
|
||||
if getattr(exc, "errno", None) == errno.EROFS:
|
||||
result["read_only_fs"] = True
|
||||
logger.warning("Could not embed art into %s: read-only filesystem", fp)
|
||||
result["failed"] += len(paths) - paths.index(fp) # remaining all fail too
|
||||
break
|
||||
logger.warning("Could not embed art into %s: %s", fp, exc)
|
||||
result["failed"] += 1
|
||||
|
||||
|
|
@ -153,5 +172,7 @@ def apply_art_to_album_files(
|
|||
download_cover_art(album_info, target_dir, context)
|
||||
result["cover_written"] = folder_has_cover_sidecar(target_dir)
|
||||
except Exception as exc:
|
||||
if getattr(exc, "errno", None) == errno.EROFS:
|
||||
result["read_only_fs"] = True
|
||||
logger.warning("cover.jpg write failed for %s: %s", target_dir, exc)
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -132,6 +132,18 @@ def _album_matches(req_artist, req_album, got_artist, got_album) -> bool:
|
|||
return False
|
||||
if not (ra <= ga or ga <= ra):
|
||||
return False
|
||||
# Sokhi: the subset tolerance exists for '(Deluxe)'/'- Remastered'
|
||||
# suffixes, but a NUMERIC difference is a different release, not a
|
||||
# suffix. 'B小町 …CD Vol.4' normalizes to {b,tv,cd,vol,4} — a subset of
|
||||
# Vol.4.5's {b,tv,cd,vol,4,5} — so volume 4 was hanging volume 4.5's
|
||||
# cover. Any number present on only ONE side (volume, part, sequel,
|
||||
# remaster year) rejects the match; the resolver then falls through to
|
||||
# the next source / the download's own art, which is the designed cost
|
||||
# of a false reject here. (Shared rule — the MusicBrainz release matcher
|
||||
# applies the same guard so the MBID-keyed CAA path can't slip either.)
|
||||
from core.text.title_match import numeric_tokens_differ
|
||||
if numeric_tokens_differ(req_album, got_album):
|
||||
return False
|
||||
ta, tg = _significant_tokens(req_artist), _significant_tokens(got_artist)
|
||||
if not ta:
|
||||
return True # requested artist unknown -> album match suffices
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from __future__ import annotations
|
|||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.request
|
||||
from ipaddress import ip_address
|
||||
from urllib.parse import quote, urlparse
|
||||
|
|
@ -289,45 +290,73 @@ def _upgrade_art_url(art_url: str) -> str:
|
|||
elif "coverartarchive.org" in art_url:
|
||||
# MusicBrainz art arrives as Cover Art Archive thumbnails
|
||||
# (/front-250 — see musicbrainz_search._cover_art_url). Upgrade to the
|
||||
# 1200px thumbnail: a huge jump from 240p yet still served by CAA's own
|
||||
# CDN. Deliberately NOT the bare /front original — that redirects to
|
||||
# archive.org, which is flaky (intermittent 500s/timeouts) and can be
|
||||
# multi-MB, nasty to embed in every track. 1200 is the sweet spot of
|
||||
# quality + reliability. `_fetch_art_bytes` falls back to the original
|
||||
# sized URL if /front-1200 is ever refused.
|
||||
return re.sub(r"/front-\d+", "/front-1200", art_url)
|
||||
# bare /front ORIGINAL — native resolution, frequently 3000px+ (#806:
|
||||
# the old /front-1200 cap left MusicBrainz as the one source still
|
||||
# below native while iTunes already shipped 3000x3000 — and bare
|
||||
# /front URLs from release-group lookups bypassed the cap anyway,
|
||||
# so the policy was inconsistent in practice). The original redirects
|
||||
# to archive.org, which can be flaky, so `_fetch_art_bytes` inserts a
|
||||
# /front-1200 midpoint fallback before the original-size URL:
|
||||
# flakiness degrades to the old 1200px behavior, never below it.
|
||||
return re.sub(r"/front(-\d+)?$", "/front", art_url)
|
||||
return art_url
|
||||
|
||||
|
||||
# Negative cache for CAA originals: art is fetched PER TRACK, and the bare
|
||||
# /front original rides archive.org. During an archive.org outage every track
|
||||
# would otherwise pay a 10s timeout before falling back — a 12-track album
|
||||
# would eat +2 minutes. One failure puts originals on cooldown; fetches go
|
||||
# straight to the 1200px CDN (the pre-#806 behavior, full speed) until then.
|
||||
_caa_original_down_until = 0.0
|
||||
_CAA_ORIGINAL_COOLDOWN_S = 600
|
||||
|
||||
|
||||
def _fetch_art_bytes(art_url: str):
|
||||
"""Fetch artwork bytes at the highest resolution the source serves.
|
||||
|
||||
Upgrades the URL via `_upgrade_art_url`, then fetches. If the upgraded
|
||||
(larger) size is refused by the CDN, retry once with the original URL so
|
||||
we never regress vs. the un-upgraded behavior. Empirically the upgrade
|
||||
works for every album tested; the fallback just defends the edge case.
|
||||
Upgrades the URL via `_upgrade_art_url`, then walks a fallback chain so a
|
||||
refused size degrades gracefully and never regresses below the original
|
||||
URL's behavior. For Cover Art Archive that chain is
|
||||
original (/front) -> 1200px CDN thumbnail -> the original sized URL.
|
||||
|
||||
Returns `(image_data, mime_type)` or `(None, None)` on failure.
|
||||
"""
|
||||
global _caa_original_down_until
|
||||
if not art_url:
|
||||
return None, None
|
||||
upgraded = _upgrade_art_url(art_url)
|
||||
try:
|
||||
with urllib.request.urlopen(upgraded, timeout=10) as response:
|
||||
return response.read(), (response.info().get_content_type() or "image/jpeg")
|
||||
except Exception as fetch_err:
|
||||
if upgraded != art_url:
|
||||
logger.info(
|
||||
"Upgraded art URL refused (%s); retrying original size", fetch_err
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(art_url, timeout=10) as response:
|
||||
return response.read(), (response.info().get_content_type() or "image/jpeg")
|
||||
except Exception as retry_err:
|
||||
logger.error("Art fetch failed after fallback: %s", retry_err)
|
||||
return None, None
|
||||
logger.error("Art fetch failed: %s", fetch_err)
|
||||
return None, None
|
||||
is_caa_original = "coverartarchive.org" in upgraded and upgraded.endswith("/front")
|
||||
|
||||
attempts = []
|
||||
if not (is_caa_original and time.time() < _caa_original_down_until):
|
||||
attempts.append(upgraded)
|
||||
if is_caa_original:
|
||||
# Midpoint fallback: the 1200px CDN thumbnail (the pre-#806 behavior),
|
||||
# tried BEFORE the original sized URL so a flaky archive.org degrades
|
||||
# to 1200px — never all the way down to the 250px thumbnail.
|
||||
attempts.append(upgraded + "-1200")
|
||||
if art_url not in attempts:
|
||||
attempts.append(art_url)
|
||||
|
||||
last_err = None
|
||||
for i, candidate in enumerate(attempts):
|
||||
try:
|
||||
with urllib.request.urlopen(candidate, timeout=10) as response:
|
||||
return response.read(), (response.info().get_content_type() or "image/jpeg")
|
||||
except Exception as fetch_err:
|
||||
last_err = fetch_err
|
||||
if is_caa_original and candidate == upgraded:
|
||||
# archive.org refused the original — cool down so the next
|
||||
# tracks of this batch skip straight to the CDN thumbnail.
|
||||
_caa_original_down_until = time.time() + _CAA_ORIGINAL_COOLDOWN_S
|
||||
logger.info(
|
||||
"CAA original refused (%s); using 1200px CDN for the next %d min",
|
||||
fetch_err, _CAA_ORIGINAL_COOLDOWN_S // 60,
|
||||
)
|
||||
elif i < len(attempts) - 1:
|
||||
logger.info("Art URL refused (%s); falling back to next size", fetch_err)
|
||||
logger.error("Art fetch failed after %d attempt(s): %s", len(attempts), last_err)
|
||||
return None, None
|
||||
|
||||
|
||||
def _min_size_art_validator(min_px):
|
||||
|
|
|
|||
|
|
@ -830,6 +830,36 @@ class MetadataCache:
|
|||
logger.error(f"Cache health stats error: {e}")
|
||||
return {}
|
||||
|
||||
def purge_artist_album_lists(self, source: str = 'spotify') -> int:
|
||||
"""One-time repair: delete cached artist-ALBUM-LIST entries.
|
||||
|
||||
Partial watchlist probes (limit=5, max_pages=1) used to be stored in
|
||||
the same unqualified ``<artist>_albums_<types>`` slot the artist
|
||||
detail page reads, so every watchlist artist's page showed only the
|
||||
newest handful of releases. The writer is fixed to skip truncated
|
||||
fetches; this clears the already-poisoned entries (30-day TTL would
|
||||
otherwise keep them for weeks). Lists rebuild lazily on the next
|
||||
artist-page visit. Returns the number of entries removed."""
|
||||
try:
|
||||
db = self._get_db()
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"DELETE FROM metadata_cache_entities "
|
||||
"WHERE source = ? AND entity_type = 'artist' "
|
||||
"AND entity_id LIKE '%\\_albums\\_%' ESCAPE '\\'",
|
||||
(source,),
|
||||
)
|
||||
count = cursor.rowcount
|
||||
conn.commit()
|
||||
return count
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.debug("artist album-list purge failed: %s", e)
|
||||
return 0
|
||||
|
||||
def clear(self, source: str = None, entity_type: str = None) -> int:
|
||||
"""Clear cache entries. Optional filters by source and/or entity_type."""
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -38,6 +38,9 @@ def resolve_canonical_for_album(
|
|||
min_score: float = 0.5,
|
||||
mode: str = MODE_ACTIVE_PREFERRED,
|
||||
primary_source: Optional[str] = None,
|
||||
fetch_alternates: Optional[
|
||||
Callable[[str, str], Optional[List[Dict[str, Any]]]]
|
||||
] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Pick the canonical release for one album, honoring the source-selection mode.
|
||||
|
||||
|
|
@ -68,48 +71,90 @@ def resolve_canonical_for_album(
|
|||
if not file_tracks:
|
||||
return None
|
||||
primary = primary_source or (source_priority[0] if source_priority else None)
|
||||
scored: List[Dict[str, Any]] = [] # every source we actually scored
|
||||
scored: List[Dict[str, Any]] = [] # every edition we actually scored
|
||||
seen: set = set() # (source, album_id) already scored — dedup linked + alternates
|
||||
|
||||
def _score(source: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||
if not source or any(e['source'] == source for e in scored):
|
||||
return next((e for e in scored if e['source'] == source), None)
|
||||
album_id = album_source_ids.get(source)
|
||||
if not album_id:
|
||||
def _score_edition(
|
||||
source: Optional[str], album_id: Any,
|
||||
tracks: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Score one concrete (source, album_id) edition, deduped by that pair.
|
||||
Fetches the tracklist when not pre-supplied. Returns the entry (existing
|
||||
on a repeat) or None when it has no resolvable tracklist."""
|
||||
if not source or not album_id:
|
||||
return None
|
||||
try:
|
||||
tracks = fetch_tracklist(source, str(album_id))
|
||||
except Exception:
|
||||
tracks = None
|
||||
key = (source, str(album_id))
|
||||
if key in seen:
|
||||
return next((e for e in scored if (e['source'], e['album_id']) == key), None)
|
||||
if tracks is None:
|
||||
try:
|
||||
tracks = fetch_tracklist(source, str(album_id))
|
||||
except Exception:
|
||||
tracks = None
|
||||
if not tracks:
|
||||
return None
|
||||
entry = {
|
||||
'source': source, 'album_id': str(album_id),
|
||||
'track_count': len(tracks), 'score': round(score_release_against_files(file_tracks, tracks), 4),
|
||||
'track_count': len(tracks),
|
||||
'score': round(score_release_against_files(file_tracks, tracks), 4),
|
||||
'_tracks': tracks,
|
||||
}
|
||||
scored.append(entry)
|
||||
seen.add(key)
|
||||
return entry
|
||||
|
||||
def _score_linked(source: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||
return _score_edition(source, album_source_ids.get(source)) if source else None
|
||||
|
||||
def _best_clearing_floor(
|
||||
entries: List[Dict[str, Any]],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
best = None
|
||||
for e in entries: # priority-ordered -> strictly-greater = priority tiebreak
|
||||
if best is None or e['score'] > best['score'] + 1e-9:
|
||||
best = e
|
||||
return best if (best and best['score'] >= min_score) else None
|
||||
|
||||
winner: Optional[Dict[str, Any]] = None
|
||||
|
||||
# Active-source modes: try the primary first.
|
||||
# Active-source modes: try the primary's linked edition first.
|
||||
if mode in (MODE_ACTIVE_ONLY, MODE_ACTIVE_PREFERRED):
|
||||
p = _score(primary)
|
||||
p = _score_linked(primary)
|
||||
if p and p['score'] >= min_score:
|
||||
winner = p
|
||||
elif mode == MODE_ACTIVE_ONLY:
|
||||
return None # never consider other sources
|
||||
|
||||
# best_fit, or active_preferred fallback: score the rest and pick the best.
|
||||
if winner is None:
|
||||
# best_fit, or active_preferred fallback: score the rest of the linked editions.
|
||||
if winner is None and mode != MODE_ACTIVE_ONLY:
|
||||
for source in source_priority:
|
||||
_score(source)
|
||||
best = None
|
||||
for e in scored: # source_priority order -> strictly-greater = priority tiebreak
|
||||
if best is None or e['score'] > best['score'] + 1e-9:
|
||||
best = e
|
||||
if best and best['score'] >= min_score:
|
||||
winner = best
|
||||
_score_linked(source)
|
||||
winner = _best_clearing_floor(scored)
|
||||
|
||||
# #767-2 expansion: no LINKED edition cleared the floor — e.g. a 1-track single
|
||||
# linked only to the 10-track deluxe, whose count_fit tanks its score to 0.1.
|
||||
# Fetch the source's OTHER editions of the same release and score those too,
|
||||
# then re-pick. Gated on winner-is-None so a well-fitting library never
|
||||
# triggers a fetch (zero behaviour change + no API cost for the common case).
|
||||
if winner is None and fetch_alternates is not None:
|
||||
if mode == MODE_ACTIVE_ONLY:
|
||||
expand_sources = [primary] if primary else []
|
||||
else:
|
||||
expand_sources = [s for s in source_priority if album_source_ids.get(s)]
|
||||
for s in album_source_ids: # any linked source not in the priority list
|
||||
if s not in expand_sources:
|
||||
expand_sources.append(s)
|
||||
for source in expand_sources:
|
||||
linked_id = album_source_ids.get(source)
|
||||
if not linked_id:
|
||||
continue
|
||||
try:
|
||||
alts = fetch_alternates(source, str(linked_id)) or []
|
||||
except Exception:
|
||||
alts = []
|
||||
for alt in alts:
|
||||
_score_edition(source, alt.get('album_id'), alt.get('tracks'))
|
||||
# active_only stays on-source; other modes re-pick across everything scored.
|
||||
pool = [e for e in scored if e['source'] == primary] if mode == MODE_ACTIVE_ONLY else scored
|
||||
winner = _best_clearing_floor(pool)
|
||||
|
||||
if winner is None:
|
||||
return None
|
||||
|
|
@ -172,6 +217,98 @@ def default_fetch_tracklist(source: str, album_id: str) -> Optional[List[Dict[st
|
|||
return out or None
|
||||
|
||||
|
||||
# Edition/format qualifiers stripped when deciding whether two album titles name
|
||||
# the SAME underlying release (so "Scatterbrain", "Scatterbrain (Deluxe)" and
|
||||
# "Scatterbrain - Single" all collapse to one key). Generous on purpose: the
|
||||
# scorer is the real precision gate, so over-including an edition is harmless —
|
||||
# it just won't win. Under-including is what hides the right single.
|
||||
_EDITION_TOKENS = frozenset({
|
||||
"deluxe", "expanded", "edition", "remaster", "remastered", "single", "ep",
|
||||
"anniversary", "special", "bonus", "explicit", "clean", "version", "extended",
|
||||
"complete", "collectors", "reissue", "original", "standard",
|
||||
})
|
||||
|
||||
|
||||
def _release_name_key(name: str) -> str:
|
||||
"""Normalise an album title to a comparison key for 'same release': lowercase,
|
||||
drop bracketed qualifiers, strip punctuation, and remove edition/format words.
|
||||
Pure — unit-tested directly."""
|
||||
import re
|
||||
if not name:
|
||||
return ""
|
||||
t = str(name).lower()
|
||||
t = re.sub(r"[\(\[].*?[\)\]]", " ", t) # (Deluxe Edition), [Remastered]
|
||||
t = re.sub(r"[^a-z0-9 ]", " ", t) # punctuation -> space ("- Single")
|
||||
toks = [w for w in t.split() if w not in _EDITION_TOKENS]
|
||||
return " ".join(toks)
|
||||
|
||||
|
||||
def _same_release(a: str, b: str) -> bool:
|
||||
"""True when two album titles name the same underlying release (edition-blind)."""
|
||||
ka, kb = _release_name_key(a), _release_name_key(b)
|
||||
return bool(ka) and ka == kb
|
||||
|
||||
|
||||
def default_fetch_alternates(
|
||||
source: str, album_id: str, *,
|
||||
artist_id: str = "", artist_name: str = "", album_title: str = "",
|
||||
max_editions: int = 6,
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Production ``fetch_alternates``: list a release's OTHER editions on a source
|
||||
and return ``[{album_id, tracks}, ...]`` for the canonical resolver to score.
|
||||
|
||||
Strategy: discover the album's artist + title (from supplied context, else one
|
||||
``get_album_for_source`` call), list the artist's albums+singles, keep the ones
|
||||
whose title is the same release (edition-blind), and fetch each one's tracklist.
|
||||
Best-effort throughout — returns ``[]`` on any miss so the resolver simply
|
||||
finds no alternates rather than erroring. Only ever called on the misfit path,
|
||||
so the artist-albums + per-edition fetches don't run for a well-fitting library."""
|
||||
try:
|
||||
from core.metadata.album_tracks import (
|
||||
get_album_for_source,
|
||||
get_artist_albums_for_source,
|
||||
)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
title = album_title
|
||||
a_id, a_name = str(artist_id or ""), str(artist_name or "")
|
||||
if not (title and (a_id or a_name)):
|
||||
try:
|
||||
meta = get_album_for_source(source, str(album_id)) or {}
|
||||
except Exception:
|
||||
meta = {}
|
||||
title = title or (_item_get(meta, "name") or _item_get(meta, "title") or "")
|
||||
a_id = a_id or str(_item_get(meta, "artist_id") or _item_get(meta, "artistId") or "")
|
||||
a_name = a_name or str(_item_get(meta, "artist") or _item_get(meta, "artist_name") or "")
|
||||
if not title or not (a_id or a_name):
|
||||
return []
|
||||
|
||||
try:
|
||||
albums = get_artist_albums_for_source(
|
||||
source, a_id, a_name, album_type="album,single", limit=50,
|
||||
) or []
|
||||
except Exception:
|
||||
albums = []
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
seen_ids: set = set()
|
||||
for alb in albums:
|
||||
if len(out) >= max_editions:
|
||||
break
|
||||
alb_title = _item_get(alb, "name") or _item_get(alb, "title") or ""
|
||||
if not _same_release(title, alb_title):
|
||||
continue
|
||||
alb_id = _item_get(alb, "id") or _item_get(alb, "album_id")
|
||||
if not alb_id or str(alb_id) in seen_ids:
|
||||
continue
|
||||
seen_ids.add(str(alb_id))
|
||||
tracks = default_fetch_tracklist(source, str(alb_id))
|
||||
if tracks:
|
||||
out.append({"album_id": str(alb_id), "tracks": tracks})
|
||||
return out
|
||||
|
||||
|
||||
def _lookup_artist_thumb(db, artist_id) -> Optional[str]:
|
||||
"""Best-effort artist thumb URL by id. Returns None on missing column / any
|
||||
error (the artists table doesn't have thumb_url in every schema)."""
|
||||
|
|
@ -199,6 +336,7 @@ def resolve_and_store_canonical_for_album(
|
|||
album_id,
|
||||
*,
|
||||
fetch_tracklist: Optional[Callable[[str, str], Any]] = None,
|
||||
fetch_alternates: Optional[Callable[[str, str], Any]] = None,
|
||||
source_priority: Optional[List[str]] = None,
|
||||
min_score: float = 0.5,
|
||||
store: bool = True,
|
||||
|
|
@ -230,6 +368,18 @@ def resolve_and_store_canonical_for_album(
|
|||
|
||||
if fetch_tracklist is None:
|
||||
fetch_tracklist = default_fetch_tracklist
|
||||
if fetch_alternates is None:
|
||||
# Default alternates fetcher, primed with the artist/title we already
|
||||
# loaded (no extra get_album call). Only fires on the misfit path.
|
||||
_art_id = str(album_data.get('artist_id') or '')
|
||||
_art_name = album_data.get('artist_name') or ''
|
||||
_title = album_data.get('title') or ''
|
||||
|
||||
def fetch_alternates(source, aid): # noqa: ANN001
|
||||
return default_fetch_alternates(
|
||||
source, aid,
|
||||
artist_id=_art_id, artist_name=_art_name, album_title=_title,
|
||||
)
|
||||
primary_source = None
|
||||
if source_priority is None:
|
||||
try:
|
||||
|
|
@ -243,6 +393,7 @@ def resolve_and_store_canonical_for_album(
|
|||
album_source_ids=source_ids,
|
||||
file_tracks=file_tracks,
|
||||
fetch_tracklist=fetch_tracklist,
|
||||
fetch_alternates=fetch_alternates,
|
||||
source_priority=source_priority,
|
||||
min_score=min_score,
|
||||
mode=mode,
|
||||
|
|
@ -278,4 +429,5 @@ __all__ = [
|
|||
"resolve_canonical_for_album",
|
||||
"resolve_and_store_canonical_for_album",
|
||||
"default_fetch_tracklist",
|
||||
"default_fetch_alternates",
|
||||
]
|
||||
|
|
|
|||
79
core/metadata/release_dates.py
Normal file
79
core/metadata/release_dates.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
"""Release-date gating (#705): keep unreleased tracks out of hot paths.
|
||||
|
||||
Watchlist scans intentionally pick up ANNOUNCED albums — singles drop early,
|
||||
the rest of the tracklist carries a future release date. Two places must not
|
||||
treat those as available:
|
||||
|
||||
- the wishlist auto-processor: searching Soulseek for a track that isn't
|
||||
out yet burns a full search+timeout per track, every cycle
|
||||
- the Fresh Tape / Release Radar builder: future albums got NEGATIVE
|
||||
days_old, which INFLATED their recency score (100 - days*7) above every
|
||||
released track — prereleases weren't just slipping in, they were favored
|
||||
|
||||
Spotify-style dates come in three precisions: YYYY, YYYY-MM, YYYY-MM-DD.
|
||||
The gate is deliberately conservative: a track is "unreleased" only when the
|
||||
date is CONFIDENTLY in the future at its stated precision. Unparseable or
|
||||
missing dates are treated as released (never block on bad data), and a
|
||||
release dated today counts as released — release-day tracks should flow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
|
||||
def is_future_release(release_date_str: Any, today: Optional[date] = None) -> bool:
|
||||
"""True only when ``release_date_str`` is confidently in the future."""
|
||||
if not release_date_str or not isinstance(release_date_str, str):
|
||||
return False
|
||||
today = today or date.today()
|
||||
parts = release_date_str.strip().split('-')
|
||||
try:
|
||||
year = int(parts[0])
|
||||
except (ValueError, IndexError):
|
||||
return False
|
||||
if len(parts) == 1 or not parts[1]:
|
||||
return year > today.year
|
||||
try:
|
||||
month = int(parts[1])
|
||||
except ValueError:
|
||||
return year > today.year
|
||||
if not 1 <= month <= 12:
|
||||
# Garbage month — fall back to year precision, never block on it.
|
||||
return year > today.year
|
||||
if len(parts) == 2 or not parts[2]:
|
||||
return (year, month) > (today.year, today.month)
|
||||
try:
|
||||
day = int(parts[2][:2])
|
||||
except ValueError:
|
||||
return (year, month) > (today.year, today.month)
|
||||
try:
|
||||
return date(year, month, day) > today
|
||||
except ValueError:
|
||||
return (year, month) > (today.year, today.month)
|
||||
|
||||
|
||||
def track_release_date(track: Dict[str, Any]) -> str:
|
||||
"""Pull the release date off a track dict in its common shapes."""
|
||||
if not isinstance(track, dict):
|
||||
return ''
|
||||
album = track.get('album')
|
||||
if isinstance(album, dict) and album.get('release_date'):
|
||||
return str(album['release_date'])
|
||||
return str(track.get('release_date') or '')
|
||||
|
||||
|
||||
def split_released_unreleased(
|
||||
tracks: Iterable[Dict[str, Any]],
|
||||
today: Optional[date] = None,
|
||||
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
||||
"""Partition tracks into (released, unreleased) by their release date."""
|
||||
released: List[Dict[str, Any]] = []
|
||||
unreleased: List[Dict[str, Any]] = []
|
||||
for t in tracks:
|
||||
if is_future_release(track_release_date(t), today=today):
|
||||
unreleased.append(t)
|
||||
else:
|
||||
released.append(t)
|
||||
return released, unreleased
|
||||
|
|
@ -90,11 +90,19 @@ def _bounded_cache_set(cache, key, value, max_entries: int) -> None:
|
|||
|
||||
|
||||
def _call_source_lookup(label: str, func, *args, **kwargs):
|
||||
# Timed: these lookups are where import post-processing silently stalls
|
||||
# when an upstream is sick or contended (measured 4m+/track against a
|
||||
# degraded MusicBrainz). A slow source must NAME itself in the log.
|
||||
started = time.time()
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except _SOURCE_NETWORK_EXCEPTIONS as exc:
|
||||
logger.warning("%s lookup failed (network): %s", label, exc)
|
||||
return None
|
||||
finally:
|
||||
elapsed = time.time() - started
|
||||
if elapsed > 2.0:
|
||||
logger.warning("%s lookup took %.1fs — upstream slow or contended", label, elapsed)
|
||||
|
||||
|
||||
SOURCE_TAG_CONFIG = {
|
||||
|
|
|
|||
|
|
@ -270,6 +270,16 @@ class MusicBrainzService:
|
|||
# Combine scores - cap at 100
|
||||
confidence = min(100, int((title_similarity * 50) + (mb_score / 100 * 30) + artist_bonus + version_bonus))
|
||||
|
||||
# Numeric difference = different release. 'Vol.4' vs 'Vol.4.5'
|
||||
# scores 0.97 string similarity, so a near-identical wrong
|
||||
# volume could win and its MBID then feeds CAA art with NO
|
||||
# downstream validation (CAA is MBID-keyed — Sokhi's wrong
|
||||
# covers). Halving lands any such candidate below the 70 gate
|
||||
# while leaving the exact-volume result untouched.
|
||||
from core.text.title_match import numeric_tokens_differ
|
||||
if numeric_tokens_differ(album_name, mb_title):
|
||||
confidence = int(confidence * 0.5)
|
||||
|
||||
if confidence > best_confidence:
|
||||
best_confidence = confidence
|
||||
best_match = result
|
||||
|
|
@ -448,31 +458,50 @@ class MusicBrainzService:
|
|||
scored.sort(key=lambda x: -x[0])
|
||||
best_score, best_mbid, best_mb_score = scored[0]
|
||||
|
||||
# The genuine cross-script match (romaji↔kanji, latin↔cyrillic)
|
||||
# has near-zero LOCAL similarity, so its COMBINED score sinks
|
||||
# below an unrelated same-script decoy — even though MB itself is
|
||||
# certain. "Sawano Hiroyuki": a decoy entity led on combined
|
||||
# (sim 0.82, mb_score 83, combined 0.82 — just under the 0.85 bar)
|
||||
# while the real artist '澤野弘之' had mb_score 100 but combined
|
||||
# 0.30, sorted last. So evaluate the MB-SCORE leader independently
|
||||
# of the combined ranking for the mb-only escape, not scored[0].
|
||||
mb_leader = max(scored, key=lambda x: x[2]) # (combined, mbid, raw_mb)
|
||||
mb_scores_desc = sorted((x[2] for x in scored), reverse=True)
|
||||
mb_unambiguous = len(mb_scores_desc) < 2 or (mb_scores_desc[0] - mb_scores_desc[1]) >= 5
|
||||
|
||||
# Trust gate. Two ways to pass:
|
||||
# 1. Combined score >= 0.85 (the historical strict bar that
|
||||
# catches same-script matches)
|
||||
# 2. MB's OWN score is very high (>= 95) AND the result is
|
||||
# unambiguous (top result clearly leads). Bridges the
|
||||
# cross-script case where local similarity is near zero
|
||||
# ("Dmitry Yablonsky" vs "Дмитрий Яблонский" sim ~0)
|
||||
# but MB's index found a high-confidence match.
|
||||
# catches same-script matches) → trust the combined leader.
|
||||
# 2. MB's OWN score is very high (>= 95) AND that MB-score leader
|
||||
# is unambiguous → trust IT. Bridges the cross-script case
|
||||
# where local similarity is near zero ("Dmitry Yablonsky" vs
|
||||
# "Дмитрий Яблонский" sim ~0) but MB's index is confident.
|
||||
passes_combined = best_score >= 0.85
|
||||
passes_mb_only = best_mb_score >= 95 and (
|
||||
len(scored) < 2 or (scored[0][2] - scored[1][2]) >= 5
|
||||
)
|
||||
passes_mb_only = mb_leader[2] >= 95 and mb_unambiguous
|
||||
if not (passes_combined or passes_mb_only):
|
||||
logger.debug(
|
||||
"lookup_artist_aliases: best match for %r below trust "
|
||||
"threshold (combined=%.2f, mb_score=%d)",
|
||||
artist_name, best_score, best_mb_score,
|
||||
"threshold (combined=%.2f, best_mb=%d, leader_mb=%d)",
|
||||
artist_name, best_score, best_mb_score, mb_leader[2],
|
||||
)
|
||||
self._save_to_cache('artist_aliases', artist_name, None, None, {'aliases': []}, 0)
|
||||
return []
|
||||
|
||||
# Pick the entity to pull aliases from. Combined-strong matches use
|
||||
# the combined leader; the mb-only escape uses the MB-score leader
|
||||
# (which may differ from scored[0] in the cross-script case above).
|
||||
if passes_combined:
|
||||
chosen_mbid, chosen_conf = best_mbid, best_score
|
||||
else:
|
||||
chosen_mbid, chosen_conf = mb_leader[1], mb_leader[2] / 100.0
|
||||
|
||||
# Ambiguity detection: when 2+ results both score high (within
|
||||
# 0.1 of the best combined), the search hit multiple distinct
|
||||
# artists with similar names. Pulling aliases for one could
|
||||
# produce wrong matches. Skip + cache empty.
|
||||
# produce wrong matches. Skip + cache empty. The unambiguous
|
||||
# MB-score leader (passes_mb_only) is exempt — its decisiveness
|
||||
# was already checked via mb_unambiguous.
|
||||
if len(scored) >= 2 and (scored[0][0] - scored[1][0]) < 0.1 and not passes_mb_only:
|
||||
logger.debug(
|
||||
"lookup_artist_aliases: ambiguous match for %r — top "
|
||||
|
|
@ -482,10 +511,10 @@ class MusicBrainzService:
|
|||
self._save_to_cache('artist_aliases', artist_name, None, None, {'aliases': []}, 0)
|
||||
return []
|
||||
|
||||
aliases = self.fetch_artist_aliases(best_mbid)
|
||||
aliases = self.fetch_artist_aliases(chosen_mbid)
|
||||
self._save_to_cache(
|
||||
'artist_aliases', artist_name, None, best_mbid,
|
||||
{'aliases': aliases}, int(best_score * 100),
|
||||
'artist_aliases', artist_name, None, chosen_mbid,
|
||||
{'aliases': aliases}, int(chosen_conf * 100),
|
||||
)
|
||||
return aliases
|
||||
|
||||
|
|
|
|||
|
|
@ -386,6 +386,30 @@ class NavidromeClient(MediaServerClient):
|
|||
params['size'] = str(size)
|
||||
return f"{self.base_url}/rest/getCoverArt?{urlencode(params)}"
|
||||
|
||||
def build_stream_url(self, track_id, max_bitrate=0) -> Optional[str]:
|
||||
"""Absolute, Subsonic-authenticated ``/rest/stream`` URL for a song.
|
||||
|
||||
Lets SoulSync play a Navidrome library track by proxying the server's
|
||||
own stream API — so playback works WITHOUT mounting the music into the
|
||||
SoulSync container (#809: SoulSync otherwise reads library files off
|
||||
disk, which fails when the user hasn't mirror-mounted the library).
|
||||
``max_bitrate`` 0 = no transcode (original file). Returns None when not
|
||||
connected / no id."""
|
||||
if not self.base_url or not track_id:
|
||||
return None
|
||||
if not self.username or not self.password:
|
||||
return None
|
||||
salt = secrets.token_hex(8)
|
||||
token = hashlib.md5((self.password + salt).encode()).hexdigest()
|
||||
params = {
|
||||
'u': self.username, 't': token, 's': salt,
|
||||
'v': '1.16.1', 'c': 'SoulSync',
|
||||
'id': str(track_id),
|
||||
}
|
||||
if max_bitrate and int(max_bitrate) > 0:
|
||||
params['maxBitRate'] = str(int(max_bitrate))
|
||||
return f"{self.base_url}/rest/stream?{urlencode(params)}"
|
||||
|
||||
# Subsonic endpoints that modify data — use POST to avoid URL length limits
|
||||
_WRITE_ENDPOINTS = frozenset({
|
||||
'createPlaylist', 'updatePlaylist', 'deletePlaylist',
|
||||
|
|
|
|||
|
|
@ -225,7 +225,7 @@ class PersonalizedPlaylistsService:
|
|||
FROM discovery_pool
|
||||
WHERE source = ?
|
||||
AND (spotify_track_id IS NOT NULL OR itunes_track_id IS NOT NULL OR deezer_track_id IS NOT NULL)
|
||||
AND LOWER(artist_name) NOT IN (SELECT LOWER(artist_name) FROM discovery_artist_blacklist)
|
||||
AND LOWER(artist_name) NOT IN (SELECT LOWER(artist_name) FROM discovery_artist_blacklist UNION SELECT LOWER(name) FROM blocklist WHERE entity_type='artist')
|
||||
{owned_clause}
|
||||
{extra_where}
|
||||
ORDER BY {order_by}
|
||||
|
|
@ -818,7 +818,7 @@ class PersonalizedPlaylistsService:
|
|||
source
|
||||
FROM discovery_pool
|
||||
WHERE (artist_name LIKE ? OR track_name LIKE ?) AND source = ?
|
||||
AND LOWER(artist_name) NOT IN (SELECT LOWER(artist_name) FROM discovery_artist_blacklist)
|
||||
AND LOWER(artist_name) NOT IN (SELECT LOWER(artist_name) FROM discovery_artist_blacklist UNION SELECT LOWER(name) FROM blocklist WHERE entity_type='artist')
|
||||
ORDER BY RANDOM()
|
||||
LIMIT ?
|
||||
""", (f'%{category}%', f'%{category}%', active_source, limit))
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ _JOB_MODULES = [
|
|||
'core.repair_jobs.duplicate_detector',
|
||||
'core.repair_jobs.acoustid_scanner',
|
||||
'core.repair_jobs.missing_cover_art',
|
||||
'core.repair_jobs.missing_lyrics',
|
||||
'core.repair_jobs.expired_download_cleaner',
|
||||
'core.repair_jobs.metadata_gap_filler',
|
||||
'core.repair_jobs.album_completeness',
|
||||
'core.repair_jobs.fake_lossless_detector',
|
||||
|
|
|
|||
206
core/repair_jobs/expired_download_cleaner.py
Normal file
206
core/repair_jobs/expired_download_cleaner.py
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
"""Expired Download Cleaner (Boulder) — retention-based cleanup of
|
||||
origin-tracked downloads.
|
||||
|
||||
Watchlist- and playlist-sourced downloads (recorded by the Download Origins
|
||||
provenance) get a per-origin retention window. Past it, a download is proposed
|
||||
for deletion UNLESS it's still in an actively-mirrored playlist / watched
|
||||
artist, or you've played it more than once. By default it creates findings to
|
||||
review; flip ``auto_delete`` to true for hands-off cleanup.
|
||||
|
||||
The expiry decision is the pure core in core.library.expired_cleanup; this job
|
||||
gathers the facts (play_count via DB, active-mirror/watch protection) and
|
||||
deletes via the shared helper the Download Origins delete also conceptually
|
||||
uses (resolve path → remove file → drop track row → drop history row).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from core.library.expired_cleanup import RETENTION_OPTIONS, select_expired
|
||||
from core.library.path_resolver import resolve_library_file_path
|
||||
from core.repair_jobs import register_job
|
||||
from core.repair_jobs.base import JobContext, JobResult, RepairJob
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("repair_jobs.expired_download_cleaner")
|
||||
|
||||
|
||||
def delete_origin_download(db, entry, config_manager) -> dict:
|
||||
"""Delete one origin-tracked download: the file on disk (resolved through
|
||||
the shared resolver), its library track row, and the history entry. A file
|
||||
that refuses deletion keeps its history row and reports the error. Returns
|
||||
{removed, file_deleted, error}."""
|
||||
raw_path = entry.get('file_path') or ''
|
||||
file_deleted = False
|
||||
error = None
|
||||
if raw_path:
|
||||
resolved = resolve_library_file_path(raw_path, config_manager=config_manager)
|
||||
if resolved and os.path.isfile(resolved):
|
||||
try:
|
||||
os.remove(resolved)
|
||||
file_deleted = True
|
||||
except OSError as e:
|
||||
error = str(e)
|
||||
# File gone or deleted → clean up the library track row either way.
|
||||
if error is None:
|
||||
try:
|
||||
db.delete_track_by_file_path(raw_path)
|
||||
except Exception as e:
|
||||
logger.debug("expired cleanup: track row delete failed: %s", e)
|
||||
removed = 0
|
||||
if error is None:
|
||||
removed = db.delete_library_history_rows([entry['id']])
|
||||
return {'removed': removed, 'file_deleted': file_deleted, 'error': error}
|
||||
|
||||
|
||||
@register_job
|
||||
class ExpiredDownloadCleanerJob(RepairJob):
|
||||
job_id = 'expired_download_cleaner'
|
||||
display_name = 'Expired Download Cleaner'
|
||||
description = 'Deletes watchlist/playlist downloads past a retention window (keeps active + played ones)'
|
||||
help_text = (
|
||||
'Cleans up downloads that came in via the watchlist or playlist sync '
|
||||
'(tracked by Download Origins) once they pass a retention window you set '
|
||||
'per origin.\n\n'
|
||||
'A download is only ever proposed for deletion when ALL are true: it is '
|
||||
'older than its origin\'s retention, it is NOT still in a playlist you '
|
||||
'actively mirror (or an artist you still watch), and you have played it '
|
||||
'fewer than the keep-threshold (default: played more than once is kept). '
|
||||
'It only touches downloads recorded from the Download Origins feature '
|
||||
'forward — never your pre-existing or manually-added library.\n\n'
|
||||
'Dry run is ON by default: it only creates findings for you to review '
|
||||
'and delete — nothing is deleted automatically. Turn Dry run OFF for '
|
||||
'hands-off auto-cleanup.\n\n'
|
||||
'Settings:\n'
|
||||
'- Watchlist retention / Playlist retention: off, or a window\n'
|
||||
'- Keep if played at least: play count that protects a track (default 2)\n'
|
||||
'- Dry run: ON = findings only (default); OFF = delete automatically'
|
||||
)
|
||||
icon = 'repair-icon-cleanup'
|
||||
default_enabled = False
|
||||
default_interval_hours = 24
|
||||
default_settings = {
|
||||
'watchlist_retention': 'off',
|
||||
'playlist_retention': 'off',
|
||||
'keep_if_played_at_least': 2,
|
||||
'dry_run': True,
|
||||
}
|
||||
setting_options = {
|
||||
'watchlist_retention': RETENTION_OPTIONS,
|
||||
'playlist_retention': RETENTION_OPTIONS,
|
||||
'dry_run': [True, False],
|
||||
}
|
||||
# Has an auto mode (dry_run off → deletes in-scan). auto_fix is a UI/metadata
|
||||
# flag only — the worker never auto-applies from it; scan() self-manages the
|
||||
# dry_run vs delete decision. Setting True surfaces the Scan → Dry Run /
|
||||
# Auto-fix flow badge (without it the job mislabels as "Scan Only").
|
||||
auto_fix = True
|
||||
|
||||
def _get_settings(self, context: JobContext) -> dict:
|
||||
merged = dict(self.default_settings)
|
||||
if context.config_manager:
|
||||
cfg = context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {}) or {}
|
||||
merged.update(cfg)
|
||||
return merged
|
||||
|
||||
def scan(self, context: JobContext) -> JobResult:
|
||||
result = JobResult()
|
||||
settings = self._get_settings(context)
|
||||
wl = (settings.get('watchlist_retention') or 'off')
|
||||
pl = (settings.get('playlist_retention') or 'off')
|
||||
if wl == 'off' and pl == 'off':
|
||||
return result # nothing configured — no-op
|
||||
try:
|
||||
min_plays = int(settings.get('keep_if_played_at_least', 2))
|
||||
except (TypeError, ValueError):
|
||||
min_plays = 2
|
||||
dry_run = bool(settings.get('dry_run', True))
|
||||
|
||||
candidates = context.db.get_origin_cleanup_candidates()
|
||||
if not candidates:
|
||||
return result
|
||||
|
||||
# Build the "protected" set: still-mirrored playlists + still-watched
|
||||
# artists (by name — what origin_context stores). Case-folded.
|
||||
mirrored_names, watched_names = set(), set()
|
||||
try:
|
||||
for p in (context.db.get_mirrored_playlists() or []):
|
||||
n = (p.get('name') if isinstance(p, dict) else None) or ''
|
||||
if n:
|
||||
mirrored_names.add(n.strip().casefold())
|
||||
except Exception as e:
|
||||
logger.debug("expired cleanup: mirrored-playlist lookup failed: %s", e)
|
||||
try:
|
||||
for a in (context.db.get_watchlist_artists() or []):
|
||||
n = getattr(a, 'artist_name', None) or ''
|
||||
if n:
|
||||
watched_names.add(n.strip().casefold())
|
||||
except Exception as e:
|
||||
logger.debug("expired cleanup: watchlist lookup failed: %s", e)
|
||||
|
||||
for c in candidates:
|
||||
ctx = (c.get('origin_context') or '').strip().casefold()
|
||||
origin = (c.get('origin') or '').strip().lower()
|
||||
c['protected'] = bool(
|
||||
(origin == 'playlist' and ctx and ctx in mirrored_names) or
|
||||
(origin == 'watchlist' and ctx and ctx in watched_names))
|
||||
|
||||
expired = select_expired(candidates, watchlist_retention=wl,
|
||||
playlist_retention=pl, min_plays=min_plays)
|
||||
result.scanned = len(candidates)
|
||||
if context.update_progress:
|
||||
context.update_progress(0, len(expired))
|
||||
|
||||
for i, entry in enumerate(expired):
|
||||
if context.check_stop():
|
||||
return result
|
||||
if not dry_run:
|
||||
try:
|
||||
res = delete_origin_download(context.db, entry, context.config_manager)
|
||||
if res.get('removed') or res.get('file_deleted'):
|
||||
result.auto_fixed += 1
|
||||
except Exception as e:
|
||||
logger.error("expired auto-delete failed for %s: %s", entry.get('title'), e)
|
||||
result.errors += 1
|
||||
elif context.create_finding:
|
||||
try:
|
||||
inserted = context.create_finding(
|
||||
job_id=self.job_id,
|
||||
finding_type='expired_download',
|
||||
severity='info',
|
||||
entity_type='track',
|
||||
entity_id=str(entry.get('id')),
|
||||
file_path=entry.get('file_path'),
|
||||
title=f'Expired: {entry.get("title") or "Unknown"}',
|
||||
description=(f'"{entry.get("title")}" by {entry.get("artist_name") or "Unknown"} '
|
||||
f'— via {entry.get("origin")} ({entry.get("origin_context") or "?"}), '
|
||||
f'past retention, not active, not replayed.'),
|
||||
details={
|
||||
'history_id': entry.get('id'),
|
||||
'file_path': entry.get('file_path'),
|
||||
'title': entry.get('title'),
|
||||
'artist': entry.get('artist_name'),
|
||||
'origin': entry.get('origin'),
|
||||
'origin_context': entry.get('origin_context'),
|
||||
})
|
||||
if inserted:
|
||||
result.findings_created += 1
|
||||
else:
|
||||
result.findings_skipped_dedup += 1
|
||||
except Exception as e:
|
||||
logger.debug("expired finding create failed: %s", e)
|
||||
result.errors += 1
|
||||
if context.update_progress and (i + 1) % 5 == 0:
|
||||
context.update_progress(i + 1, len(expired))
|
||||
|
||||
logger.info("[Expired Cleaner] %d candidates, %d expired (%s)",
|
||||
len(candidates), len(expired),
|
||||
"findings created (dry run)" if dry_run else "auto-deleted")
|
||||
return result
|
||||
|
||||
def estimate_scope(self, context: JobContext) -> int:
|
||||
try:
|
||||
return len(context.db.get_origin_cleanup_candidates())
|
||||
except Exception:
|
||||
return 0
|
||||
|
|
@ -12,6 +12,7 @@ finding. The apply handler lives in repair_worker (_fix_library_retag).
|
|||
|
||||
import os
|
||||
|
||||
from core.library.path_resolver import resolve_library_file_path
|
||||
from core.library.retag_planner import (
|
||||
MODE_FILL_MISSING,
|
||||
MODE_OVERWRITE,
|
||||
|
|
@ -71,14 +72,19 @@ def _run_full_enrich(file_path, full_meta) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def apply_track_plans(track_plans, cover_action=None, cover_url=None, full=False) -> dict:
|
||||
"""Write each plan's tags in place (+ optionally embed/refresh cover art),
|
||||
reusing tag_writer.write_tags_to_file. ``file_path`` on each plan must be a
|
||||
real, reachable path (caller resolves Docker paths). Shared by the dry-run=
|
||||
False auto-apply and the repair_worker fix handler. Never raises.
|
||||
"""
|
||||
def apply_track_plans(track_plans, cover_action=None, cover_url=None, full=False,
|
||||
lyrics_action=False) -> dict:
|
||||
"""Write each plan's tags in place (+ optionally embed/refresh cover art,
|
||||
+ optionally fetch/refresh .lrc lyrics), reusing tag_writer.write_tags_to_file.
|
||||
``file_path`` on each plan must be a real, reachable path (caller resolves
|
||||
Docker paths). Shared by the dry-run=False auto-apply and the repair_worker
|
||||
fix handler. Never raises.
|
||||
|
||||
``lyrics_action`` (Sokhi): when True, after a track's tags are written, fetch
|
||||
+ write its .lrc and embed the lyrics — the same LyricsClient the import
|
||||
pipeline uses (fetch if missing, re-embed if a sidecar already exists)."""
|
||||
import os as _os
|
||||
result = {'written': 0, 'failed': 0, 'skipped': 0, 'cover_written': False}
|
||||
result = {'written': 0, 'failed': 0, 'skipped': 0, 'cover_written': False, 'lyrics_written': 0}
|
||||
embed_cover = bool(cover_action and cover_url)
|
||||
cover_data = None
|
||||
if embed_cover:
|
||||
|
|
@ -89,12 +95,20 @@ def apply_track_plans(track_plans, cover_action=None, cover_url=None, full=False
|
|||
logger.debug("retag cover download failed: %s", e)
|
||||
embed_cover = embed_cover and cover_data is not None
|
||||
|
||||
_lyrics_client = None
|
||||
if lyrics_action:
|
||||
try:
|
||||
from core.lyrics_client import lyrics_client as _lyrics_client
|
||||
except Exception as e:
|
||||
logger.debug("retag lyrics client unavailable: %s", e)
|
||||
_lyrics_client = None
|
||||
|
||||
from core.tag_writer import write_tags_to_file
|
||||
last_dir = None
|
||||
for tp in track_plans or []:
|
||||
fp = tp.get('file_path')
|
||||
db_data = tp.get('db_data') or {}
|
||||
if not fp or not _os.path.isfile(fp) or (not db_data and not embed_cover):
|
||||
if not fp or not _os.path.isfile(fp) or (not db_data and not embed_cover and not _lyrics_client):
|
||||
result['skipped'] += 1
|
||||
continue
|
||||
try:
|
||||
|
|
@ -110,6 +124,28 @@ def apply_track_plans(track_plans, cover_action=None, cover_url=None, full=False
|
|||
logger.warning("retag write failed for %s: %s", fp, e)
|
||||
result['failed'] += 1
|
||||
|
||||
# Lyrics: fetch/refresh the .lrc for this track (independent of tag write
|
||||
# success — a track with no tag changes may still be missing lyrics).
|
||||
# Query metadata comes from the plan's READ-only lyrics_meta (never
|
||||
# db_data, so nothing here can leak into a tag write). Falls back to
|
||||
# db_data for plans that predate lyrics_meta.
|
||||
if _lyrics_client:
|
||||
lm = tp.get('lyrics_meta') or {}
|
||||
title = lm.get('title') or db_data.get('title') or ''
|
||||
artist = lm.get('artist') or db_data.get('artist') or ''
|
||||
if title:
|
||||
try:
|
||||
dur = lm.get('duration') or db_data.get('duration')
|
||||
wrote = _lyrics_client.create_lrc_file(
|
||||
fp, title, artist,
|
||||
album_name=lm.get('album') or db_data.get('album'),
|
||||
duration_seconds=int(dur) if dur else None,
|
||||
)
|
||||
if wrote:
|
||||
result['lyrics_written'] += 1
|
||||
except Exception as e:
|
||||
logger.debug("retag lyrics fetch failed for %s: %s", fp, e)
|
||||
|
||||
if cover_action and cover_data and last_dir:
|
||||
try:
|
||||
cover_path = _os.path.join(last_dir, 'cover.jpg')
|
||||
|
|
@ -223,12 +259,14 @@ class LibraryRetagJob(RepairJob):
|
|||
'depth': 'light',
|
||||
'mode': MODE_OVERWRITE,
|
||||
'cover_art': 'replace',
|
||||
'lyrics': 'skip',
|
||||
'source': 'auto',
|
||||
}
|
||||
setting_options = {
|
||||
'depth': ['light', 'full'],
|
||||
'mode': [MODE_OVERWRITE, MODE_FILL_MISSING],
|
||||
'cover_art': ['replace', 'fill_missing', 'skip'],
|
||||
'lyrics': ['fetch', 'skip'],
|
||||
'source': ['auto', 'spotify', 'itunes', 'deezer', 'musicbrainz'],
|
||||
}
|
||||
auto_fix = True
|
||||
|
|
@ -251,6 +289,7 @@ class LibraryRetagJob(RepairJob):
|
|||
settings = self._get_settings(context)
|
||||
mode = settings.get('mode', MODE_OVERWRITE)
|
||||
cover_mode = settings.get('cover_art', 'replace')
|
||||
lyrics_action = (settings.get('lyrics', 'skip') or 'skip').lower() == 'fetch'
|
||||
dry_run = settings.get('dry_run', True)
|
||||
depth = settings.get('depth', 'light')
|
||||
source_order = self._source_order(settings)
|
||||
|
|
@ -300,7 +339,8 @@ class LibraryRetagJob(RepairJob):
|
|||
|
||||
try:
|
||||
self._scan_album(context, result, album_id, album_title, artist_name,
|
||||
source, album_source_id, mode, cover_mode, dry_run, depth)
|
||||
source, album_source_id, mode, cover_mode, dry_run, depth,
|
||||
lyrics_action=lyrics_action)
|
||||
except Exception as e:
|
||||
logger.debug("Library re-tag: album %s failed: %s", album_id, e)
|
||||
result.errors += 1
|
||||
|
|
@ -312,7 +352,8 @@ class LibraryRetagJob(RepairJob):
|
|||
return result
|
||||
|
||||
def _scan_album(self, context, result, album_id, album_title, artist_name,
|
||||
source, album_source_id, mode, cover_mode, dry_run=True, depth='light'):
|
||||
source, album_source_id, mode, cover_mode, dry_run=True, depth='light',
|
||||
lyrics_action=False):
|
||||
# Local tracks for this album.
|
||||
with context.db._get_connection() as conn:
|
||||
cur = conn.cursor()
|
||||
|
|
@ -366,43 +407,90 @@ class LibraryRetagJob(RepairJob):
|
|||
cover_action = self._cover_action(cover_mode, cover_url, library_tracks)
|
||||
|
||||
pairs = match_source_tracks(source_tracks, library_tracks)
|
||||
download_folder = (context.config_manager.get('soulseek.download_path', '')
|
||||
if context.config_manager else None)
|
||||
track_plans = []
|
||||
unmatched = []
|
||||
unreachable = 0
|
||||
for lib, src in pairs:
|
||||
# Resolve container/host path mismatches the same way the apply
|
||||
# handler does. The old bare os.path.isfile() on the raw DB path
|
||||
# failed for EVERY track on path-mapped setups (Docker mounts), so
|
||||
# cover-mode scans produced "(0 track(s))" findings that the apply
|
||||
# then rejected with "No tracks to re-tag in finding".
|
||||
rp = resolve_library_file_path(
|
||||
lib['file_path'],
|
||||
transfer_folder=getattr(context, 'transfer_folder', None),
|
||||
download_folder=download_folder,
|
||||
config_manager=context.config_manager,
|
||||
)
|
||||
if not rp:
|
||||
unreachable += 1
|
||||
continue # genuinely unreachable from this process
|
||||
if src is None:
|
||||
unmatched.append(lib['title'] or os.path.basename(lib['file_path']))
|
||||
# No source match means no re-tag — but album cover art and/or
|
||||
# lyrics still apply to the file, so those modes include an
|
||||
# art/lyrics-only plan (empty db_data → apply writes NO tags).
|
||||
if cover_action or lyrics_action:
|
||||
plan_row = {
|
||||
'file_path': rp,
|
||||
'track_id': lib['id'],
|
||||
'title': lib['title'],
|
||||
'changes': [],
|
||||
'db_data': {}, # never write tags for an unmatched track
|
||||
}
|
||||
if lyrics_action:
|
||||
# READ-only metadata for the lyrics query — kept OUT of
|
||||
# db_data so it can never be written as tags.
|
||||
plan_row['lyrics_meta'] = {
|
||||
'title': lib.get('title'), 'artist': artist_name,
|
||||
'album': album_title}
|
||||
track_plans.append(plan_row)
|
||||
continue
|
||||
if not os.path.isfile(lib['file_path']):
|
||||
continue # not reachable at the stored path — skip (apply resolves paths)
|
||||
current = _read_current_tags(lib['file_path'])
|
||||
current = _read_current_tags(rp)
|
||||
plan = plan_track(current, src, album_meta, mode=mode)
|
||||
# Include a track when its tags change, OR when there's a cover action
|
||||
# to apply to it (db_data may be empty — apply embeds art either way).
|
||||
if plan['changes'] or cover_action:
|
||||
# Include a track when its tags change, OR there's a cover action,
|
||||
# OR lyrics are being fetched (db_data may be empty — apply still
|
||||
# embeds art / writes the .lrc).
|
||||
if plan['changes'] or cover_action or lyrics_action:
|
||||
db_data = plan['db_data']
|
||||
_add_source_ids(db_data, source, album_source_id, src)
|
||||
tp = {
|
||||
'file_path': lib['file_path'],
|
||||
'file_path': rp,
|
||||
'track_id': lib['id'],
|
||||
'title': lib['title'],
|
||||
'changes': plan['changes'],
|
||||
'db_data': db_data,
|
||||
}
|
||||
if lyrics_action:
|
||||
# READ-only lyrics query metadata (never written as tags).
|
||||
tp['lyrics_meta'] = {
|
||||
'title': lib.get('title'), 'artist': artist_name,
|
||||
'album': album_title}
|
||||
if depth == 'full':
|
||||
tp['full_meta'] = _build_full_meta(
|
||||
db_data, src, album_title, artist_name, lib['title'])
|
||||
track_plans.append(tp)
|
||||
|
||||
tag_change_tracks = sum(1 for tp in track_plans if tp['changes'])
|
||||
if not tag_change_tracks and not cover_action:
|
||||
if (not tag_change_tracks and not cover_action and not lyrics_action) or not track_plans:
|
||||
# Nothing actionable. The second clause covers cover-action albums
|
||||
# where no track is reachable/included — creating a finding there
|
||||
# gives an unappliable "(0 track(s))" entry.
|
||||
if cover_action and not track_plans:
|
||||
logger.debug(
|
||||
"Library re-tag: album %s skipped — cover action but no usable tracks "
|
||||
"(%d unreachable, %d unmatched)", album_id, unreachable, len(unmatched))
|
||||
result.skipped += 1
|
||||
return
|
||||
|
||||
# Not dry-run: apply the tags in place now (the track paths were already
|
||||
# isfile-checked above) and count it as an auto-fix — no finding.
|
||||
if not dry_run:
|
||||
res = apply_track_plans(track_plans, cover_action, cover_url, full=(depth == 'full'))
|
||||
if res['written'] or res['cover_written']:
|
||||
res = apply_track_plans(track_plans, cover_action, cover_url, full=(depth == 'full'),
|
||||
lyrics_action=lyrics_action)
|
||||
if res['written'] or res['cover_written'] or res.get('lyrics_written'):
|
||||
result.auto_fixed += 1
|
||||
else:
|
||||
result.errors += 1
|
||||
|
|
@ -419,7 +507,14 @@ class LibraryRetagJob(RepairJob):
|
|||
desc = (f'Album "{album_title}" by {artist_name or "Unknown"} would be re-tagged from '
|
||||
f'{source} ({", ".join(summary_bits)}).')
|
||||
if unmatched:
|
||||
desc += f' {len(unmatched)} track(s) could not be matched to the source and are left untouched.'
|
||||
desc += (f' {len(unmatched)} track(s) could not be matched to the source — '
|
||||
f'tags left untouched{" (cover art still applied)" if cover_action else ""}.')
|
||||
if unreachable:
|
||||
desc += f' {unreachable} track(s) not reachable on disk and skipped.'
|
||||
|
||||
# Cover-only findings say so instead of the puzzling "(0 track(s))".
|
||||
title_what = (f'{tag_change_tracks} track(s)' if tag_change_tracks
|
||||
else f'cover art, {len(track_plans)} track(s)')
|
||||
|
||||
if context.create_finding:
|
||||
inserted = context.create_finding(
|
||||
|
|
@ -429,7 +524,7 @@ class LibraryRetagJob(RepairJob):
|
|||
entity_type='album',
|
||||
entity_id=str(album_id),
|
||||
file_path=None,
|
||||
title=f'Re-tag: {album_title or "Unknown"} ({tag_change_tracks} track(s))',
|
||||
title=f'Re-tag: {album_title or "Unknown"} ({title_what})',
|
||||
description=desc,
|
||||
details={
|
||||
'album_id': album_id,
|
||||
|
|
@ -442,6 +537,7 @@ class LibraryRetagJob(RepairJob):
|
|||
'cover_mode': cover_mode,
|
||||
'cover_url': cover_url,
|
||||
'cover_action': cover_action,
|
||||
'lyrics_action': lyrics_action,
|
||||
'tracks': track_plans, # each carries its db_data for a deterministic apply
|
||||
'unmatched': unmatched,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -203,6 +203,12 @@ class MissingCoverArtJob(RepairJob):
|
|||
log_line=f'Found art: {title or "Unknown"}',
|
||||
log_type='success'
|
||||
)
|
||||
# Also search for an artist image so the finding can offer it as
|
||||
# an independently-applyable target (Pache711). Searched even
|
||||
# when the artist already has art, so a wrong existing image can
|
||||
# be swapped; the UI only surfaces it when it differs from the
|
||||
# current one.
|
||||
found_artist_url = self._find_artist_art(artist_name, source_priority)
|
||||
# Create finding for user to approve
|
||||
if context.create_finding:
|
||||
try:
|
||||
|
|
@ -222,6 +228,13 @@ class MissingCoverArtJob(RepairJob):
|
|||
'found_artwork_url': artwork_url,
|
||||
'spotify_album_id': spotify_album_id,
|
||||
'artist_thumb_url': artist_thumb or None,
|
||||
# Found artist image (None if none matched, or it
|
||||
# equals the current one — nothing to offer then).
|
||||
'found_artist_url': (
|
||||
found_artist_url
|
||||
if found_artist_url and found_artist_url != artist_thumb
|
||||
else None
|
||||
),
|
||||
# Where the files live + what was missing, so the
|
||||
# apply can embed into the audio + write cover.jpg.
|
||||
'album_folder': os.path.dirname(rep_path) if rep_path else None,
|
||||
|
|
@ -287,6 +300,34 @@ class MissingCoverArtJob(RepairJob):
|
|||
logger.debug("%s art lookup failed for '%s': %s", source.capitalize(), title, e)
|
||||
return None
|
||||
|
||||
def _find_artist_art(self, artist_name, source_priority):
|
||||
"""Search the configured sources for an artist image, in priority
|
||||
order. Returns the first confidently name-matched artist image URL,
|
||||
or None. Mirrors _try_source but for artists (Pache711: let the
|
||||
Cover Art Filler offer artist art as its own fixable target)."""
|
||||
if not artist_name:
|
||||
return None
|
||||
for source in source_priority:
|
||||
client = get_client_for_source(source)
|
||||
if not client or not hasattr(client, 'search_artists'):
|
||||
continue
|
||||
try:
|
||||
for res in (client.search_artists(artist_name, limit=5) or []):
|
||||
r_name = getattr(res, 'name', None)
|
||||
if isinstance(res, dict):
|
||||
r_name = res.get('name')
|
||||
# Exact significant-word match — never hang a wrong artist
|
||||
# photo on someone just because the search was fuzzy.
|
||||
if not r_name or _name_tokens(r_name) != _name_tokens(artist_name):
|
||||
continue
|
||||
url = self._extract_artwork_url(res)
|
||||
if url:
|
||||
return url
|
||||
except Exception as e:
|
||||
logger.debug("%s artist-art lookup failed for '%s': %s",
|
||||
source.capitalize(), artist_name, e)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _result_title_artist(item):
|
||||
"""Pull (title, artist) from a search result that may be a dict or an
|
||||
|
|
|
|||
197
core/repair_jobs/missing_lyrics.py
Normal file
197
core/repair_jobs/missing_lyrics.py
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
"""Missing Lyrics maintenance job (Sokhi) — the lyrics sibling of the Cover
|
||||
Art Filler.
|
||||
|
||||
Scans the library for tracks that have no ``.lrc`` sidecar, asks LRClib
|
||||
whether lyrics actually exist for them (so instrumentals/interludes that
|
||||
genuinely have no lyrics are never flagged — Option A), and creates a finding
|
||||
for each fixable track. Applying a finding fetches + writes the ``.lrc`` and
|
||||
embeds the lyrics, reusing the same LyricsClient the import pipeline uses.
|
||||
|
||||
Mirrors MissingCoverArtJob's "only surface actionable findings" design.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from core.repair_jobs import register_job
|
||||
from core.repair_jobs.base import JobContext, JobResult, RepairJob
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("repair_jobs.missing_lyrics")
|
||||
|
||||
|
||||
def _has_lrc_sidecar(file_path: str) -> bool:
|
||||
"""True if a .lrc (or .txt lyrics) sidecar already sits next to the file."""
|
||||
if not file_path:
|
||||
return False
|
||||
base = os.path.splitext(file_path)[0]
|
||||
return os.path.exists(base + '.lrc') or os.path.exists(base + '.txt')
|
||||
|
||||
|
||||
@register_job
|
||||
class MissingLyricsJob(RepairJob):
|
||||
job_id = 'missing_lyrics'
|
||||
display_name = 'Lyrics Filler'
|
||||
description = 'Finds tracks with no .lrc lyrics and fetches synced lyrics from LRClib'
|
||||
help_text = (
|
||||
'Scans your library for tracks that have no .lrc lyrics file next to them. '
|
||||
'For each one it asks LRClib whether lyrics actually exist — tracks with no '
|
||||
'lyrics available (instrumentals, interludes) are skipped, so only fixable '
|
||||
'tracks are surfaced.\n\n'
|
||||
'When lyrics are found, a finding is created so you can review and apply it. '
|
||||
'Applying writes a synced .lrc sidecar (or plain text if no synced version '
|
||||
'exists) and embeds the lyrics in the file — the same way the import pipeline '
|
||||
'and the Library Re-tag tool do.\n\n'
|
||||
'Requires LRClib to be enabled (Settings > Metadata Enhancement).'
|
||||
)
|
||||
icon = 'repair-icon-lyrics'
|
||||
default_enabled = False
|
||||
default_interval_hours = 48
|
||||
default_settings = {}
|
||||
auto_fix = False
|
||||
|
||||
def scan(self, context: JobContext) -> JobResult:
|
||||
result = JobResult()
|
||||
|
||||
# Respect the same LRClib master toggle the import pipeline uses.
|
||||
if context.config_manager and context.config_manager.get(
|
||||
'metadata_enhancement.lrclib_enabled', True) is False:
|
||||
logger.info("[Lyrics Filler] LRClib disabled in settings — skipping scan")
|
||||
return result
|
||||
|
||||
try:
|
||||
from core.lyrics_client import lyrics_client
|
||||
except Exception as e:
|
||||
logger.warning("[Lyrics Filler] lyrics client unavailable: %s", e)
|
||||
return result
|
||||
if not getattr(lyrics_client, 'api', None):
|
||||
logger.info("[Lyrics Filler] LRClib API not available — skipping scan")
|
||||
return result
|
||||
|
||||
rows = []
|
||||
conn = None
|
||||
try:
|
||||
conn = context.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT t.id, t.title, ar.name, al.title, t.file_path, t.duration
|
||||
FROM tracks t
|
||||
LEFT JOIN albums al ON al.id = t.album_id
|
||||
LEFT JOIN artists ar ON ar.id = t.artist_id
|
||||
WHERE t.file_path IS NOT NULL AND t.file_path != ''
|
||||
AND t.title IS NOT NULL AND t.title != ''
|
||||
""")
|
||||
rows = cursor.fetchall()
|
||||
except Exception as e:
|
||||
logger.error("[Lyrics Filler] Error reading tracks: %s", e, exc_info=True)
|
||||
result.errors += 1
|
||||
return result
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
total = len(rows)
|
||||
if context.update_progress:
|
||||
context.update_progress(0, total)
|
||||
if context.report_progress:
|
||||
context.report_progress(phase=f'Checking lyrics for {total} tracks...', total=total)
|
||||
|
||||
for i, row in enumerate(rows):
|
||||
if context.check_stop():
|
||||
return result
|
||||
if i % 10 == 0 and context.wait_if_paused():
|
||||
return result
|
||||
|
||||
track_id, title, artist_name, album_title, file_path, duration = row[:6]
|
||||
result.scanned += 1
|
||||
|
||||
# Already has a sidecar on disk → nothing to do.
|
||||
if _has_lrc_sidecar(file_path):
|
||||
result.skipped += 1
|
||||
continue
|
||||
|
||||
# Option A: only flag tracks LRClib actually has lyrics for. An
|
||||
# instrumental returns nothing here and is silently skipped (never
|
||||
# re-flagged on future scans).
|
||||
# tracks.duration is stored in MILLISECONDS (schema) — LRClib's
|
||||
# exact-match wants SECONDS, so convert (else exact-match never
|
||||
# hits and it silently falls back to a fuzzier search).
|
||||
try:
|
||||
duration_s = int(int(duration) / 1000) if duration else None
|
||||
if duration_s is not None and duration_s <= 0:
|
||||
duration_s = None
|
||||
except (TypeError, ValueError):
|
||||
duration_s = None
|
||||
try:
|
||||
available = lyrics_client.has_remote_lyrics(
|
||||
title, artist_name or '', album_title, duration_s)
|
||||
except Exception as e:
|
||||
logger.debug("[Lyrics Filler] availability check failed for '%s': %s", title, e)
|
||||
available = False
|
||||
|
||||
if not available:
|
||||
result.skipped += 1
|
||||
if context.update_progress and (i + 1) % 10 == 0:
|
||||
context.update_progress(i + 1, total)
|
||||
continue
|
||||
|
||||
if context.report_progress:
|
||||
context.report_progress(
|
||||
scanned=i + 1, total=total,
|
||||
log_line=f'Found lyrics: {title} — {artist_name or "Unknown"}',
|
||||
log_type='success')
|
||||
|
||||
if context.create_finding:
|
||||
try:
|
||||
inserted = context.create_finding(
|
||||
job_id=self.job_id,
|
||||
finding_type='missing_lyrics',
|
||||
severity='info',
|
||||
entity_type='track',
|
||||
entity_id=str(track_id),
|
||||
file_path=file_path,
|
||||
title=f'Missing lyrics: {title or "Unknown"}',
|
||||
description=f'"{title}" by {artist_name or "Unknown"} has no .lrc — lyrics found on LRClib.',
|
||||
details={
|
||||
'track_id': track_id,
|
||||
'track_title': title,
|
||||
'artist': artist_name,
|
||||
'album_title': album_title,
|
||||
'file_path': file_path,
|
||||
'duration': duration_s,
|
||||
})
|
||||
if inserted:
|
||||
result.findings_created += 1
|
||||
else:
|
||||
result.findings_skipped_dedup += 1
|
||||
except Exception as e:
|
||||
logger.debug("[Lyrics Filler] create finding failed for track %s: %s", track_id, e)
|
||||
result.errors += 1
|
||||
|
||||
if context.update_progress and (i + 1) % 5 == 0:
|
||||
context.update_progress(i + 1, total)
|
||||
|
||||
if context.update_progress:
|
||||
context.update_progress(total, total)
|
||||
logger.info("[Lyrics Filler] %d tracks checked, %d with lyrics found, %d skipped",
|
||||
result.scanned, result.findings_created, result.skipped)
|
||||
return result
|
||||
|
||||
def estimate_scope(self, context: JobContext) -> int:
|
||||
conn = None
|
||||
try:
|
||||
conn = context.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM tracks
|
||||
WHERE file_path IS NOT NULL AND file_path != ''
|
||||
AND title IS NOT NULL AND title != ''
|
||||
""")
|
||||
row = cursor.fetchone()
|
||||
return row[0] if row else 0
|
||||
except Exception:
|
||||
return 0
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
|
@ -963,6 +963,8 @@ class RepairWorker:
|
|||
'orphan_file': self._fix_orphan_file,
|
||||
'track_number_mismatch': self._fix_track_number,
|
||||
'missing_cover_art': self._fix_missing_cover_art,
|
||||
'missing_lyrics': self._fix_missing_lyrics,
|
||||
'expired_download': self._fix_expired_download,
|
||||
'metadata_gap': self._fix_metadata_gap,
|
||||
'duplicate_tracks': self._fix_duplicates,
|
||||
'single_album_redundant': self._fix_single_album_redundant,
|
||||
|
|
@ -1277,17 +1279,58 @@ class RepairWorker:
|
|||
logger.error("Error fixing track number for %s: %s", file_path, e)
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
def _fix_artist_art(self, album_id, details):
|
||||
"""Apply the found ARTIST image to the album's artist (DB thumb only —
|
||||
artist art has no per-file embed). Pache711: independently applyable
|
||||
from the album art on the same finding."""
|
||||
artist_url = details.get('found_artist_url')
|
||||
if not artist_url:
|
||||
return {'success': False, 'error': 'No artist image found in finding details'}
|
||||
conn = None
|
||||
try:
|
||||
conn = self.db._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"UPDATE artists SET thumb_url = ?, updated_at = CURRENT_TIMESTAMP "
|
||||
"WHERE id = (SELECT artist_id FROM albums WHERE id = ?)",
|
||||
(artist_url, album_id))
|
||||
conn.commit()
|
||||
if cursor.rowcount == 0:
|
||||
return {'success': False, 'error': 'Artist not found for this album'}
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
return {'success': True, 'action': 'applied_artist_art',
|
||||
'message': 'Applied artist image'}
|
||||
|
||||
def _fix_missing_cover_art(self, entity_type, entity_id, file_path, details):
|
||||
"""Apply found artwork: update the DB thumbnail AND embed art into the
|
||||
album's audio files + write cover.jpg (using the post-processing
|
||||
standard, so the user's album_art_order preference is honored)."""
|
||||
artwork_url = details.get('found_artwork_url')
|
||||
if not artwork_url:
|
||||
return {'success': False, 'error': 'No artwork URL found in finding details'}
|
||||
"""Apply found artwork. ``_fix_action`` selects the target (Pache711):
|
||||
'album' (default — DB thumb + embed into files + cover.jpg), 'artist'
|
||||
(the artist's DB image), or 'both'. Defaulting to 'album' keeps the
|
||||
plain "Apply Art" button behaving exactly as before."""
|
||||
target = (details.get('_fix_action') or 'album').strip().lower()
|
||||
if target not in ('album', 'artist', 'both'):
|
||||
target = 'album'
|
||||
|
||||
album_id = details.get('album_id') or entity_id
|
||||
if not album_id:
|
||||
return {'success': False, 'error': 'No album ID associated with this finding'}
|
||||
|
||||
# Artist-only path: nothing to do with album files.
|
||||
if target == 'artist':
|
||||
return self._fix_artist_art(album_id, details)
|
||||
|
||||
artist_result = None
|
||||
if target == 'both':
|
||||
artist_result = self._fix_artist_art(album_id, details)
|
||||
|
||||
artwork_url = details.get('found_artwork_url')
|
||||
if not artwork_url:
|
||||
# 'both' but no album art — report the artist outcome if that ran.
|
||||
if artist_result is not None:
|
||||
return artist_result
|
||||
return {'success': False, 'error': 'No artwork URL found in finding details'}
|
||||
|
||||
conn = None
|
||||
track_paths = []
|
||||
album_title = details.get('album_title')
|
||||
|
|
@ -1332,8 +1375,10 @@ class RepairWorker:
|
|||
|
||||
if not resolved:
|
||||
# Media-server-only album (no local files): DB thumbnail is all we can set.
|
||||
return {'success': True, 'action': 'applied_cover_art',
|
||||
'message': 'Applied cover art to album (database only — no local files found)'}
|
||||
msg = 'Applied cover art to album (database only — no local files found)'
|
||||
if artist_result is not None and artist_result.get('success'):
|
||||
msg += ' + applied artist image'
|
||||
return {'success': True, 'action': 'applied_cover_art', 'message': msg}
|
||||
|
||||
from core.metadata.art_apply import apply_art_to_album_files
|
||||
metadata = {
|
||||
|
|
@ -1349,14 +1394,73 @@ class RepairWorker:
|
|||
art_result = apply_art_to_album_files(resolved, metadata, album_info, folder=folder)
|
||||
|
||||
embedded = art_result.get('embedded', 0)
|
||||
if art_result.get('read_only_fs'):
|
||||
# The music folder is genuinely read-only at the OS level (the
|
||||
# write raised EROFS). Most common cause is a docker ':ro' volume,
|
||||
# but it can also be a read-only host mount (NFS/SMB exported ro),
|
||||
# a mergerfs/union read-only branch, or the library mounted from
|
||||
# another container as read-only — chmod can't change any of these.
|
||||
return {'success': False, 'action': 'applied_cover_art',
|
||||
'error': ('Your music folder is READ-ONLY — the container cannot '
|
||||
'write to it (chmod cannot change this). Check that the '
|
||||
"volume isn't mapped ':ro', and that the underlying host "
|
||||
'mount (NFS/SMB/mergerfs) is read-write, then recreate the '
|
||||
'container. (Database thumbnail was still updated.)'),
|
||||
'art_result': art_result}
|
||||
msg = f'Applied cover art: embedded into {embedded}/{len(resolved)} file(s)'
|
||||
if art_result.get('cover_written'):
|
||||
msg += ' + wrote cover.jpg'
|
||||
if embedded == 0 and not art_result.get('cover_written'):
|
||||
# DB updated but nothing reached disk (e.g. read-only mount).
|
||||
# DB updated but nothing reached disk (e.g. permissions).
|
||||
msg = 'Updated database thumbnail, but could not write art to files (read-only?)'
|
||||
if artist_result is not None and artist_result.get('success'):
|
||||
msg += ' + applied artist image'
|
||||
return {'success': True, 'action': 'applied_cover_art', 'message': msg, 'art_result': art_result}
|
||||
|
||||
def _fix_missing_lyrics(self, entity_type, entity_id, file_path, details):
|
||||
"""Apply a missing-lyrics finding: fetch + write the .lrc sidecar and
|
||||
embed the lyrics, via the same LyricsClient the import pipeline uses."""
|
||||
raw_path = details.get('file_path') or file_path
|
||||
if not raw_path:
|
||||
return {'success': False, 'error': 'No file path in finding'}
|
||||
download_folder = self._config_manager.get('soulseek.download_path', '') if self._config_manager else None
|
||||
resolved = _resolve_file_path(raw_path, self.transfer_folder, download_folder,
|
||||
config_manager=self._config_manager) or raw_path
|
||||
if not os.path.isfile(resolved):
|
||||
return {'success': False, 'error': f'File not found on disk: {os.path.basename(raw_path)}'}
|
||||
try:
|
||||
from core.lyrics_client import lyrics_client
|
||||
duration = details.get('duration')
|
||||
ok = lyrics_client.create_lrc_file(
|
||||
resolved,
|
||||
details.get('track_title') or '',
|
||||
details.get('artist') or '',
|
||||
album_name=details.get('album_title'),
|
||||
duration_seconds=int(duration) if duration else None,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Lyrics fix failed for %s: %s", os.path.basename(raw_path), e)
|
||||
return {'success': False, 'error': str(e)}
|
||||
if not ok:
|
||||
# Lyrics vanished between scan and apply (rare) — report, don't crash.
|
||||
return {'success': False, 'error': 'Could not fetch lyrics (no longer available?)'}
|
||||
return {'success': True, 'action': 'applied_lyrics', 'message': 'Wrote lyrics (.lrc) + embedded'}
|
||||
|
||||
def _fix_expired_download(self, entity_type, entity_id, file_path, details):
|
||||
"""Apply an expired-download finding: delete the file + library row +
|
||||
history entry, via the same helper the cleaner's auto mode uses."""
|
||||
from core.repair_jobs.expired_download_cleaner import delete_origin_download
|
||||
entry = {'id': details.get('history_id') or entity_id,
|
||||
'file_path': details.get('file_path') or file_path}
|
||||
if not entry['id']:
|
||||
return {'success': False, 'error': 'No history id in finding'}
|
||||
res = delete_origin_download(self.db, entry, self._config_manager)
|
||||
if res.get('error'):
|
||||
return {'success': False, 'action': 'deleted_expired',
|
||||
'error': f"Could not delete file: {res['error']}"}
|
||||
verb = 'deleted file + entry' if res.get('file_deleted') else 'removed entry (file already gone)'
|
||||
return {'success': True, 'action': 'deleted_expired', 'message': f'Expired download — {verb}'}
|
||||
|
||||
def _fix_library_retag(self, entity_type, entity_id, file_path, details):
|
||||
"""Apply a library re-tag finding: write each track's planned tags in
|
||||
place (core.tag_writer.write_tags_to_file) + optionally embed/refresh
|
||||
|
|
@ -1378,13 +1482,16 @@ class RepairWorker:
|
|||
plan = {'file_path': rp, 'db_data': t.get('db_data') or {}}
|
||||
if t.get('full_meta'):
|
||||
plan['full_meta'] = t['full_meta']
|
||||
if t.get('lyrics_meta'):
|
||||
plan['lyrics_meta'] = t['lyrics_meta'] # read-only lyrics query metadata
|
||||
resolved_plans.append(plan)
|
||||
|
||||
from core.repair_jobs.library_retag import apply_track_plans
|
||||
res = apply_track_plans(resolved_plans, details.get('cover_action'), details.get('cover_url'),
|
||||
full=(details.get('depth') == 'full'))
|
||||
full=(details.get('depth') == 'full'),
|
||||
lyrics_action=details.get('lyrics_action', False))
|
||||
|
||||
if res['written'] == 0 and not res['cover_written']:
|
||||
if res['written'] == 0 and not res['cover_written'] and not res.get('lyrics_written'):
|
||||
return {'success': False,
|
||||
'error': 'Nothing could be written — files unreachable or read-only?'}
|
||||
msg = f"Re-tagged {res['written']} track(s)"
|
||||
|
|
@ -3088,7 +3195,7 @@ class RepairWorker:
|
|||
|
||||
# Build query for pending fixable findings
|
||||
fixable_types = ('dead_file', 'orphan_file', 'track_number_mismatch',
|
||||
'missing_cover_art', 'metadata_gap', 'duplicate_tracks',
|
||||
'missing_cover_art', 'missing_lyrics', 'expired_download', 'metadata_gap', 'duplicate_tracks',
|
||||
'single_album_redundant', 'mbid_mismatch',
|
||||
'album_mbid_mismatch',
|
||||
'album_tag_inconsistency',
|
||||
|
|
|
|||
|
|
@ -471,6 +471,14 @@ class Album:
|
|||
artist_ids=[artist['id'] for artist in album_data['artists']]
|
||||
)
|
||||
|
||||
# The web UI's virtual "Liked Songs" playlist id. There is NO real playlist
|
||||
# behind a user's liked songs — Spotify serves them via /me/tracks (saved
|
||||
# tracks), and passing this id to the playlist endpoint 400s with
|
||||
# "Unsupported URL / URI". Anything resolving playlists by id must special-case
|
||||
# it (get_playlist_by_id does).
|
||||
LIKED_SONGS_PLAYLIST_ID = "spotify:liked-songs"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Playlist:
|
||||
id: str
|
||||
|
|
@ -495,6 +503,32 @@ class Playlist:
|
|||
total_tracks=(playlist_data.get('tracks') or playlist_data.get('items') or {}).get('total', 0)
|
||||
)
|
||||
|
||||
def describe_spotify_unavailable(*, configured: bool, rate_limited: bool,
|
||||
ban_seconds_left: int = 0, in_cooldown: bool = False,
|
||||
cooldown_seconds_left: int = 0, has_token: bool = True) -> str:
|
||||
"""Human reason Spotify can't serve a request right now.
|
||||
|
||||
``is_spotify_authenticated()`` returns False for five distinct reasons, but
|
||||
every API call site logged the same bare "Not authenticated with Spotify" —
|
||||
so a rate-limit ban looked like a logout. This maps the real state to a
|
||||
clear message (priority matches is_spotify_authenticated): not-configured →
|
||||
rate-limited → post-ban cooldown → no-token → unknown probe failure. Pure.
|
||||
"""
|
||||
if not configured:
|
||||
return "Spotify not configured (add credentials in Settings)"
|
||||
if rate_limited:
|
||||
if ban_seconds_left and ban_seconds_left > 0:
|
||||
return f"Spotify rate-limited — ban ~{max(1, round(ban_seconds_left / 60))}m left (not a logout)"
|
||||
return "Spotify rate-limited — waiting out a ban (not a logout)"
|
||||
if in_cooldown:
|
||||
if cooldown_seconds_left and cooldown_seconds_left > 0:
|
||||
return f"Spotify post-ban cooldown — ~{cooldown_seconds_left}s left (not a logout)"
|
||||
return "Spotify post-ban cooldown (not a logout)"
|
||||
if not has_token:
|
||||
return "Spotify not connected — no saved token; re-authenticate in Settings"
|
||||
return "Spotify auth check failed (token refresh may have failed — see debug log)"
|
||||
|
||||
|
||||
class SpotifyClient:
|
||||
def __init__(self):
|
||||
self.sp: Optional[spotipy.Spotify] = None
|
||||
|
|
@ -612,9 +646,12 @@ class SpotifyClient:
|
|||
term covers the brief window before the auth cache refreshes. When authed
|
||||
+ healthy the official path returns first, so this never opens.
|
||||
|
||||
Two activations fall out of this: a no-auth user who chose Spotify Free
|
||||
(free is their source), and a connected user mid-rate-limit (free bridges
|
||||
the ban) — see _free_wanted()."""
|
||||
Three activations fall out of this: a no-auth user who chose Spotify
|
||||
Free (free is their source), a connected user mid-rate-limit (free
|
||||
bridges the ban), and a connected user who has spent the enrichment
|
||||
worker's real-API daily budget (``_budget_exhausted_use_free``, set by
|
||||
the worker) — so a Spotify-Free user is never paused by the budget, it
|
||||
just switches to the uncapped free source. See _free_wanted()."""
|
||||
from core.spotify_free_metadata import should_use_free_fallback
|
||||
if not self._free_available():
|
||||
return False
|
||||
|
|
@ -622,7 +659,10 @@ class SpotifyClient:
|
|||
authed = self.is_spotify_authenticated()
|
||||
except Exception:
|
||||
authed = False
|
||||
return should_use_free_fallback(authed, _is_globally_rate_limited())
|
||||
return should_use_free_fallback(
|
||||
authed, _is_globally_rate_limited(),
|
||||
getattr(self, '_budget_exhausted_use_free', False),
|
||||
)
|
||||
|
||||
@property
|
||||
def _free_meta(self):
|
||||
|
|
@ -648,12 +688,20 @@ class SpotifyClient:
|
|||
return
|
||||
|
||||
try:
|
||||
# Tokens live in the database-backed config store, not a loose
|
||||
# file: config/.spotify_cache sat in /app/config, which is only
|
||||
# persistent when the user's compose maps it — an anonymous
|
||||
# volume is recreated empty on every container pull, so tokens
|
||||
# died nightly while every other setting survived ("it keeps
|
||||
# unauthenticating" daily, wolf39us). The handler imports the
|
||||
# legacy file once if the store is empty.
|
||||
from core.spotify_token_cache import DatabaseTokenCache
|
||||
auth_manager = SpotifyOAuth(
|
||||
client_id=config['client_id'],
|
||||
client_secret=config['client_secret'],
|
||||
redirect_uri=config.get('redirect_uri', "http://127.0.0.1:8888/callback"),
|
||||
scope="user-library-read user-read-private playlist-read-private playlist-read-collaborative user-read-email user-follow-read",
|
||||
cache_path='config/.spotify_cache'
|
||||
cache_handler=DatabaseTokenCache(config_manager)
|
||||
)
|
||||
|
||||
self.sp = spotipy.Spotify(auth_manager=auth_manager, retries=0, requests_timeout=15)
|
||||
|
|
@ -842,6 +890,28 @@ class SpotifyClient:
|
|||
|
||||
return result
|
||||
|
||||
def _auth_unavailable_reason(self) -> str:
|
||||
"""Real reason an API call can't reach Spotify right now, for logging —
|
||||
distinguishes a rate-limit ban / cooldown from a genuine logout instead
|
||||
of the old catch-all 'Not authenticated'. Side-effect-free: reads the
|
||||
cached token (no refresh, no API probe)."""
|
||||
has_token = True
|
||||
try:
|
||||
if self.sp is not None:
|
||||
ch = getattr(self.sp.auth_manager, 'cache_handler', None)
|
||||
has_token = ch is None or ch.get_cached_token() is not None
|
||||
except Exception:
|
||||
has_token = True # unknown → don't wrongly claim "not connected"
|
||||
rl = _get_rate_limit_info() if _is_globally_rate_limited() else None
|
||||
return describe_spotify_unavailable(
|
||||
configured=self.sp is not None,
|
||||
rate_limited=_is_globally_rate_limited(),
|
||||
ban_seconds_left=(rl or {}).get('remaining_seconds', 0) or 0,
|
||||
in_cooldown=_is_in_post_ban_cooldown(),
|
||||
cooldown_seconds_left=_get_post_ban_cooldown_remaining() or 0,
|
||||
has_token=has_token,
|
||||
)
|
||||
|
||||
def disconnect(self):
|
||||
"""Disconnect Spotify: clear client, delete cache, invalidate auth cache, clear rate limit"""
|
||||
import os
|
||||
|
|
@ -862,13 +932,12 @@ class SpotifyClient:
|
|||
except Exception as e:
|
||||
logger.debug("publish_spotify_status disconnect: %s", e)
|
||||
|
||||
cache_path = 'config/.spotify_cache'
|
||||
try:
|
||||
if os.path.exists(cache_path):
|
||||
os.remove(cache_path)
|
||||
logger.info("Deleted Spotify cache file")
|
||||
from core.spotify_token_cache import DatabaseTokenCache
|
||||
DatabaseTokenCache(config_manager).clear()
|
||||
logger.info("Cleared Spotify token cache (database + legacy file)")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete Spotify cache: {e}")
|
||||
logger.warning(f"Failed to clear Spotify token cache: {e}")
|
||||
|
||||
logger.info("Spotify client disconnected")
|
||||
|
||||
|
|
@ -909,7 +978,7 @@ class SpotifyClient:
|
|||
@rate_limited
|
||||
def get_user_playlists(self) -> List[Playlist]:
|
||||
if not self.is_spotify_authenticated():
|
||||
logger.error("Not authenticated with Spotify")
|
||||
logger.error("Spotify request skipped — %s", self._auth_unavailable_reason())
|
||||
return []
|
||||
|
||||
if not self._ensure_user_id():
|
||||
|
|
@ -954,7 +1023,7 @@ class SpotifyClient:
|
|||
def get_user_playlists_metadata_only(self) -> List[Playlist]:
|
||||
"""Get playlists without fetching all track details for faster loading"""
|
||||
if not self.is_spotify_authenticated():
|
||||
logger.error("Not authenticated with Spotify")
|
||||
logger.error("Spotify request skipped — %s", self._auth_unavailable_reason())
|
||||
return []
|
||||
|
||||
if not self._ensure_user_id():
|
||||
|
|
@ -1027,7 +1096,7 @@ class SpotifyClient:
|
|||
def get_saved_tracks_count(self) -> int:
|
||||
"""Get the total count of user's saved/liked songs without fetching all tracks"""
|
||||
if not self.is_spotify_authenticated():
|
||||
logger.error("Not authenticated with Spotify")
|
||||
logger.error("Spotify request skipped — %s", self._auth_unavailable_reason())
|
||||
return 0
|
||||
|
||||
try:
|
||||
|
|
@ -1046,7 +1115,7 @@ class SpotifyClient:
|
|||
def get_saved_tracks(self) -> List[Track]:
|
||||
"""Fetch all user's saved/liked songs from Spotify"""
|
||||
if not self.is_spotify_authenticated():
|
||||
logger.error("Not authenticated with Spotify")
|
||||
logger.error("Spotify request skipped — %s", self._auth_unavailable_reason())
|
||||
return []
|
||||
|
||||
tracks = []
|
||||
|
|
@ -1097,7 +1166,7 @@ class SpotifyClient:
|
|||
List of dicts with album metadata ready for DB upsert.
|
||||
"""
|
||||
if not self.is_spotify_authenticated():
|
||||
logger.error("Not authenticated with Spotify")
|
||||
logger.error("Spotify request skipped — %s", self._auth_unavailable_reason())
|
||||
return []
|
||||
|
||||
albums = []
|
||||
|
|
@ -1233,15 +1302,56 @@ class SpotifyClient:
|
|||
def get_playlist_by_id(self, playlist_id: str) -> Optional[Playlist]:
|
||||
if not self.is_spotify_authenticated():
|
||||
return None
|
||||
|
||||
|
||||
# "Liked Songs" is virtual — no playlist URI exists for it, so the
|
||||
# playlist endpoint 400s ("Unsupported URL / URI"). Serve it from the
|
||||
# saved-tracks endpoint instead, so mirrored playlists / anything that
|
||||
# re-resolves by stored id can refresh it like a normal playlist.
|
||||
if playlist_id in (LIKED_SONGS_PLAYLIST_ID, 'liked-songs'):
|
||||
return self._liked_songs_as_playlist()
|
||||
|
||||
try:
|
||||
playlist_data = self.sp.playlist(playlist_id)
|
||||
tracks = self._get_playlist_tracks(playlist_id)
|
||||
return Playlist.from_spotify_playlist(playlist_data, tracks)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching playlist {playlist_id}: {e}")
|
||||
return None
|
||||
|
||||
def _liked_songs_as_playlist(self) -> Optional[Playlist]:
|
||||
"""Build a ``Playlist`` for the virtual Liked Songs collection from the
|
||||
saved-tracks endpoint (the only API that serves it)."""
|
||||
try:
|
||||
tracks = self.get_saved_tracks()
|
||||
if not tracks:
|
||||
# get_saved_tracks swallows fetch errors into [] — indistinguishable
|
||||
# from "no likes". The virtual playlist is only offered when likes
|
||||
# exist, so treat empty as a FAILED refresh (None) rather than hand
|
||||
# the sync a valid-looking empty playlist it might mirror by
|
||||
# clearing the server-side copy.
|
||||
logger.error("Liked Songs resolve: saved-tracks fetch returned nothing — treating as failed refresh")
|
||||
return None
|
||||
owner = 'You'
|
||||
try:
|
||||
info = self.get_user_info()
|
||||
if info and info.get('display_name'):
|
||||
owner = info['display_name']
|
||||
except Exception: # noqa: S110 — owner label is cosmetic; default stands
|
||||
pass
|
||||
return Playlist(
|
||||
id=LIKED_SONGS_PLAYLIST_ID,
|
||||
name='Liked Songs',
|
||||
description='Your liked songs on Spotify',
|
||||
owner=owner,
|
||||
public=False,
|
||||
collaborative=False,
|
||||
tracks=tracks,
|
||||
total_tracks=len(tracks),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error building Liked Songs playlist: {e}")
|
||||
return None
|
||||
|
||||
@rate_limited
|
||||
def get_followed_artists(self) -> list:
|
||||
|
|
@ -1756,6 +1866,7 @@ class SpotifyClient:
|
|||
try:
|
||||
albums = []
|
||||
raw_items = []
|
||||
truncated = False # did we stop while more pages existed?
|
||||
# Spotify caps artist_albums at 10 per page
|
||||
results = self.sp.artist_albums(artist_id, album_type=album_type, limit=min(limit, 10))
|
||||
pages_fetched = 1
|
||||
|
|
@ -1768,6 +1879,7 @@ class SpotifyClient:
|
|||
|
||||
# Stop if we've hit the page limit (0 = unlimited)
|
||||
if max_pages and pages_fetched >= max_pages:
|
||||
truncated = bool(results.get('next'))
|
||||
break
|
||||
|
||||
# Get next batch if available — throttle pagination to respect rate limits
|
||||
|
|
@ -1790,8 +1902,17 @@ class SpotifyClient:
|
|||
(f" (page limit: {max_pages})" if max_pages else ""))
|
||||
|
||||
# Cache the full artist albums result (wrapped in dict for cache compatibility)
|
||||
# Only cache COMPLETE discographies. The cache key carries no
|
||||
# limit/page info, so a partial probe (the watchlist's
|
||||
# new-release check: limit=5, max_pages=1) stored here used to
|
||||
# POISON the slot — the artist detail page then showed only
|
||||
# the 5-10 newest releases for every watchlist artist until
|
||||
# the 30-day TTL expired ("Taylor Swift has 8 albums, nothing
|
||||
# before 2022"). Individual albums are still cached — they're
|
||||
# complete entities regardless of how many pages we walked.
|
||||
if raw_items:
|
||||
cache.store_entity('spotify', 'artist', cache_key, {'name': f'albums_{artist_id}', '_albums': raw_items})
|
||||
if not truncated:
|
||||
cache.store_entity('spotify', 'artist', cache_key, {'name': f'albums_{artist_id}', '_albums': raw_items})
|
||||
# Also cache individual albums opportunistically
|
||||
entries = [(ad.get('id'), ad) for ad in raw_items if ad.get('id')]
|
||||
if entries:
|
||||
|
|
|
|||
|
|
@ -46,13 +46,17 @@ def spotify_free_installed() -> bool:
|
|||
return _installed_cache
|
||||
|
||||
|
||||
def should_use_free_fallback(authenticated: bool, rate_limited: bool) -> bool:
|
||||
def should_use_free_fallback(authenticated: bool, rate_limited: bool,
|
||||
budget_exhausted: bool = False) -> bool:
|
||||
"""The per-request gate: the no-creds SpotipyFree source may serve a request
|
||||
ONLY when official Spotify can't — i.e. the user has no Spotify auth, or
|
||||
we're currently rate-limited. When authed AND healthy the official path
|
||||
returns before any fallback, so this never opens.
|
||||
ONLY when official Spotify can't — i.e. the user has no Spotify auth, we're
|
||||
currently rate-limited, OR the worker's self-imposed real-API daily budget
|
||||
is spent. The daily budget is a real-API ban protection; once it's hit, the
|
||||
free source (which isn't subject to it) is the natural place to keep going,
|
||||
so a Spotify-Free user is never paused by the budget. When authed AND
|
||||
healthy AND under budget, the official path returns before any fallback.
|
||||
"""
|
||||
return (not authenticated) or rate_limited
|
||||
return (not authenticated) or rate_limited or budget_exhausted
|
||||
|
||||
|
||||
def should_offer_spotify_metadata(authenticated: bool, free_available: bool) -> bool:
|
||||
|
|
|
|||
84
core/spotify_token_cache.py
Normal file
84
core/spotify_token_cache.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""Database-backed Spotify token cache (wolf39us's daily-deauth fix).
|
||||
|
||||
Spotipy's default cache is a loose file — ours lived at
|
||||
``config/.spotify_cache``. In Docker, ``/app/config`` is a declared VOLUME,
|
||||
but a compose file that doesn't map it explicitly gets an ANONYMOUS volume,
|
||||
and anonymous volumes don't survive container recreation. Net effect: a
|
||||
nightly Watchtower pull kept the user's settings (config now lives in the
|
||||
database) but silently dropped the OAuth tokens — "it keeps unauthenticating"
|
||||
every day, while a manual re-auth always "fixed" it until the next pull.
|
||||
|
||||
This handler stores the token payload in the same database-backed config
|
||||
store as every other setting (``spotify.token_info``), so tokens survive
|
||||
exactly as long as the rest of the configuration does. The legacy cache file
|
||||
is imported once if the store is empty, then left in place for rollback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from spotipy.cache_handler import CacheHandler
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("spotify_token_cache")
|
||||
|
||||
_CONFIG_KEY = "spotify.token_info"
|
||||
LEGACY_CACHE_PATH = "config/.spotify_cache"
|
||||
|
||||
|
||||
class DatabaseTokenCache(CacheHandler):
|
||||
"""Spotipy CacheHandler persisting the token in the config database."""
|
||||
|
||||
def __init__(self, config_manager, legacy_path: str = LEGACY_CACHE_PATH):
|
||||
self._config = config_manager
|
||||
self._legacy_path = legacy_path
|
||||
|
||||
def get_cached_token(self) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
token = self._config.get(_CONFIG_KEY, None)
|
||||
if isinstance(token, str):
|
||||
token = json.loads(token)
|
||||
if isinstance(token, dict) and token.get("access_token"):
|
||||
return token
|
||||
except Exception as e:
|
||||
logger.debug("token cache read failed: %s", e)
|
||||
|
||||
# One-time import from the legacy file cache, so an upgrade doesn't
|
||||
# force a re-auth when the file happens to still be around.
|
||||
try:
|
||||
if self._legacy_path and os.path.isfile(self._legacy_path):
|
||||
with open(self._legacy_path, "r", encoding="utf-8") as fh:
|
||||
legacy = json.load(fh)
|
||||
if isinstance(legacy, dict) and legacy.get("access_token"):
|
||||
logger.info(
|
||||
"Imported Spotify token from legacy file cache into the "
|
||||
"database store (tokens now survive container recreation)")
|
||||
self.save_token_to_cache(legacy)
|
||||
return legacy
|
||||
except Exception as e:
|
||||
logger.debug("legacy token import failed: %s", e)
|
||||
return None
|
||||
|
||||
def save_token_to_cache(self, token_info: Dict[str, Any]) -> None:
|
||||
try:
|
||||
self._config.set(_CONFIG_KEY, token_info)
|
||||
except Exception as e:
|
||||
# Never let a cache write break an API call — worst case the
|
||||
# refreshed token isn't persisted and the next run refreshes again.
|
||||
logger.warning("token cache write failed: %s", e)
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Logout: drop the stored token (and the legacy file if present)."""
|
||||
try:
|
||||
self._config.set(_CONFIG_KEY, None)
|
||||
except Exception as e:
|
||||
logger.debug("token cache clear failed: %s", e)
|
||||
try:
|
||||
if self._legacy_path and os.path.isfile(self._legacy_path):
|
||||
os.remove(self._legacy_path)
|
||||
except OSError as e:
|
||||
logger.debug("legacy cache remove failed: %s", e)
|
||||
|
|
@ -126,13 +126,17 @@ class SpotifyWorker:
|
|||
rate_limit_info = self.client.get_rate_limit_info() if rate_limited else None
|
||||
in_cooldown = self.client.get_post_ban_cooldown_remaining() > 0
|
||||
authenticated = self.client.sp is not None
|
||||
# Is the worker still serving via the no-creds Spotify Free source
|
||||
# despite the real-API ban? Only check WHEN rate-limited: during a
|
||||
# ban is_spotify_authenticated() returns False without an API probe,
|
||||
# so is_spotify_metadata_available() reduces to "is free available"
|
||||
# (no quota cost). Lets the UI show "via Spotify Free" instead of a
|
||||
# misleading "rate limited / waiting" while the worker keeps matching.
|
||||
using_free = bool(rate_limited and self.client.is_spotify_metadata_available())
|
||||
# Is the worker serving via the no-creds Spotify Free source right
|
||||
# now? Two cases: bridging a rate-limit ban (only checked WHEN
|
||||
# rate-limited — there is_spotify_authenticated() returns False
|
||||
# without an API probe, so this is quota-free), OR bridging the spent
|
||||
# real-API daily budget (the worker set _budget_exhausted_use_free —
|
||||
# a cheap attribute read). Lets the UI show "Running (Spotify Free)"
|
||||
# instead of a misleading "rate limited" / "daily limit reached".
|
||||
using_free = bool(
|
||||
(rate_limited and self.client.is_spotify_metadata_available())
|
||||
or getattr(self.client, '_budget_exhausted_use_free', False)
|
||||
)
|
||||
except Exception:
|
||||
authenticated = False
|
||||
rate_limited = False
|
||||
|
|
@ -214,14 +218,35 @@ class SpotifyWorker:
|
|||
# protect the REAL authenticated API from bans — they don't apply
|
||||
# to free (a different, anonymous path). Computed once and reused
|
||||
# below; the loop already probes auth, so no extra quota cost.
|
||||
budget_exhausted = self._is_daily_budget_exhausted()
|
||||
|
||||
# Daily budget is a REAL-API ban protection. When it's spent, if
|
||||
# the no-creds free source is available, BRIDGE to it (uncapped)
|
||||
# for the rest of the day instead of pausing — so a Spotify-Free
|
||||
# user is never stopped by the budget. The flag makes the client
|
||||
# route subsequent calls to free; it clears on the daily reset.
|
||||
try:
|
||||
if budget_exhausted and self.client._free_available():
|
||||
self.client._budget_exhausted_use_free = True
|
||||
elif not budget_exhausted and getattr(self.client, '_budget_exhausted_use_free', False):
|
||||
self.client._budget_exhausted_use_free = False
|
||||
except Exception: # noqa: S110 — budget→free toggle is best-effort
|
||||
pass
|
||||
|
||||
# Is the worker serving via the no-creds Spotify Free source this
|
||||
# iteration? The daily budget and post-ban cooldown both exist to
|
||||
# protect the REAL authenticated API from bans — they don't apply
|
||||
# to free (a different, anonymous path). _free_active() now also
|
||||
# returns True when the budget-bridge flag above is set. Computed
|
||||
# once and reused below; the loop already probes auth, no extra cost.
|
||||
try:
|
||||
free_serving = self.client._free_active()
|
||||
except Exception:
|
||||
free_serving = False
|
||||
|
||||
# Daily budget guard — worker-only cap to avoid saturating the REAL
|
||||
# Spotify API. Skipped while serving via free (free isn't that API).
|
||||
if not free_serving and self._is_daily_budget_exhausted():
|
||||
# Daily budget guard — pause ONLY when the budget is spent AND we
|
||||
# can't serve via free (no free available). Otherwise free took over.
|
||||
if not free_serving and budget_exhausted:
|
||||
budget = self._get_daily_budget_info()
|
||||
resets_in = budget['resets_in_seconds']
|
||||
logger.info(f"Daily enrichment budget exhausted ({budget['used']}/{budget['limit']}), "
|
||||
|
|
|
|||
|
|
@ -31,14 +31,18 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
# Must live under the soulsync.* namespace — handlers are only attached there,
|
||||
# so a bare getLogger(__name__) ("core.streaming.prepare") logged into the void
|
||||
# and made stream-prep failures invisible in app.log.
|
||||
logger = get_logger("streaming.prepare")
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -38,6 +38,10 @@ def _fresh_state() -> Dict[str, Any]:
|
|||
"progress": 0,
|
||||
"track_info": None,
|
||||
"file_path": None,
|
||||
# Set instead of file_path when a library track is played by proxying
|
||||
# the media server's own stream API (Navidrome/Subsonic, #809) rather
|
||||
# than reading the file off disk. /stream/audio proxies this URL.
|
||||
"stream_url": None,
|
||||
"error_message": None,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -78,4 +78,60 @@ def titles_plausibly_same(
|
|||
return not ta.isdisjoint(tb)
|
||||
|
||||
|
||||
__all__ = ["titles_plausibly_same"]
|
||||
_QUALIFIER_RE = re.compile(r"[\(\[]([^\)\]]*)[\)\]]")
|
||||
|
||||
|
||||
def strip_redundant_context_qualifiers(title: str, *context_texts: str) -> str:
|
||||
"""Remove parenthetical/bracket qualifiers that merely restate known context.
|
||||
|
||||
A qualifier whose text appears (word-bounded) in one of ``context_texts``
|
||||
— typically the release's album title, or the other side of a comparison —
|
||||
is album context, not a version difference. #808: the wishlist held
|
||||
'Champagne Supernova (OurVinyl Sessions)' while the library track was the
|
||||
bare 'Champagne Supernova' on the album '… (OurVinyl Sessions)'; the
|
||||
qualifier restated the album, but the length-ratio penalty treated the
|
||||
pair as different songs and the cleanup never recognised the owned
|
||||
edition. Version markers that do NOT appear in any context ('(Live)',
|
||||
'(Remix)' on a studio album) are kept, so their mismatch penalty stands.
|
||||
"""
|
||||
if not title:
|
||||
return title
|
||||
|
||||
contexts = [c.casefold() for c in context_texts if c]
|
||||
if not contexts:
|
||||
return title
|
||||
|
||||
def _drop(match: re.Match) -> str:
|
||||
inner = match.group(1).strip().casefold()
|
||||
if not inner:
|
||||
return " "
|
||||
pattern = r"\b" + re.escape(inner) + r"\b"
|
||||
for ctx in contexts:
|
||||
if re.search(pattern, ctx):
|
||||
return " "
|
||||
return match.group(0)
|
||||
|
||||
out = _QUALIFIER_RE.sub(_drop, title)
|
||||
return re.sub(r"\s+", " ", out).strip()
|
||||
|
||||
|
||||
def numeric_tokens_differ(title_a: str, title_b: str) -> bool:
|
||||
"""True when the digit-bearing tokens of two titles differ — 'Vol.4' vs
|
||||
'Vol.4.5', 'Album' vs 'Album 2'. A numeric difference is a different
|
||||
release (volume / part / sequel), never a '(Deluxe)'-style suffix:
|
||||
string similarity ('Vol.4' vs 'Vol.4.5' = 0.97) and token-subset checks
|
||||
both wave these through, which hung volume 4.5's cover art on volume 4
|
||||
(Sokhi). Shared digits on both sides ('1989' vs '1989 (Deluxe)') are
|
||||
fine."""
|
||||
def _digit_tokens(text: str) -> frozenset:
|
||||
tokens = re.sub(r"[^a-z0-9]+", " ", (text or "").casefold()).split()
|
||||
return frozenset(t for t in tokens if any(c.isdigit() for c in t))
|
||||
|
||||
return _digit_tokens(title_a) != _digit_tokens(title_b)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"titles_plausibly_same",
|
||||
"strip_redundant_context_qualifiers",
|
||||
"numeric_tokens_differ",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1748,12 +1748,21 @@ class WatchlistScanner:
|
|||
def _match_to_itunes(self, artist_name: str) -> Optional[str]:
|
||||
"""Match artist name to iTunes ID using fuzzy name comparison."""
|
||||
try:
|
||||
if hasattr(self, '_metadata_service') and self._metadata_service:
|
||||
results = self._metadata_service.itunes.search_artists(artist_name, limit=5)
|
||||
else:
|
||||
logger.warning("Cannot match to iTunes - MetadataService not available")
|
||||
# Use the canonical iTunes client like _match_to_deezer/_discogs/
|
||||
# _musicbrainz do. The old path read the PRIVATE _metadata_service
|
||||
# attr (None when the scanner is built from a spotify_client — the
|
||||
# normal web_server wiring) and just gave up — the only matcher
|
||||
# with no fallback — so watchlist iTunes backfills failed wholesale
|
||||
# ("MetadataService not available" ×8, Backfilled 0/8). And even
|
||||
# when set, metadata_service.itunes is the FALLBACK-client slot,
|
||||
# which may actually be a DeezerClient (see _match_to_deezer) —
|
||||
# matching against it could store a Deezer ID as itunes_artist_id.
|
||||
from core.metadata.registry import get_itunes_client
|
||||
client = get_itunes_client()
|
||||
if client is None:
|
||||
logger.warning("Cannot match to iTunes - client unavailable")
|
||||
return None
|
||||
|
||||
results = client.search_artists(artist_name, limit=5)
|
||||
return self._best_artist_match(results, artist_name)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not match {artist_name} to iTunes: {e}")
|
||||
|
|
@ -3613,13 +3622,24 @@ class WatchlistScanner:
|
|||
if not album_data or 'tracks' not in album_data:
|
||||
continue
|
||||
|
||||
# #705: announced-but-unreleased albums must not reach the
|
||||
# radar — and worse, their NEGATIVE days_old used to INFLATE
|
||||
# the recency score (100 - days*7) above every released
|
||||
# track, which is why Fresh Tape filled up with prereleases.
|
||||
from core.metadata.release_dates import is_future_release
|
||||
if is_future_release(album.get('release_date', '')):
|
||||
logger.debug(
|
||||
f"Release Radar: skipping unreleased album "
|
||||
f"'{album.get('album_name', '?')}' ({album.get('release_date')})")
|
||||
continue
|
||||
|
||||
# Calculate days since release for recency score
|
||||
days_old = 14
|
||||
try:
|
||||
release_date_str = album.get('release_date', '')
|
||||
if release_date_str and len(release_date_str) >= 10:
|
||||
release_date = datetime.strptime(release_date_str[:10], "%Y-%m-%d")
|
||||
days_old = (datetime.now() - release_date).days
|
||||
days_old = max(0, (datetime.now() - release_date).days)
|
||||
except Exception as e:
|
||||
logger.debug("release-date parse: %s", e)
|
||||
|
||||
|
|
|
|||
|
|
@ -729,6 +729,19 @@ def _prepare_and_run_manual_wishlist_batch(
|
|||
logger.warning(f"[Manual-Wishlist] Found and removed {duplicates_found} duplicate tracks during sanitization")
|
||||
logger.info(f"[Manual-Wishlist] Sanitized {len(wishlist_tracks)} tracks from wishlist service")
|
||||
|
||||
# #705: don't burn a search+timeout on tracks that aren't out yet
|
||||
# (watchlist scans add announced albums on purpose). They stay in the
|
||||
# wishlist and start processing the cycle after their release date
|
||||
# passes. An explicit track selection overrides the gate — the user
|
||||
# asked for those specifically.
|
||||
if not track_ids:
|
||||
from core.metadata.release_dates import split_released_unreleased
|
||||
wishlist_tracks, _unreleased = split_released_unreleased(wishlist_tracks)
|
||||
if _unreleased:
|
||||
logger.info(
|
||||
f"[Manual-Wishlist] Skipping {len(_unreleased)} unreleased track(s) "
|
||||
f"(future release date) — they'll be searched once released")
|
||||
|
||||
if track_ids:
|
||||
track_lookup = {}
|
||||
for track in wishlist_tracks:
|
||||
|
|
@ -919,6 +932,17 @@ def process_wishlist_automatically(runtime: WishlistAutoProcessingRuntime, autom
|
|||
logger.warning(f"[Auto-Wishlist] Found and removed {duplicates_found} duplicate tracks during sanitization")
|
||||
logger.info(f"[Auto-Wishlist] Sanitized {len(wishlist_tracks)} tracks from wishlist service")
|
||||
|
||||
# #705: skip tracks with a future release date — searching for
|
||||
# them burns a full search+timeout per track, every cycle, for
|
||||
# files that can't exist yet. They stay in the wishlist and
|
||||
# join the cycle automatically once their date passes.
|
||||
from core.metadata.release_dates import split_released_unreleased
|
||||
wishlist_tracks, _unreleased = split_released_unreleased(wishlist_tracks)
|
||||
if _unreleased:
|
||||
logger.info(
|
||||
f"[Auto-Wishlist] Skipping {len(_unreleased)} unreleased track(s) "
|
||||
f"(future release date) — they'll be searched once released")
|
||||
|
||||
# CYCLE FILTERING: Get current cycle and filter tracks by category
|
||||
current_cycle = get_wishlist_cycle(lambda: music_database)
|
||||
|
||||
|
|
|
|||
|
|
@ -93,6 +93,29 @@ class YouTubeSearchResult:
|
|||
|
||||
from core.download_plugins.base import DownloadSourcePlugin
|
||||
|
||||
_JS_RUNTIME_WARNED = False
|
||||
|
||||
|
||||
def _warn_if_no_js_runtime():
|
||||
"""One-time startup check: YouTube gates downloadable formats behind JS
|
||||
challenges (nsig), and yt-dlp needs a JavaScript runtime (Deno, its only
|
||||
default-enabled one) to solve them. Without it every stream / music-video
|
||||
download dies with the cryptic "Requested format is not available" — say
|
||||
so plainly in the log instead. Docker images bundle Deno; bare-metal
|
||||
installs need it on PATH."""
|
||||
global _JS_RUNTIME_WARNED
|
||||
if _JS_RUNTIME_WARNED:
|
||||
return
|
||||
_JS_RUNTIME_WARNED = True
|
||||
import shutil as _sh
|
||||
if not _sh.which('deno'):
|
||||
logger.warning(
|
||||
"No JavaScript runtime found (deno not on PATH). YouTube streaming and "
|
||||
"music-video downloads will fail with 'Requested format is not available'. "
|
||||
"Install Deno (https://docs.deno.com/runtime/) and restart SoulSync. "
|
||||
"Windows: winget install DenoLand.Deno"
|
||||
)
|
||||
|
||||
|
||||
class YouTubeClient(DownloadSourcePlugin):
|
||||
"""
|
||||
|
|
@ -110,6 +133,7 @@ class YouTubeClient(DownloadSourcePlugin):
|
|||
self.download_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"YouTube client using download path: {self.download_path}")
|
||||
_warn_if_no_js_runtime()
|
||||
|
||||
# Callback for shutdown check (avoids circular imports)
|
||||
self.shutdown_check = None
|
||||
|
|
|
|||
|
|
@ -396,6 +396,7 @@ class MusicDatabase:
|
|||
|
||||
# Add per-artist preferred_metadata_source column (migration)
|
||||
self._add_watchlist_preferred_metadata_source_column(cursor)
|
||||
self._clear_deezer_ids_stored_as_itunes(cursor)
|
||||
|
||||
# Make spotify_artist_id nullable for iTunes-only artists (migration)
|
||||
self._fix_watchlist_spotify_id_nullable(cursor)
|
||||
|
|
@ -639,6 +640,15 @@ class MusicDatabase:
|
|||
cursor.execute(f"ALTER TABLE library_history ADD COLUMN {_col} TEXT")
|
||||
logger.info(f"Added {_col} column to library_history")
|
||||
|
||||
# Migration: download-origin provenance — what TRIGGERED a download
|
||||
# ('watchlist' + artist / 'playlist' + playlist name). Read by the
|
||||
# origin-history modal on the watchlist + sync pages.
|
||||
for _col in ['origin', 'origin_context']:
|
||||
if _col not in lh_cols:
|
||||
cursor.execute(f"ALTER TABLE library_history ADD COLUMN {_col} TEXT")
|
||||
logger.info(f"Added {_col} column to library_history")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_lh_origin ON library_history (origin, created_at DESC)")
|
||||
|
||||
# Auto-import history — tracks auto-import scan results and processing status
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS auto_import_history (
|
||||
|
|
@ -1920,6 +1930,31 @@ class MusicDatabase:
|
|||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_dab_name ON discovery_artist_blacklist (artist_name COLLATE NOCASE)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_dab_spotify ON discovery_artist_blacklist (spotify_artist_id)")
|
||||
|
||||
# Unified artist/album/track blocklist (the "proper" blacklist —
|
||||
# distinct from download_blacklist, which is source-file skipping).
|
||||
# ID-keyed across metadata sources so a ban survives a source
|
||||
# switch; profile-scoped; enforced at add_to_wishlist. The old
|
||||
# discovery_artist_blacklist is migrated in below.
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS blocklist (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
profile_id INTEGER NOT NULL DEFAULT 1,
|
||||
entity_type TEXT NOT NULL, -- 'artist' | 'album' | 'track'
|
||||
name TEXT NOT NULL COLLATE NOCASE,
|
||||
spotify_id TEXT,
|
||||
itunes_id TEXT,
|
||||
deezer_id TEXT,
|
||||
musicbrainz_id TEXT,
|
||||
parent_name TEXT, -- display only (album's/track's artist)
|
||||
match_status TEXT DEFAULT 'pending', -- pending | matched (cross-source backfill)
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_blocklist_profile_type ON blocklist (profile_id, entity_type)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_blocklist_spotify ON blocklist (spotify_id)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_blocklist_name ON blocklist (name COLLATE NOCASE)")
|
||||
self._migrate_discovery_blacklist_into_blocklist(cursor)
|
||||
|
||||
# Liked artists pool — aggregated followed/liked artists from connected services
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS liked_artists_pool (
|
||||
|
|
@ -2105,6 +2140,38 @@ class MusicDatabase:
|
|||
except Exception as e:
|
||||
logger.error(f"Error adding preferred_metadata_source column to watchlist_artists: {e}")
|
||||
|
||||
def _clear_deezer_ids_stored_as_itunes(self, cursor):
|
||||
"""Repair: watchlist iTunes ids that are actually Deezer ids.
|
||||
|
||||
The watchlist scanner's _match_to_itunes used to search via
|
||||
MetadataService.itunes — which holds the PRIMARY source's client, not
|
||||
iTunes — so with a Deezer primary it stored DEEZER artist ids in
|
||||
itunes_artist_id (verified live: Taylor Swift's "iTunes" id was her
|
||||
Deezer id 12246; real one is 159260351). The backfill only fills
|
||||
EMPTY ids, so these wrong ids would never self-heal. The corruption
|
||||
signature is itunes == deezer (distinct id spaces — a legit equal
|
||||
pair is effectively impossible; worst case is a NULL that re-matches
|
||||
correctly on the next scan). Idempotent: clearing kills the equality.
|
||||
"""
|
||||
try:
|
||||
cursor.execute("PRAGMA table_info(watchlist_artists)")
|
||||
columns = [column[1] for column in cursor.fetchall()]
|
||||
if 'itunes_artist_id' not in columns or 'deezer_artist_id' not in columns:
|
||||
return
|
||||
cursor.execute("""
|
||||
UPDATE watchlist_artists SET itunes_artist_id = NULL
|
||||
WHERE itunes_artist_id = deezer_artist_id
|
||||
AND deezer_artist_id IS NOT NULL AND deezer_artist_id != ''
|
||||
""")
|
||||
if cursor.rowcount:
|
||||
logger.info(
|
||||
"Cleared %d watchlist iTunes id(s) that were actually Deezer ids "
|
||||
"(pre-fix _match_to_itunes searched the primary source); they "
|
||||
"re-match via real iTunes on the next watchlist scan",
|
||||
cursor.rowcount)
|
||||
except Exception as e:
|
||||
logger.error(f"Error clearing deezer-as-itunes watchlist ids: {e}")
|
||||
|
||||
def _add_similar_artists_last_featured_column(self, cursor):
|
||||
"""Add last_featured column to similar_artists for hero slider cycling"""
|
||||
try:
|
||||
|
|
@ -7530,6 +7597,31 @@ class MusicDatabase:
|
|||
# Titles differ in length by more than 30% — penalize heavily
|
||||
best_title_similarity *= len_ratio
|
||||
|
||||
# #808: a parenthetical qualifier that merely RESTATES the release
|
||||
# context is album context, not a version difference. Wishlist
|
||||
# title 'Champagne Supernova (OurVinyl Sessions)' vs the library's
|
||||
# bare 'Champagne Supernova' on the album '… (OurVinyl Sessions)':
|
||||
# the qualifier appears in the album title, yet the length-ratio
|
||||
# penalty above crushed the pair to ~0.17 and wishlist cleanup
|
||||
# never recognised the owned edition. Strip qualifiers confirmed
|
||||
# by the db album title (or by the other title) and score that
|
||||
# variant with its OWN length guard — genuine version markers
|
||||
# ('(Live)' on a studio album) appear in no context, keep their
|
||||
# qualifier, and keep their penalty.
|
||||
db_album_norm = self._normalize_for_comparison(
|
||||
getattr(db_track, 'album_title', '') or '')
|
||||
from core.text.title_match import strip_redundant_context_qualifiers
|
||||
ctx_search = strip_redundant_context_qualifiers(
|
||||
search_title_norm, db_album_norm, db_title_norm)
|
||||
ctx_db = strip_redundant_context_qualifiers(
|
||||
db_title_norm, db_album_norm, search_title_norm)
|
||||
if (ctx_search, ctx_db) != (search_title_norm, db_title_norm) and ctx_search and ctx_db:
|
||||
ctx_sim = self._string_similarity(ctx_search, ctx_db)
|
||||
ctx_ratio = min(len(ctx_search), len(ctx_db)) / max(len(ctx_search), len(ctx_db))
|
||||
if ctx_ratio < 0.7:
|
||||
ctx_sim *= ctx_ratio # 'Believe' vs 'Believe In Me' still penalised
|
||||
best_title_similarity = max(best_title_similarity, ctx_sim)
|
||||
|
||||
# Word-level guard: SequenceMatcher's char ratio over-credits
|
||||
# different songs that share a long substring or only a stopword
|
||||
# ("Dani California" vs "Californication" = 0.67; "Under The Bridge"
|
||||
|
|
@ -8007,7 +8099,49 @@ class MusicDatabase:
|
|||
return presets.get(preset_name, presets["balanced"])
|
||||
|
||||
# Wishlist management methods
|
||||
|
||||
|
||||
def blocklist_reason_for_track(self, profile_id, track_data, source=None):
|
||||
"""Return (entity_type, label) if this track is blocklisted for the
|
||||
profile, else None. Shared by the wishlist guard (Phase 1) and the
|
||||
download-queue guard (Phase 2a). Pure matching lives in core.blocklist;
|
||||
this pulls the candidate's source IDs out of the payload and asks.
|
||||
|
||||
``source`` overrides/falls back to the payload's provider — the
|
||||
download-queue path knows the batch source even when the track dict
|
||||
doesn't carry a 'provider' field."""
|
||||
try:
|
||||
from core.blocklist import build_index, candidate_block_reason
|
||||
rows = self.get_blocklist_rows_for_matching(profile_id)
|
||||
if not rows:
|
||||
return None
|
||||
index = build_index(rows)
|
||||
if index.is_empty:
|
||||
return None
|
||||
td = track_data or {}
|
||||
source = ((td.get('provider') or td.get('source') or source or '')
|
||||
.strip().lower() or None)
|
||||
album = td.get('album') if isinstance(td.get('album'), dict) else {}
|
||||
# Normalise artists to [{'id','name'}] from track + album credits.
|
||||
artists = []
|
||||
for a in (td.get('artists') or []):
|
||||
if isinstance(a, dict):
|
||||
artists.append({'id': a.get('id'), 'name': a.get('name')})
|
||||
elif a:
|
||||
artists.append({'id': None, 'name': str(a)})
|
||||
for a in (album.get('artists') or []):
|
||||
if isinstance(a, dict):
|
||||
artists.append({'id': a.get('id'), 'name': a.get('name')})
|
||||
return candidate_block_reason(
|
||||
index, source=source,
|
||||
track_id=td.get('id'), track_name=td.get('name'),
|
||||
album_id=album.get('id'), album_name=album.get('name'),
|
||||
artists=artists,
|
||||
)
|
||||
except Exception as e:
|
||||
# Never let the blocklist check break a wishlist add — fail open.
|
||||
logger.debug("blocklist guard skipped: %s", e)
|
||||
return None
|
||||
|
||||
def add_to_wishlist(
|
||||
self,
|
||||
spotify_track_data: Dict[str, Any] = None,
|
||||
|
|
@ -8031,6 +8165,15 @@ class MusicDatabase:
|
|||
logger.error("Cannot add track to wishlist: missing track ID")
|
||||
return False
|
||||
|
||||
# Blocklist guard (Phase 1): every auto-acquisition path funnels
|
||||
# through here, so one check blocks a banned artist/album/track
|
||||
# (with artist→album→track cascade) before it can be queued.
|
||||
_blocked = self.blocklist_reason_for_track(profile_id, spotify_track_data)
|
||||
if _blocked:
|
||||
logger.info("Skipping wishlist add — %s is blocklisted: '%s'",
|
||||
_blocked[0], _blocked[1])
|
||||
return False
|
||||
|
||||
from core.library import manual_library_match as _mlm
|
||||
if _mlm.get_match_for_track(self, profile_id, spotify_track_data):
|
||||
logger.info(
|
||||
|
|
@ -10931,16 +11074,197 @@ class MusicDatabase:
|
|||
return []
|
||||
|
||||
def get_discovery_blacklist_names(self) -> set:
|
||||
"""Get set of blacklisted artist names (lowercased) for fast filtering."""
|
||||
"""Set of blacklisted artist names (lowercased) for discovery filtering.
|
||||
|
||||
Unions the legacy discovery_artist_blacklist with the new unified
|
||||
blocklist's artist entries (across all profiles), so a ban added via
|
||||
either path filters discovery. The legacy table is migrated into the
|
||||
blocklist on upgrade but kept as a rollback safety net."""
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT LOWER(artist_name) FROM discovery_artist_blacklist")
|
||||
return {r[0] for r in cursor.fetchall()}
|
||||
names = {r[0] for r in cursor.fetchall()}
|
||||
try:
|
||||
cursor.execute("SELECT LOWER(name) FROM blocklist WHERE entity_type = 'artist'")
|
||||
names.update(r[0] for r in cursor.fetchall())
|
||||
except Exception as _bl_err: # noqa: BLE001 — old schema may predate blocklist
|
||||
logger.debug("blocklist union skipped in discovery names: %s", _bl_err)
|
||||
return names
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting discovery blacklist names: {e}")
|
||||
return set()
|
||||
|
||||
# ==================== Blocklist (artist/album/track) ====================
|
||||
|
||||
def _migrate_discovery_blacklist_into_blocklist(self, cursor):
|
||||
"""One-time safe migration of the legacy global discovery blacklist into
|
||||
the new profile-scoped blocklist as artist entries.
|
||||
|
||||
Replicated to EVERY existing profile so no existing discovery ban
|
||||
silently stops working under the new per-profile model. Idempotent
|
||||
(skips a (profile, name) already present). The old table is left in
|
||||
place as a rollback safety net."""
|
||||
try:
|
||||
cursor.execute(
|
||||
"SELECT artist_name, spotify_artist_id, itunes_artist_id, deezer_artist_id "
|
||||
"FROM discovery_artist_blacklist")
|
||||
legacy = cursor.fetchall()
|
||||
if not legacy:
|
||||
return
|
||||
try:
|
||||
cursor.execute("SELECT id FROM profiles")
|
||||
profile_ids = [r[0] for r in cursor.fetchall()] or [1]
|
||||
except Exception:
|
||||
profile_ids = [1]
|
||||
|
||||
migrated = 0
|
||||
for row in legacy:
|
||||
name = row[0]
|
||||
if not name:
|
||||
continue
|
||||
for pid in profile_ids:
|
||||
cursor.execute(
|
||||
"SELECT 1 FROM blocklist WHERE profile_id = ? AND entity_type = 'artist' "
|
||||
"AND name = ? COLLATE NOCASE LIMIT 1", (pid, name))
|
||||
if cursor.fetchone():
|
||||
continue
|
||||
cursor.execute(
|
||||
"INSERT INTO blocklist (profile_id, entity_type, name, spotify_id, "
|
||||
"itunes_id, deezer_id, match_status) VALUES (?, 'artist', ?, ?, ?, ?, 'matched')",
|
||||
(pid, name, row[1], row[2], row[3]))
|
||||
migrated += 1
|
||||
if migrated:
|
||||
logger.info("Migrated %d discovery-blacklist artist entr(ies) into the "
|
||||
"unified blocklist across %d profile(s)", migrated, len(profile_ids))
|
||||
except Exception as e:
|
||||
logger.debug("discovery→blocklist migration skipped: %s", e)
|
||||
|
||||
def add_blocklist_entry(self, profile_id: int, entity_type: str, name: str,
|
||||
spotify_id: str = None, itunes_id: str = None,
|
||||
deezer_id: str = None, musicbrainz_id: str = None,
|
||||
parent_name: str = None) -> Optional[int]:
|
||||
"""Add an artist/album/track to the blocklist. Returns the new row id,
|
||||
or an existing row's id if a matching (profile, type, id/name) is already
|
||||
present. match_status starts 'pending' until the backfill resolves the
|
||||
other sources (unless we already have multiple ids)."""
|
||||
if entity_type not in ('artist', 'album', 'track') or not name:
|
||||
return None
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
# Dedup: same profile+type with any overlapping source id, or same name.
|
||||
cursor.execute(
|
||||
"""SELECT id FROM blocklist WHERE profile_id = ? AND entity_type = ?
|
||||
AND ( (spotify_id IS NOT NULL AND spotify_id = ?)
|
||||
OR (itunes_id IS NOT NULL AND itunes_id = ?)
|
||||
OR (deezer_id IS NOT NULL AND deezer_id = ?)
|
||||
OR (musicbrainz_id IS NOT NULL AND musicbrainz_id = ?)
|
||||
OR name = ? COLLATE NOCASE ) LIMIT 1""",
|
||||
(profile_id, entity_type, spotify_id, itunes_id, deezer_id, musicbrainz_id, name))
|
||||
existing = cursor.fetchone()
|
||||
if existing:
|
||||
return existing[0]
|
||||
id_count = sum(1 for x in (spotify_id, itunes_id, deezer_id, musicbrainz_id) if x)
|
||||
status = 'matched' if id_count >= 2 else 'pending'
|
||||
cursor.execute(
|
||||
"""INSERT INTO blocklist (profile_id, entity_type, name, spotify_id, itunes_id,
|
||||
deezer_id, musicbrainz_id, parent_name, match_status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(profile_id, entity_type, name, spotify_id, itunes_id, deezer_id,
|
||||
musicbrainz_id, parent_name, status))
|
||||
conn.commit()
|
||||
return cursor.lastrowid
|
||||
except Exception as e:
|
||||
logger.error(f"Error adding blocklist entry: {e}")
|
||||
return None
|
||||
|
||||
def remove_blocklist_entry(self, profile_id: int, entry_id: int) -> bool:
|
||||
"""Remove a blocklist entry (scoped to the profile that owns it)."""
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM blocklist WHERE id = ? AND profile_id = ?",
|
||||
(int(entry_id), profile_id))
|
||||
conn.commit()
|
||||
return cursor.rowcount > 0
|
||||
except Exception as e:
|
||||
logger.error(f"Error removing blocklist entry: {e}")
|
||||
return False
|
||||
|
||||
def get_blocklist(self, profile_id: int, entity_type: str = None) -> list:
|
||||
"""List blocklist entries for a profile, newest first."""
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
if entity_type:
|
||||
cursor.execute(
|
||||
"SELECT id, profile_id, entity_type, name, spotify_id, itunes_id, deezer_id, "
|
||||
"musicbrainz_id, parent_name, match_status, created_at FROM blocklist "
|
||||
"WHERE profile_id = ? AND entity_type = ? ORDER BY created_at DESC",
|
||||
(profile_id, entity_type))
|
||||
else:
|
||||
cursor.execute(
|
||||
"SELECT id, profile_id, entity_type, name, spotify_id, itunes_id, deezer_id, "
|
||||
"musicbrainz_id, parent_name, match_status, created_at FROM blocklist "
|
||||
"WHERE profile_id = ? ORDER BY created_at DESC", (profile_id,))
|
||||
return [dict(r) for r in cursor.fetchall()]
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting blocklist: {e}")
|
||||
return []
|
||||
|
||||
def get_blocklist_rows_for_matching(self, profile_id: int) -> list:
|
||||
"""Lightweight rows (entity_type + id columns + name) for building the
|
||||
in-memory match index — used by the add_to_wishlist guard."""
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT entity_type, name, spotify_id, itunes_id, deezer_id, musicbrainz_id "
|
||||
"FROM blocklist WHERE profile_id = ?", (profile_id,))
|
||||
return [dict(r) for r in cursor.fetchall()]
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting blocklist match rows: {e}")
|
||||
return []
|
||||
|
||||
def get_blocklist_entries_needing_backfill(self) -> list:
|
||||
"""Entries still 'pending' cross-source ID resolution."""
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT id, profile_id, entity_type, name, spotify_id, itunes_id, deezer_id, "
|
||||
"musicbrainz_id, parent_name FROM blocklist WHERE match_status = 'pending'")
|
||||
return [dict(r) for r in cursor.fetchall()]
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting blocklist backfill entries: {e}")
|
||||
return []
|
||||
|
||||
def update_blocklist_entry_ids(self, entry_id: int, *, spotify_id: str = None,
|
||||
itunes_id: str = None, deezer_id: str = None,
|
||||
musicbrainz_id: str = None, mark_matched: bool = True) -> bool:
|
||||
"""Backfill resolved source IDs onto an entry (only fills NULLs)."""
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
sets, params = [], []
|
||||
for col, val in (("spotify_id", spotify_id), ("itunes_id", itunes_id),
|
||||
("deezer_id", deezer_id), ("musicbrainz_id", musicbrainz_id)):
|
||||
if val:
|
||||
sets.append(f"{col} = COALESCE({col}, ?)")
|
||||
params.append(val)
|
||||
if mark_matched:
|
||||
sets.append("match_status = 'matched'")
|
||||
if not sets:
|
||||
return False
|
||||
params.append(int(entry_id))
|
||||
cursor.execute(f"UPDATE blocklist SET {', '.join(sets)} WHERE id = ?", params)
|
||||
conn.commit()
|
||||
return cursor.rowcount > 0
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating blocklist entry ids: {e}")
|
||||
return False
|
||||
|
||||
# ==================== Liked Artists Pool Methods ====================
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -12129,8 +12453,13 @@ class MusicDatabase:
|
|||
def add_library_history_entry(self, event_type, title, artist_name=None, album_name=None,
|
||||
quality=None, server_source=None, file_path=None, thumb_url=None,
|
||||
download_source=None, source_track_id=None, source_track_title=None,
|
||||
source_filename=None, acoustid_result=None, source_artist=None):
|
||||
"""Record a download or import event to the library history table."""
|
||||
source_filename=None, acoustid_result=None, source_artist=None,
|
||||
origin=None, origin_context=None):
|
||||
"""Record a download or import event to the library history table.
|
||||
|
||||
``origin``/``origin_context`` record what TRIGGERED the download
|
||||
('watchlist' + artist name, 'playlist' + playlist name) — the
|
||||
origin-history modal reads them. None for manual/unclassified."""
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
|
@ -12138,17 +12467,113 @@ class MusicDatabase:
|
|||
INSERT INTO library_history (event_type, title, artist_name, album_name,
|
||||
quality, server_source, file_path, thumb_url, download_source,
|
||||
source_track_id, source_track_title, source_filename,
|
||||
acoustid_result, source_artist)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
acoustid_result, source_artist, origin, origin_context)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (event_type, title, artist_name, album_name, quality, server_source, file_path, thumb_url,
|
||||
download_source, source_track_id, source_track_title, source_filename,
|
||||
acoustid_result, source_artist))
|
||||
acoustid_result, source_artist, origin, origin_context))
|
||||
conn.commit()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"Error adding library history entry: {e}")
|
||||
return False
|
||||
|
||||
def get_origin_cleanup_candidates(self):
|
||||
"""Origin-tracked downloads (watchlist/playlist) annotated with the
|
||||
matching library track's play_count, for the Expired Download Cleaner.
|
||||
play_count is 0 when no library track matches the recorded path
|
||||
(orphan history row → treated as not-listened)."""
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT lh.id, lh.origin, lh.origin_context, lh.created_at,
|
||||
lh.file_path, lh.title, lh.artist_name,
|
||||
COALESCE(t.play_count, 0) AS play_count
|
||||
FROM library_history lh
|
||||
LEFT JOIN tracks t ON t.file_path = lh.file_path
|
||||
WHERE lh.event_type = 'download'
|
||||
AND lh.origin IN ('watchlist', 'playlist')
|
||||
""")
|
||||
cols = ['id', 'origin', 'origin_context', 'created_at',
|
||||
'file_path', 'title', 'artist_name', 'play_count']
|
||||
return [dict(zip(cols, row, strict=True)) for row in cursor.fetchall()]
|
||||
except Exception as e:
|
||||
logger.debug(f"Error getting origin cleanup candidates: {e}")
|
||||
return []
|
||||
|
||||
def get_download_origin_entries(self, origin, limit=200, offset=0):
|
||||
"""Downloads triggered by ``origin`` ('watchlist' / 'playlist'),
|
||||
newest first. Returns (entries, total_count)."""
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT COUNT(*) FROM library_history WHERE event_type = 'download' AND origin = ?",
|
||||
(origin,))
|
||||
total = cursor.fetchone()[0]
|
||||
cursor.execute("""
|
||||
SELECT id, title, artist_name, album_name, quality, file_path,
|
||||
thumb_url, download_source, origin, origin_context, created_at
|
||||
FROM library_history
|
||||
WHERE event_type = 'download' AND origin = ?
|
||||
ORDER BY created_at DESC, id DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""", (origin, int(limit), int(offset)))
|
||||
cols = ['id', 'title', 'artist_name', 'album_name', 'quality', 'file_path',
|
||||
'thumb_url', 'download_source', 'origin', 'origin_context', 'created_at']
|
||||
return [dict(zip(cols, row, strict=True)) for row in cursor.fetchall()], total
|
||||
except Exception as e:
|
||||
logger.debug(f"Error querying download origins: {e}")
|
||||
return [], 0
|
||||
|
||||
def get_library_history_rows_by_ids(self, ids):
|
||||
"""Fetch history rows (id, file_path, title) for a list of ids."""
|
||||
if not ids:
|
||||
return []
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
placeholders = ','.join('?' * len(ids))
|
||||
cursor.execute(
|
||||
f"SELECT id, file_path, title FROM library_history WHERE id IN ({placeholders})",
|
||||
[int(i) for i in ids])
|
||||
return [{'id': r[0], 'file_path': r[1], 'title': r[2]} for r in cursor.fetchall()]
|
||||
except Exception as e:
|
||||
logger.debug(f"Error fetching history rows: {e}")
|
||||
return []
|
||||
|
||||
def delete_library_history_rows(self, ids):
|
||||
"""Delete history rows by id. Returns the number removed."""
|
||||
if not ids:
|
||||
return 0
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
placeholders = ','.join('?' * len(ids))
|
||||
cursor.execute(
|
||||
f"DELETE FROM library_history WHERE id IN ({placeholders})",
|
||||
[int(i) for i in ids])
|
||||
conn.commit()
|
||||
return cursor.rowcount
|
||||
except Exception as e:
|
||||
logger.debug(f"Error deleting history rows: {e}")
|
||||
return 0
|
||||
|
||||
def delete_track_by_file_path(self, file_path):
|
||||
"""Delete a library track row whose stored path matches. Returns count."""
|
||||
if not file_path:
|
||||
return 0
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DELETE FROM tracks WHERE file_path = ?", (file_path,))
|
||||
conn.commit()
|
||||
return cursor.rowcount
|
||||
except Exception as e:
|
||||
logger.debug(f"Error deleting track by path: {e}")
|
||||
return 0
|
||||
|
||||
def get_library_history(self, event_type=None, page=1, limit=50):
|
||||
"""Query library history with optional type filter and pagination.
|
||||
|
||||
|
|
|
|||
31
pr.md
Normal file
31
pr.md
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
# 2.6.6 → 2.6.7
|
||||
|
||||
big release. headline stuff:
|
||||
|
||||
**spotify free (#798)** — new no-credentials spotify metadata source. pick it in settings → metadata and you get spotify search + enrichment without connecting an account (uses the public web-player data). for connected users it also bridges rate-limit bans automatically: if your real auth gets banned, enrichment keeps running through the free source instead of stalling, then snaps back to your real auth once the ban lifts. resumable mid-ban, shows "running (spotify free)" instead of looking stuck, and the daily api budget never pauses a spotify-free user — free work isn't counted against it, and once the real-api budget is spent for the day it switches to free (uncapped) instead of stopping, back to real auth on the daily reset.
|
||||
|
||||
**import ids from file tags** — your files already carry the spotify/musicbrainz/itunes/deezer/etc ids picard or soulsync embedded, but the media-server scan can't see them. this reads them straight into the db so the enrichment workers skip those lookups — big api savings on an already-tagged library. gap-fill only, never overwrites a match. new tracks get it automatically as the last phase of every library scan too.
|
||||
|
||||
**library re-tag** — proper re-tag job, replaces the old retag tool. matches each file to its source tracklist and rewrites tags + cover art + the embedded source ids, with a per-track old→new diff and a dry run before anything's written.
|
||||
|
||||
**paste a link (#775)** — paste a spotify/itunes/deezer artist/album/track url on the search page and it opens that exact item instead of running a name search.
|
||||
|
||||
**mobile v2 (#793, #795)** — full responsive pass. artist page, enhanced track table, player + now-playing, sync buttons, discover carousels, downloads, notification panel — all usable on a phone now.
|
||||
|
||||
**reconcile sync mode (#792)** — new playlist sync mode that edits the server playlist in place (keeps your custom image/description) instead of delete-and-recreate.
|
||||
|
||||
## fixes
|
||||
- **#758** manual album match now LOCKS the edition — the auto canonical resolver can't drag it back to the deluxe
|
||||
- **#800** write tags won't overwrite a correct file with placeholder junk (various artists / [unknown album])
|
||||
- **#797** acoustid stops false-quarantining correct downloads of non-english artists
|
||||
- **#799** manual playlist fixes stop reverting to "wing it" on the next mirrored sync
|
||||
- **#787** find & add matches survive a library rescan
|
||||
- **#789** navidrome respects the selected music library (and survives renames)
|
||||
- **#785** file/csv playlists match raw "artist - title" titles
|
||||
- **#790** torrent client url without http:// connects
|
||||
- **#796** soulseek album bundle stops leaving completed files in the slskd folder
|
||||
- streamed tracks no longer play silent (suspended web-audio context)
|
||||
- wrong artist on duplicated/ambiguous source ids fixed (the kendrick/jorja class of bug) + a one-time startup repair
|
||||
- candidate_pool spotify-sync crash already squashed
|
||||
|
||||
~69 commits since 2.6.6. full changelog lives in the what's new panel. suite's clean — only the pre-existing soundcloud /app env failures remain.
|
||||
|
|
@ -46,7 +46,14 @@ tzdata>=2024.1
|
|||
# keeps existing schedules running at the same wall-clock time.
|
||||
tzlocal>=5.0
|
||||
|
||||
# YouTube support -- unpinned; yt-dlp must track upstream releases to stay functional
|
||||
# YouTube support -- unpinned; yt-dlp must track upstream releases to stay functional.
|
||||
# NOTE: pip resolves this to the latest STABLE release, which can lag months
|
||||
# behind a breaking YouTube change. If streams/music-video downloads fail with
|
||||
# "Requested format is not available", switch to the nightly channel:
|
||||
# pip install -U --pre "yt-dlp[default]"
|
||||
# YouTube extraction ALSO requires a JavaScript runtime (Deno) on the system --
|
||||
# it is NOT a Python package and cannot be listed here. Docker images bundle
|
||||
# it; bare-metal installs need it on PATH: https://docs.deno.com/runtime/
|
||||
yt-dlp>=2026.3.17
|
||||
|
||||
# Lyrics support
|
||||
|
|
|
|||
0
tests/blocklist/__init__.py
Normal file
0
tests/blocklist/__init__.py
Normal file
48
tests/blocklist/test_backfill.py
Normal file
48
tests/blocklist/test_backfill.py
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
"""Cross-source ID backfill (pure resolver layer)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.blocklist.backfill import resolve_missing_ids
|
||||
|
||||
|
||||
def _resolvers(**by_source):
|
||||
# each value is the id that source returns (or None)
|
||||
return {src: (lambda et, n, p, _v=v: _v) for src, v in by_source.items()}
|
||||
|
||||
|
||||
def test_fills_only_missing_sources():
|
||||
entry = {"entity_type": "artist", "name": "Drake", "spotify_id": "sp-known"}
|
||||
resolvers = _resolvers(spotify="SHOULD-NOT-BE-USED", itunes="it-new", deezer="dz-new")
|
||||
out = resolve_missing_ids(entry, resolvers)
|
||||
assert out == {"itunes_id": "it-new", "deezer_id": "dz-new"} # spotify skipped (known)
|
||||
|
||||
|
||||
def test_resolver_returning_none_leaves_source_unmatched():
|
||||
entry = {"entity_type": "album", "name": "Some Album"}
|
||||
resolvers = _resolvers(spotify="sp", itunes=None, deezer=None, musicbrainz="mb")
|
||||
out = resolve_missing_ids(entry, resolvers)
|
||||
assert out == {"spotify_id": "sp", "musicbrainz_id": "mb"}
|
||||
|
||||
|
||||
def test_resolver_exception_is_swallowed():
|
||||
def boom(et, n, p):
|
||||
raise RuntimeError("source down")
|
||||
entry = {"entity_type": "artist", "name": "X"}
|
||||
out = resolve_missing_ids(entry, {"spotify": boom, "deezer": lambda et, n, p: "dz"})
|
||||
assert out == {"deezer_id": "dz"}
|
||||
|
||||
|
||||
def test_no_name_or_type_returns_empty():
|
||||
assert resolve_missing_ids({"entity_type": "artist"}, _resolvers(spotify="x")) == {}
|
||||
assert resolve_missing_ids({"name": "X"}, _resolvers(spotify="x")) == {}
|
||||
|
||||
|
||||
def test_resolver_receives_type_name_and_parent():
|
||||
seen = {}
|
||||
def capture(et, n, p):
|
||||
seen.update(entity_type=et, name=n, parent=p)
|
||||
return "id1"
|
||||
resolve_missing_ids(
|
||||
{"entity_type": "album", "name": "Scorpion", "parent_name": "Drake"},
|
||||
{"spotify": capture})
|
||||
assert seen == {"entity_type": "album", "name": "Scorpion", "parent": "Drake"}
|
||||
114
tests/blocklist/test_blocklist_api.py
Normal file
114
tests/blocklist/test_blocklist_api.py
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
"""Blocklist HTTP API: search / add (+ synchronous cross-source backfill) /
|
||||
list / delete, end to end through the Flask test client."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
web_server = pytest.importorskip("web_server")
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client(monkeypatch):
|
||||
web_server.app.config["TESTING"] = True
|
||||
monkeypatch.setattr(web_server, "get_current_profile_id", lambda: 1)
|
||||
return web_server.app.test_client()
|
||||
|
||||
|
||||
def test_search_proxies_active_source(client):
|
||||
with patch.object(web_server, "_search_service", return_value=[
|
||||
{"id": "drake-sp", "name": "Drake", "image": None, "extra": "", "provider": "spotify"}]):
|
||||
r = client.get("/api/blocklist/search?type=artist&q=drake")
|
||||
assert r.status_code == 200
|
||||
body = r.get_json()
|
||||
assert body["success"] and body["results"][0]["name"] == "Drake"
|
||||
|
||||
|
||||
def test_add_resolves_other_sources_then_list_and_delete(client):
|
||||
resolvers = {
|
||||
"itunes": lambda et, n, p: "drake-it",
|
||||
"deezer": lambda et, n, p: None,
|
||||
"spotify": lambda et, n, p: None,
|
||||
"musicbrainz": lambda et, n, p: None,
|
||||
}
|
||||
with patch("core.blocklist.runtime.build_resolvers", return_value=resolvers):
|
||||
r = client.post("/api/blocklist", json={
|
||||
"entity_type": "artist", "name": "Drake Test One",
|
||||
"source": "spotify", "source_id": "drake-sp1"})
|
||||
assert r.status_code == 200 and r.get_json()["success"]
|
||||
eid = r.get_json()["id"]
|
||||
|
||||
rows = client.get("/api/blocklist?entity_type=artist").get_json()["entries"]
|
||||
row = next(x for x in rows if x["id"] == eid)
|
||||
assert row["spotify_id"] == "drake-sp1"
|
||||
assert row["itunes_id"] == "drake-it" # resolved at add time
|
||||
assert row["match_status"] == "matched"
|
||||
|
||||
assert client.delete(f"/api/blocklist/{eid}").get_json()["success"] is True
|
||||
rows = client.get("/api/blocklist?entity_type=artist").get_json()["entries"]
|
||||
assert all(x["id"] != eid for x in rows)
|
||||
|
||||
|
||||
def test_add_requires_type_and_name(client):
|
||||
r = client.post("/api/blocklist", json={"entity_type": "artist"})
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
def test_invalid_entity_type_rejected(client):
|
||||
assert client.get("/api/blocklist?entity_type=bogus").status_code == 400
|
||||
assert client.get("/api/blocklist/search?type=bogus&q=x").status_code == 400
|
||||
|
||||
|
||||
# ── Phase 2b: download-time guards (manual + modal up-front) ─────────────────
|
||||
|
||||
@pytest.fixture()
|
||||
def banned_artist(client):
|
||||
"""Block 'Banned Guy' for the test profile; clean up after."""
|
||||
db = web_server.get_database()
|
||||
eid = db.add_blocklist_entry(1, "artist", "Banned Guy", spotify_id="bg-sp")
|
||||
yield
|
||||
db.remove_blocklist_entry(1, eid)
|
||||
|
||||
|
||||
def test_manual_download_blocked_by_artist_name(client, banned_artist):
|
||||
r = client.post("/api/download", json={
|
||||
"result_type": "track", "username": "peer", "filename": "x.flac",
|
||||
"artist": "Banned Guy", "title": "Song"})
|
||||
assert r.status_code == 409
|
||||
body = r.get_json()
|
||||
assert body["blocked"] is True and body["blocked_entity_type"] == "artist"
|
||||
|
||||
|
||||
def test_manual_download_unrelated_artist_not_blocked(client, banned_artist):
|
||||
# Allowed artist → guard passes (the download itself may fail offline; we
|
||||
# only assert it wasn't blocked by the blocklist).
|
||||
r = client.post("/api/download", json={
|
||||
"result_type": "track", "username": "peer", "filename": "y.flac",
|
||||
"artist": "Allowed Artist", "title": "Song"})
|
||||
assert not (r.get_json() or {}).get("blocked")
|
||||
|
||||
|
||||
def test_manual_download_override_passes_guard(client, banned_artist):
|
||||
r = client.post("/api/download", json={
|
||||
"result_type": "track", "username": "peer", "filename": "x.flac",
|
||||
"artist": "Banned Guy", "title": "Song", "ignore_blocklist": True})
|
||||
assert not (r.get_json() or {}).get("blocked") # override skips the guard
|
||||
|
||||
|
||||
def test_modal_blocked_album_returns_409_before_starting(client):
|
||||
db = web_server.get_database()
|
||||
eid = db.add_blocklist_entry(1, "album", "Banned Album", spotify_id="ba-sp")
|
||||
try:
|
||||
r = client.post("/api/playlists/artist_album_test/start-missing-process", json={
|
||||
"tracks": [{"id": "t1", "name": "Track"}],
|
||||
"is_album_download": True,
|
||||
"album_context": {"id": "ba-sp", "name": "Banned Album"},
|
||||
"artist_context": {"id": "ar1", "name": "Someone"},
|
||||
})
|
||||
assert r.status_code == 409
|
||||
body = r.get_json()
|
||||
assert body["blocked"] is True and body["blocked_entity_type"] == "album"
|
||||
finally:
|
||||
db.remove_blocklist_entry(1, eid)
|
||||
183
tests/blocklist/test_blocklist_db.py
Normal file
183
tests/blocklist/test_blocklist_db.py
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
"""Blocklist DB layer + the add_to_wishlist enforcement guard.
|
||||
|
||||
Verified against a real SQLite DB: CRUD + profile scoping + dedup + backfill
|
||||
update, the discovery→blocklist migration, and the end-to-end wishlist guard
|
||||
(a banned artist's track is refused; cascade; profile isolation).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db(tmp_path):
|
||||
return MusicDatabase(str(tmp_path / "m.db"))
|
||||
|
||||
|
||||
def _track(track_id, name, artist_id, artist_name, album_id="al0", album_name="Al", source="spotify"):
|
||||
return {
|
||||
"id": track_id, "name": name, "source": source,
|
||||
"artists": [{"id": artist_id, "name": artist_name}],
|
||||
"album": {"id": album_id, "name": album_name, "artists": [{"id": artist_id, "name": artist_name}]},
|
||||
}
|
||||
|
||||
|
||||
# ── CRUD + scoping ───────────────────────────────────────────────────────────
|
||||
|
||||
def test_add_list_remove(db):
|
||||
eid = db.add_blocklist_entry(1, "artist", "Drake", spotify_id="drake-sp")
|
||||
assert eid
|
||||
rows = db.get_blocklist(1)
|
||||
assert len(rows) == 1 and rows[0]["name"] == "Drake"
|
||||
assert db.remove_blocklist_entry(1, eid) is True
|
||||
assert db.get_blocklist(1) == []
|
||||
|
||||
|
||||
def test_dedup_by_id_and_name(db):
|
||||
a = db.add_blocklist_entry(1, "artist", "Drake", spotify_id="sp")
|
||||
b = db.add_blocklist_entry(1, "artist", "Drake", spotify_id="sp") # same id
|
||||
c = db.add_blocklist_entry(1, "artist", "drake") # same name (NOCASE)
|
||||
assert a == b == c
|
||||
assert len(db.get_blocklist(1)) == 1
|
||||
|
||||
|
||||
def test_profile_isolation(db):
|
||||
db.add_blocklist_entry(1, "artist", "Drake", spotify_id="sp")
|
||||
assert len(db.get_blocklist(1)) == 1
|
||||
assert db.get_blocklist(2) == []
|
||||
# remove is profile-scoped — profile 2 can't delete profile 1's row
|
||||
eid = db.get_blocklist(1)[0]["id"]
|
||||
assert db.remove_blocklist_entry(2, eid) is False
|
||||
assert len(db.get_blocklist(1)) == 1
|
||||
|
||||
|
||||
def test_match_status_pending_vs_matched(db):
|
||||
one = db.add_blocklist_entry(1, "artist", "A", spotify_id="sp")
|
||||
two = db.add_blocklist_entry(1, "artist", "B", spotify_id="sp2", deezer_id="dz2")
|
||||
rows = {r["id"]: r for r in db.get_blocklist(1)}
|
||||
assert rows[one]["match_status"] == "pending" # single id → needs backfill
|
||||
assert rows[two]["match_status"] == "matched" # 2+ ids → already cross-known
|
||||
|
||||
|
||||
def test_backfill_update_fills_only_nulls(db):
|
||||
eid = db.add_blocklist_entry(1, "artist", "A", spotify_id="sp")
|
||||
db.update_blocklist_entry_ids(eid, deezer_id="dz", spotify_id="SHOULD-NOT-OVERWRITE")
|
||||
row = db.get_blocklist(1)[0]
|
||||
assert row["spotify_id"] == "sp" # COALESCE: existing kept
|
||||
assert row["deezer_id"] == "dz"
|
||||
assert row["match_status"] == "matched"
|
||||
assert db.get_blocklist_entries_needing_backfill() == []
|
||||
|
||||
|
||||
# ── the wishlist guard, end to end ───────────────────────────────────────────
|
||||
|
||||
def test_blocked_artist_track_refused_from_wishlist(db):
|
||||
db.add_blocklist_entry(1, "artist", "Drake", spotify_id="drake-sp")
|
||||
ok = db.add_to_wishlist(
|
||||
spotify_track_data=_track("t1", "God's Plan", "drake-sp", "Drake"),
|
||||
profile_id=1)
|
||||
assert ok is False # refused
|
||||
# And nothing landed in the wishlist.
|
||||
assert db.get_wishlist_count(profile_id=1) == 0
|
||||
|
||||
|
||||
def test_unblocked_artist_track_is_added(db):
|
||||
db.add_blocklist_entry(1, "artist", "Drake", spotify_id="drake-sp")
|
||||
ok = db.add_to_wishlist(
|
||||
spotify_track_data=_track("t2", "Hello", "adele-sp", "Adele"),
|
||||
profile_id=1)
|
||||
assert ok is True
|
||||
assert db.get_wishlist_count(profile_id=1) == 1
|
||||
|
||||
|
||||
def test_block_is_profile_scoped_at_the_guard(db):
|
||||
db.add_blocklist_entry(1, "artist", "Drake", spotify_id="drake-sp")
|
||||
# Profile 2 has no such ban → the same track is allowed.
|
||||
ok = db.add_to_wishlist(
|
||||
spotify_track_data=_track("t3", "Nice For What", "drake-sp", "Drake"),
|
||||
profile_id=2)
|
||||
assert ok is True
|
||||
|
||||
|
||||
def test_album_block_cascades_to_track_at_guard(db):
|
||||
db.add_blocklist_entry(1, "album", "Scorpion", spotify_id="scorp-sp")
|
||||
ok = db.add_to_wishlist(
|
||||
spotify_track_data=_track("t4", "Nonstop", "drake-sp", "Drake",
|
||||
album_id="scorp-sp", album_name="Scorpion"),
|
||||
profile_id=1)
|
||||
assert ok is False
|
||||
|
||||
|
||||
def test_discovery_blacklist_migrated_into_blocklist(tmp_path):
|
||||
import database.music_database as mdb
|
||||
|
||||
def _reinit(path):
|
||||
# An app upgrade = a fresh process with an empty init memo. Clear the
|
||||
# per-process "already initialized" set so init (and the migration)
|
||||
# actually re-runs against the existing DB file.
|
||||
mdb._database_initialized_paths.discard(str(mdb.Path(path).resolve()))
|
||||
return MusicDatabase(path)
|
||||
|
||||
path = str(tmp_path / "mig.db")
|
||||
db = MusicDatabase(path)
|
||||
conn = db._get_connection()
|
||||
cur = conn.cursor()
|
||||
cur.execute("INSERT INTO discovery_artist_blacklist (artist_name, spotify_artist_id) "
|
||||
"VALUES ('Nickelback', 'nb-sp')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
db2 = _reinit(path) # upgrade → migration runs (committed)
|
||||
rows = db2.get_blocklist(1)
|
||||
assert any(r["name"] == "Nickelback" and r["entity_type"] == "artist" for r in rows)
|
||||
|
||||
db3 = _reinit(path) # idempotent — a second upgrade doesn't duplicate
|
||||
nb = [r for r in db3.get_blocklist(1) if r["name"] == "Nickelback"]
|
||||
assert len(nb) == 1
|
||||
|
||||
# The migrated ban actually enforces at the wishlist guard.
|
||||
ok = db3.add_to_wishlist(
|
||||
spotify_track_data=_track("t5", "Photograph", "nb-sp", "Nickelback"),
|
||||
profile_id=1)
|
||||
assert ok is False
|
||||
|
||||
|
||||
# ── Phase 2a: shared guard with source fallback (download-queue path) ────────
|
||||
|
||||
def test_blocklist_reason_source_fallback(db):
|
||||
"""The download-queue guard passes the batch source explicitly because the
|
||||
analysis track dict may not carry a 'provider' field. Uses an ALBUM ban
|
||||
(id-only, no name fallback) to isolate the source-driven ID match."""
|
||||
db.add_blocklist_entry(1, "album", "Scorpion", deezer_id="scorp-dz")
|
||||
track = {"id": "t1", "name": "Nonstop",
|
||||
"artists": [{"id": "drake", "name": "Drake"}],
|
||||
"album": {"id": "scorp-dz", "name": "Scorpion"}}
|
||||
assert db.blocklist_reason_for_track(1, track) is None # no source → album id can't match
|
||||
assert db.blocklist_reason_for_track(1, track, source="spotify") is None # wrong source
|
||||
assert db.blocklist_reason_for_track(1, track, source="deezer") # right source → match
|
||||
|
||||
|
||||
def test_artist_name_fallback_works_without_source(db):
|
||||
# Artists DO fall back to name, so a ban matches even when the source is
|
||||
# unknown (covers the cross-source backfill window).
|
||||
db.add_blocklist_entry(1, "artist", "Drake", deezer_id="drake-dz")
|
||||
track = {"id": "t1", "name": "Track", "artists": [{"id": "x", "name": "Drake"}],
|
||||
"album": {"id": "al", "name": "Al"}}
|
||||
assert db.blocklist_reason_for_track(1, track) is not None
|
||||
|
||||
|
||||
def test_blocklist_reason_simulates_queue_filter(db):
|
||||
"""Mirror what master.py does: filter a missing-tracks list by the guard."""
|
||||
db.add_blocklist_entry(1, "artist", "Blocked Guy", spotify_id="bg-sp")
|
||||
missing = [
|
||||
{"track": {"id": "t1", "name": "Keep Me",
|
||||
"artists": [{"id": "ok", "name": "Good Artist"}], "album": {"id": "a1", "name": "A"}}},
|
||||
{"track": {"id": "t2", "name": "Drop Me",
|
||||
"artists": [{"id": "bg-sp", "name": "Blocked Guy"}], "album": {"id": "a2", "name": "B"}}},
|
||||
]
|
||||
kept = [r for r in missing
|
||||
if not db.blocklist_reason_for_track(1, r["track"], source="spotify")]
|
||||
assert [r["track"]["id"] for r in kept] == ["t1"]
|
||||
130
tests/blocklist/test_matching.py
Normal file
130
tests/blocklist/test_matching.py
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
"""Pure blocklist matching — the cascade + ID/name rules.
|
||||
|
||||
Block an artist/album/track by metadata-source ID; a candidate track is
|
||||
blocked if its track, album, or any artist matches (cascade). Same-source ID
|
||||
match is primary; artist NAME is a fallback (covers the backfill window);
|
||||
albums/tracks are ID-only to avoid common-title false positives.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.blocklist.matching import (
|
||||
ENTITY_ALBUM,
|
||||
ENTITY_ARTIST,
|
||||
ENTITY_TRACK,
|
||||
build_index,
|
||||
candidate_block_reason,
|
||||
)
|
||||
|
||||
|
||||
def _row(entity_type, name="", **ids):
|
||||
return {"entity_type": entity_type, "name": name, **ids}
|
||||
|
||||
|
||||
def _check(index, source="spotify", **kw):
|
||||
return candidate_block_reason(index, source=source, **kw)
|
||||
|
||||
|
||||
# ── empty / no-match ─────────────────────────────────────────────────────────
|
||||
|
||||
def test_empty_blocklist_blocks_nothing():
|
||||
idx = build_index([])
|
||||
assert idx.is_empty
|
||||
assert _check(idx, track_id="t1", album_id="al1",
|
||||
artists=[{"id": "ar1", "name": "X"}]) is None
|
||||
|
||||
|
||||
def test_unrelated_candidate_not_blocked():
|
||||
idx = build_index([_row(ENTITY_ARTIST, "Drake", spotify_id="drake-sp")])
|
||||
assert _check(idx, artists=[{"id": "other", "name": "Adele"}]) is None
|
||||
|
||||
|
||||
# ── artist level + cascade ───────────────────────────────────────────────────
|
||||
|
||||
def test_artist_blocked_by_id_blocks_their_track():
|
||||
idx = build_index([_row(ENTITY_ARTIST, "Drake", spotify_id="drake-sp")])
|
||||
reason = _check(idx, track_id="t9", album_id="al9",
|
||||
artists=[{"id": "drake-sp", "name": "Drake"}])
|
||||
assert reason == (ENTITY_ARTIST, "Drake")
|
||||
|
||||
|
||||
def test_artist_blocked_by_name_fallback():
|
||||
# No id resolved for this source yet (backfill window) — name still catches it.
|
||||
idx = build_index([_row(ENTITY_ARTIST, "Drake", deezer_id="drake-dz")])
|
||||
reason = _check(idx, source="spotify", artists=[{"id": "drake-sp", "name": "drake"}])
|
||||
assert reason == (ENTITY_ARTIST, "drake")
|
||||
|
||||
|
||||
def test_artist_name_match_is_case_insensitive():
|
||||
idx = build_index([_row(ENTITY_ARTIST, "Tyler, The Creator", spotify_id="x")])
|
||||
assert _check(idx, artists=[{"id": None, "name": "tyler, the creator"}]) is not None
|
||||
|
||||
|
||||
# ── album level ──────────────────────────────────────────────────────────────
|
||||
|
||||
def test_album_blocked_by_id_blocks_its_track():
|
||||
idx = build_index([_row(ENTITY_ALBUM, "Scorpion", spotify_id="scorp-sp")])
|
||||
reason = _check(idx, track_id="t1", album_id="scorp-sp", album_name="Scorpion",
|
||||
artists=[{"id": "drake-sp", "name": "Drake"}])
|
||||
assert reason == (ENTITY_ALBUM, "Scorpion")
|
||||
|
||||
|
||||
def test_album_name_does_not_match_without_id():
|
||||
# Common title must NOT block across artists on name alone.
|
||||
idx = build_index([_row(ENTITY_ALBUM, "Greatest Hits", spotify_id="gh-queen")])
|
||||
reason = _check(idx, album_id="gh-abba", album_name="Greatest Hits",
|
||||
artists=[{"id": "abba", "name": "ABBA"}])
|
||||
assert reason is None
|
||||
|
||||
|
||||
# ── track level ──────────────────────────────────────────────────────────────
|
||||
|
||||
def test_track_blocked_by_id():
|
||||
idx = build_index([_row(ENTITY_TRACK, "Hotline Bling", spotify_id="hb-sp")])
|
||||
reason = _check(idx, track_id="hb-sp", track_name="Hotline Bling",
|
||||
album_id="al", artists=[{"id": "drake", "name": "Drake"}])
|
||||
assert reason == (ENTITY_TRACK, "Hotline Bling")
|
||||
|
||||
|
||||
def test_track_name_alone_does_not_block():
|
||||
idx = build_index([_row(ENTITY_TRACK, "Intro", spotify_id="intro-1")])
|
||||
assert _check(idx, track_id="intro-2", track_name="Intro",
|
||||
artists=[{"id": "z", "name": "Z"}]) is None
|
||||
|
||||
|
||||
# ── source isolation (numeric id collision guard) ────────────────────────────
|
||||
|
||||
def test_same_id_different_source_does_not_collide():
|
||||
# Deezer artist 12246 is blocked; an iTunes artist that happens to be 12246
|
||||
# is a DIFFERENT entity and must NOT match (candidate source = itunes).
|
||||
idx = build_index([_row(ENTITY_ARTIST, "Some Deezer Artist", deezer_id="12246")])
|
||||
reason = _check(idx, source="itunes", artists=[{"id": "12246", "name": "Other"}])
|
||||
assert reason is None
|
||||
|
||||
|
||||
def test_same_id_same_source_matches():
|
||||
idx = build_index([_row(ENTITY_ARTIST, "A", deezer_id="12246")])
|
||||
reason = _check(idx, source="deezer", artists=[{"id": "12246", "name": "A"}])
|
||||
assert reason is not None
|
||||
|
||||
|
||||
# ── multi-source row ─────────────────────────────────────────────────────────
|
||||
|
||||
def test_row_with_multiple_source_ids_matches_each():
|
||||
idx = build_index([_row(ENTITY_ARTIST, "Drake",
|
||||
spotify_id="sp", itunes_id="it", deezer_id="dz")])
|
||||
assert _check(idx, source="spotify", artists=[{"id": "sp", "name": "Drake"}])
|
||||
assert _check(idx, source="itunes", artists=[{"id": "it", "name": "Drake"}])
|
||||
assert _check(idx, source="deezer", artists=[{"id": "dz", "name": "Drake"}])
|
||||
|
||||
|
||||
# ── cascade precedence ───────────────────────────────────────────────────────
|
||||
|
||||
def test_track_hit_reported_before_artist():
|
||||
idx = build_index([
|
||||
_row(ENTITY_ARTIST, "Drake", spotify_id="drake"),
|
||||
_row(ENTITY_TRACK, "God's Plan", spotify_id="gp"),
|
||||
])
|
||||
reason = _check(idx, track_id="gp", track_name="God's Plan",
|
||||
artists=[{"id": "drake", "name": "Drake"}])
|
||||
assert reason[0] == ENTITY_TRACK # most specific hit wins the reason
|
||||
|
|
@ -364,6 +364,49 @@ def test_playlist_image_uploaded_to_jellyfin(patched_db):
|
|||
assert jf.image_calls == [('PJF', 'https://img/y.png')]
|
||||
|
||||
|
||||
def test_append_mode_preserves_playlist_image(patched_db):
|
||||
"""Append edits in place — it must NOT re-push the source image over the
|
||||
user's custom poster (#811)."""
|
||||
jf = _FakeJellyfin()
|
||||
cfg = _FakeConfig(server='jellyfin')
|
||||
result = _FakeSyncResult(synced_tracks=3)
|
||||
svc = _FakeSyncService(media_client=_FakeMediaClient(), sync_result=result)
|
||||
deps = _build_deps(sync_service=svc, jellyfin=jf, config=cfg)
|
||||
|
||||
ds.run_sync_task('pA', 'PA', [_track()],
|
||||
playlist_image_url='https://img/a.png', deps=deps, sync_mode='append')
|
||||
|
||||
assert jf.image_calls == [] # preserved, not clobbered
|
||||
|
||||
|
||||
def test_reconcile_mode_preserves_playlist_image(patched_db):
|
||||
"""Reconcile likewise preserves the image (#792)."""
|
||||
jf = _FakeJellyfin()
|
||||
cfg = _FakeConfig(server='jellyfin')
|
||||
result = _FakeSyncResult(synced_tracks=3)
|
||||
svc = _FakeSyncService(media_client=_FakeMediaClient(), sync_result=result)
|
||||
deps = _build_deps(sync_service=svc, jellyfin=jf, config=cfg)
|
||||
|
||||
ds.run_sync_task('pR', 'PR', [_track()],
|
||||
playlist_image_url='https://img/r.png', deps=deps, sync_mode='reconcile')
|
||||
|
||||
assert jf.image_calls == []
|
||||
|
||||
|
||||
def test_replace_mode_still_pushes_playlist_image(patched_db):
|
||||
"""Replace recreates from scratch, so it does push the source image."""
|
||||
jf = _FakeJellyfin()
|
||||
cfg = _FakeConfig(server='jellyfin')
|
||||
result = _FakeSyncResult(synced_tracks=3)
|
||||
svc = _FakeSyncService(media_client=_FakeMediaClient(), sync_result=result)
|
||||
deps = _build_deps(sync_service=svc, jellyfin=jf, config=cfg)
|
||||
|
||||
ds.run_sync_task('pRep', 'PRep', [_track()],
|
||||
playlist_image_url='https://img/rep.png', deps=deps, sync_mode='replace')
|
||||
|
||||
assert jf.image_calls == [('PRep', 'https://img/rep.png')]
|
||||
|
||||
|
||||
def test_no_image_upload_when_zero_synced(patched_db):
|
||||
"""synced_tracks == 0 → no playlist image upload."""
|
||||
plex = _FakePlex()
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ class _FakeClient:
|
|||
self._results = results if results is not None else []
|
||||
self.mode = mode
|
||||
self.search_calls = []
|
||||
self.exclude_calls = [] # exclude_sources arg per search() call
|
||||
self._client_map = {}
|
||||
for k, v in (subclients or {}).items():
|
||||
if k in self._CLIENT_NAMES:
|
||||
|
|
@ -58,6 +59,7 @@ class _FakeClient:
|
|||
|
||||
async def search(self, query, timeout=30, exclude_sources=None):
|
||||
self.search_calls.append((query, timeout))
|
||||
self.exclude_calls.append(exclude_sources)
|
||||
return (self._results, None)
|
||||
|
||||
|
||||
|
|
@ -506,6 +508,190 @@ def test_hybrid_fallback_skipped_when_mode_not_hybrid():
|
|||
assert yt.search_calls == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cached-first quarantine retry: walk already-found candidates before any
|
||||
# re-search; never re-search a source whose candidates are spent (only switch
|
||||
# to a not-yet-searched source). See task_worker cached-first phase.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class _Cand:
|
||||
def __init__(self, username, filename):
|
||||
self.username = username
|
||||
self.filename = filename
|
||||
|
||||
|
||||
def test_quarantine_retry_tries_cached_candidates_without_searching():
|
||||
_seed_task(
|
||||
_quarantine_retry=True,
|
||||
cached_candidates=[_Cand('peerA', 'f1.flac'), _Cand('peerB', 'f2.flac')],
|
||||
used_sources={'peerA_f1.flac'}, # peerA already tried
|
||||
)
|
||||
attempted = []
|
||||
|
||||
def _attempt(task_id, candidates, track, batch_id):
|
||||
attempted.append([getattr(c, 'filename', None) for c in candidates])
|
||||
return True
|
||||
|
||||
sk = _FakeClient(results=['should-not-be-used'])
|
||||
deps, _ = _build_deps(
|
||||
soulseek=sk,
|
||||
matching=_FakeMatchEngine(queries=['q1']),
|
||||
attempt_download_with_candidates=_attempt,
|
||||
)
|
||||
tw.download_track_worker('t1', 'b1', deps)
|
||||
# No search performed — cached candidates used directly.
|
||||
assert sk.search_calls == []
|
||||
# Only the unused candidate (peerB) was passed to attempt.
|
||||
assert attempted == [['f2.flac']]
|
||||
|
||||
|
||||
def test_quarantine_retry_skips_cached_from_exhausted_source():
|
||||
# hifi is budget-exhausted; its cached candidate must be skipped, soulseek's tried.
|
||||
_seed_task(
|
||||
_quarantine_retry=True,
|
||||
cached_candidates=[_Cand('hifi', 'h.flac'), _Cand('peerB', 'f2.flac')],
|
||||
used_sources=set(),
|
||||
exhausted_download_sources={'hifi'},
|
||||
)
|
||||
attempted = []
|
||||
|
||||
def _attempt(task_id, candidates, track, batch_id):
|
||||
attempted.append([getattr(c, 'username', None) for c in candidates])
|
||||
return True
|
||||
|
||||
sk = _FakeClient(results=['x'])
|
||||
deps, _ = _build_deps(
|
||||
soulseek=sk,
|
||||
matching=_FakeMatchEngine(queries=['q1']),
|
||||
attempt_download_with_candidates=_attempt,
|
||||
)
|
||||
tw.download_track_worker('t1', 'b1', deps)
|
||||
assert sk.search_calls == []
|
||||
assert attempted == [['peerB']] # hifi candidate excluded
|
||||
|
||||
|
||||
# A track_info that yields exactly the engine queries (no artist → no
|
||||
# first-word legacy query; name equals a query → track-only query dedupes away),
|
||||
# so the generated query set is deterministic for cached-first assertions.
|
||||
_SOLO_TRACK = {'id': 'sp-1', 'name': 'Solo', 'artists': [],
|
||||
'album': 'A', 'duration_ms': 1000}
|
||||
|
||||
|
||||
def test_quarantine_retry_searches_unsearched_query_without_excluding_source():
|
||||
# Cache spent + 'Solo' already searched, but 'q2' is NOT yet searched. The
|
||||
# retry must search q2 against the SAME source (no source-level exclusion) so
|
||||
# every query is exhausted per source before the chain switches sources
|
||||
# (lazy multi-query retry).
|
||||
_seed_task(
|
||||
track_info=dict(_SOLO_TRACK),
|
||||
_quarantine_retry=True,
|
||||
cached_candidates=[_Cand('peerA', 'f1.flac')],
|
||||
used_sources={'peerA_f1.flac'},
|
||||
searched_queries={'Solo'},
|
||||
)
|
||||
sk = _FakeClient(results=[], mode='hybrid',
|
||||
subclients={'hybrid_order': ['soulseek', 'hifi']})
|
||||
deps, _ = _build_deps(
|
||||
soulseek=sk,
|
||||
matching=_FakeMatchEngine(queries=['Solo', 'q2']),
|
||||
)
|
||||
tw.download_track_worker('t1', 'b1', deps)
|
||||
# Only q2 was searched — 'Solo' skipped because its candidates were cached.
|
||||
assert [c[0] for c in sk.search_calls] == ['q2']
|
||||
# Soulseek NOT excluded: the not-yet-searched query still hits it.
|
||||
assert all((not ex) or 'soulseek' not in ex for ex in sk.exclude_calls)
|
||||
|
||||
|
||||
def test_quarantine_retry_skips_already_searched_query_no_research():
|
||||
# The only generated query is already searched and cache spent → it is NOT
|
||||
# re-searched (its candidates live in cache). This is the wasteful repeat the
|
||||
# cached-first design removes.
|
||||
_seed_task(
|
||||
track_info=dict(_SOLO_TRACK),
|
||||
_quarantine_retry=True,
|
||||
cached_candidates=[_Cand('peerA', 'f1.flac')],
|
||||
used_sources={'peerA_f1.flac'},
|
||||
searched_queries={'Solo'},
|
||||
)
|
||||
sk = _FakeClient(results=[], mode='hybrid',
|
||||
subclients={'hybrid_order': ['soulseek', 'hifi']})
|
||||
deps, _ = _build_deps(
|
||||
soulseek=sk,
|
||||
matching=_FakeMatchEngine(queries=['Solo']),
|
||||
)
|
||||
tw.download_track_worker('t1', 'b1', deps)
|
||||
assert sk.search_calls == [] # 'Solo' not re-searched
|
||||
|
||||
|
||||
def test_quarantine_retry_still_excludes_budget_exhausted_source():
|
||||
# A source whose per-source budget is spent (exhaustive mode) stays excluded
|
||||
# so the chain falls through to the next source.
|
||||
_seed_task(
|
||||
_quarantine_retry=True,
|
||||
cached_candidates=[],
|
||||
searched_queries=set(),
|
||||
exhausted_download_sources={'soulseek'},
|
||||
)
|
||||
sk = _FakeClient(results=[], mode='hybrid',
|
||||
subclients={'hybrid_order': ['soulseek', 'hifi']})
|
||||
deps, _ = _build_deps(
|
||||
soulseek=sk,
|
||||
matching=_FakeMatchEngine(queries=['q1']),
|
||||
)
|
||||
tw.download_track_worker('t1', 'b1', deps)
|
||||
assert sk.search_calls
|
||||
assert all(ex and 'soulseek' in ex for ex in sk.exclude_calls)
|
||||
|
||||
|
||||
def test_search_records_searched_queries():
|
||||
# Every query the worker actually runs is recorded so a later quarantine
|
||||
# retry can skip re-searching it.
|
||||
_seed_task(status='pending')
|
||||
sk = _FakeClient(results=['r1'])
|
||||
deps, _ = _build_deps(
|
||||
soulseek=sk,
|
||||
matching=_FakeMatchEngine(queries=['q1']),
|
||||
get_valid_candidates=lambda r, t, q: [], # no candidates → loop completes
|
||||
)
|
||||
tw.download_track_worker('t1', 'b1', deps)
|
||||
assert 'q1' in download_tasks['t1'].get('searched_queries', set())
|
||||
|
||||
|
||||
def test_non_quarantine_run_resets_searched_queries():
|
||||
# A fresh / dead-connection retry starts a new search generation: the stale
|
||||
# searched-query memory is cleared so every query can be searched again.
|
||||
_seed_task(
|
||||
track_info=dict(_SOLO_TRACK),
|
||||
cached_candidates=[],
|
||||
searched_queries={'stale-q'},
|
||||
)
|
||||
sk = _FakeClient(results=[])
|
||||
deps, _ = _build_deps(
|
||||
soulseek=sk,
|
||||
matching=_FakeMatchEngine(queries=['Solo']),
|
||||
)
|
||||
tw.download_track_worker('t1', 'b1', deps)
|
||||
sq = download_tasks['t1'].get('searched_queries', set())
|
||||
assert 'stale-q' not in sq # stale generation cleared
|
||||
assert 'Solo' in sq # current generation recorded fresh
|
||||
|
||||
|
||||
def test_non_quarantine_run_ignores_cached_first_and_searches():
|
||||
# A fresh (non-quarantine) run must NOT use cached-first — it searches.
|
||||
_seed_task(
|
||||
cached_candidates=[_Cand('peerA', 'f1.flac')],
|
||||
used_sources=set(),
|
||||
)
|
||||
sk = _FakeClient(results=[])
|
||||
deps, _ = _build_deps(
|
||||
soulseek=sk,
|
||||
matching=_FakeMatchEngine(queries=['q1']),
|
||||
attempt_download_with_candidates=lambda *a, **kw: False,
|
||||
)
|
||||
tw.download_track_worker('t1', 'b1', deps)
|
||||
assert sk.search_calls # searched, did not short-circuit on cache
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Top-level exception path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
27
tests/imports/test_expected_duration_for_check.py
Normal file
27
tests/imports/test_expected_duration_for_check.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
"""Duration-agreement leg is skipped for local/manual imports (#804).
|
||||
|
||||
The duration check catches truncated/wrong slskd downloads. A manual import is
|
||||
the user's own file being sorted — duration-agreeing it against a re-resolved
|
||||
release false-quarantines (Coldplay 'Yellow' album file vs a single's length).
|
||||
"""
|
||||
|
||||
from core.imports.file_integrity import expected_duration_for_check
|
||||
|
||||
|
||||
def test_local_import_skips_duration_leg():
|
||||
# Even with a valid expected duration, a local import returns None (skip).
|
||||
assert expected_duration_for_check(266000, is_local_import=True) is None
|
||||
|
||||
|
||||
def test_download_keeps_expected_duration():
|
||||
assert expected_duration_for_check(266000, is_local_import=False) == 266000
|
||||
|
||||
|
||||
def test_zero_or_missing_expected_is_none():
|
||||
assert expected_duration_for_check(0, is_local_import=False) is None
|
||||
assert expected_duration_for_check(None, is_local_import=False) is None
|
||||
assert expected_duration_for_check("nan", is_local_import=False) is None
|
||||
|
||||
|
||||
def test_string_numeric_expected_coerced():
|
||||
assert expected_duration_for_check("266000", is_local_import=False) == 266000
|
||||
|
|
@ -304,3 +304,368 @@ def test_verification_wrapper_applies_quarantine_entry_id_on_integrity_failure(m
|
|||
runtime_state.download_tasks.update(original)
|
||||
runtime_state.matched_downloads_context.clear()
|
||||
runtime_state.matched_downloads_context.update(original_ctx)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Next-best-candidate retry on AcoustID / integrity quarantine. When a
|
||||
# verification or integrity check quarantines the wrong/broken file, the wrapper
|
||||
# asks the monitor to re-run the worker on the next candidate (skipping the bad
|
||||
# source) instead of failing the task outright.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _wire_retry_engine(monkeypatch):
|
||||
"""Wire monitor's retry globals to capture the worker re-submission."""
|
||||
import core.downloads.monitor as monitor
|
||||
|
||||
submitted = []
|
||||
|
||||
class _Exec:
|
||||
def submit(self, fn, *args):
|
||||
submitted.append(args)
|
||||
|
||||
monkeypatch.setattr(monitor, "missing_download_executor", _Exec())
|
||||
monkeypatch.setattr(monitor, "_download_track_worker", lambda task_id, batch_id: None)
|
||||
monkeypatch.setattr(monitor, "MAX_QUARANTINE_RETRIES", 5)
|
||||
|
||||
# Pin the retry toggle ON instead of reading the runner's ambient config —
|
||||
# CI's fresh default config vs a dev's lived-in config.json must not
|
||||
# decide whether these tests pass (they did: 7 failures, CI-only).
|
||||
real_get = monitor.config_manager.get
|
||||
|
||||
def _pinned_get(key, default=None):
|
||||
if key == "post_processing.retry_next_candidate_on_mismatch":
|
||||
return True
|
||||
return real_get(key, default)
|
||||
|
||||
monkeypatch.setattr(monitor.config_manager, "get", _pinned_get)
|
||||
return submitted
|
||||
|
||||
|
||||
def _patch_config(monkeypatch, overrides):
|
||||
"""Override specific config keys for the monitor's config_manager reads."""
|
||||
import core.downloads.monitor as monitor
|
||||
|
||||
real_get = monitor.config_manager.get
|
||||
|
||||
def fake_get(key, default=None):
|
||||
if key in overrides:
|
||||
return overrides[key]
|
||||
return real_get(key, default)
|
||||
|
||||
monkeypatch.setattr(monitor.config_manager, "get", fake_get)
|
||||
|
||||
|
||||
def _run_wrapper_with_quarantine(monkeypatch, flag_setter, task_extra=None):
|
||||
task_id, batch_id, context_key = "rtask", "rbatch", "rctx"
|
||||
context = {"track_info": {}, "task_id": task_id, "batch_id": batch_id}
|
||||
|
||||
monkeypatch.setattr(import_pipeline, "post_process_matched_download", flag_setter)
|
||||
|
||||
original = dict(runtime_state.download_tasks)
|
||||
original_ctx = dict(runtime_state.matched_downloads_context)
|
||||
try:
|
||||
runtime_state.download_tasks.clear()
|
||||
task = {
|
||||
"track_info": {}, "status": "downloading",
|
||||
"username": "hifi", "filename": "123||A - B", "used_sources": set(),
|
||||
}
|
||||
if task_extra:
|
||||
task.update(task_extra)
|
||||
runtime_state.download_tasks[task_id] = task
|
||||
runtime_state.matched_downloads_context.clear()
|
||||
runtime_state.matched_downloads_context[context_key] = context
|
||||
|
||||
completion = []
|
||||
runtime = types.SimpleNamespace(
|
||||
automation_engine=None,
|
||||
on_download_completed=lambda b, t, success: completion.append((b, t, success)),
|
||||
web_scan_manager=None,
|
||||
repair_worker=None,
|
||||
)
|
||||
import_pipeline.post_process_matched_download_with_verification(
|
||||
context_key, context, "/tmp/source.flac", task_id, batch_id, runtime,
|
||||
)
|
||||
return dict(runtime_state.download_tasks[task_id]), completion, context_key
|
||||
finally:
|
||||
runtime_state.download_tasks.clear()
|
||||
runtime_state.download_tasks.update(original)
|
||||
runtime_state.matched_downloads_context.clear()
|
||||
runtime_state.matched_downloads_context.update(original_ctx)
|
||||
|
||||
|
||||
def test_acoustid_mismatch_requeues_next_candidate(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
|
||||
def _fake_inner(ck, ctx, fp, runtime, metadata_runtime=None):
|
||||
ctx["_acoustid_quarantined"] = True
|
||||
ctx["_acoustid_failure_msg"] = "wrong song"
|
||||
|
||||
task, completion, context_key = _run_wrapper_with_quarantine(monkeypatch, _fake_inner)
|
||||
|
||||
# Task goes back to searching for the next candidate — NOT failed.
|
||||
assert task["status"] == "searching"
|
||||
assert task["quarantine_retry_count"] == 1
|
||||
# The quarantined source is flagged so the re-run won't re-pick it.
|
||||
assert "hifi_123||A - B" in task["used_sources"]
|
||||
# Stale download identity cleared; worker re-submitted; no batch failure.
|
||||
assert "download_id" not in task and "username" not in task
|
||||
assert submitted == [("rtask", "rbatch")]
|
||||
assert completion == []
|
||||
# Old context cleaned up (the re-run builds a fresh one for the new pick).
|
||||
assert context_key not in runtime_state.matched_downloads_context
|
||||
|
||||
|
||||
def test_requeue_flags_quarantine_retry_for_cached_first(monkeypatch):
|
||||
_wire_retry_engine(monkeypatch)
|
||||
|
||||
def _fake_inner(ck, ctx, fp, runtime, metadata_runtime=None):
|
||||
ctx["_acoustid_quarantined"] = True
|
||||
ctx["_acoustid_failure_msg"] = "wrong song"
|
||||
|
||||
task, _, _ = _run_wrapper_with_quarantine(monkeypatch, _fake_inner)
|
||||
|
||||
# The re-run is flagged so the worker walks cached candidates before
|
||||
# re-searching (cached-first), rather than re-running the full search.
|
||||
assert task["_quarantine_retry"] is True
|
||||
|
||||
|
||||
def test_integrity_mismatch_requeues_next_candidate(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
|
||||
def _fake_inner(ck, ctx, fp, runtime, metadata_runtime=None):
|
||||
ctx["_integrity_failure_msg"] = "Duration mismatch: file is 231.0s, expected 271.0s"
|
||||
|
||||
task, completion, _ = _run_wrapper_with_quarantine(monkeypatch, _fake_inner)
|
||||
|
||||
assert task["status"] == "searching"
|
||||
assert task["quarantine_retry_count"] == 1
|
||||
assert submitted == [("rtask", "rbatch")]
|
||||
assert completion == []
|
||||
|
||||
|
||||
def test_manual_pick_does_not_requeue_on_mismatch(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
|
||||
def _fake_inner(ck, ctx, fp, runtime, metadata_runtime=None):
|
||||
ctx["_integrity_failure_msg"] = "Duration mismatch"
|
||||
|
||||
task, completion, _ = _run_wrapper_with_quarantine(
|
||||
monkeypatch, _fake_inner, task_extra={"_user_manual_pick": True},
|
||||
)
|
||||
|
||||
# User explicitly chose this file — fail it, don't silently swap.
|
||||
assert task["status"] == "failed"
|
||||
assert submitted == []
|
||||
assert completion == [("rbatch", "rtask", False)]
|
||||
|
||||
|
||||
def test_retry_budget_exhausted_fails_task(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
import core.downloads.monitor as monitor
|
||||
monkeypatch.setattr(monitor, "MAX_QUARANTINE_RETRIES", 2)
|
||||
|
||||
def _fake_inner(ck, ctx, fp, runtime, metadata_runtime=None):
|
||||
ctx["_acoustid_quarantined"] = True
|
||||
ctx["_acoustid_failure_msg"] = "wrong song"
|
||||
|
||||
task, completion, _ = _run_wrapper_with_quarantine(
|
||||
monkeypatch, _fake_inner, task_extra={"quarantine_retry_count": 2},
|
||||
)
|
||||
|
||||
# Cap reached — fall through to normal failure handling.
|
||||
assert task["status"] == "failed"
|
||||
assert submitted == []
|
||||
assert completion == [("rbatch", "rtask", False)]
|
||||
|
||||
|
||||
def _acoustid_quarantine(ck, ctx, fp, runtime, metadata_runtime=None):
|
||||
ctx["_acoustid_quarantined"] = True
|
||||
ctx["_acoustid_failure_msg"] = "wrong song"
|
||||
|
||||
|
||||
def test_exhaustive_mode_uses_per_source_budget(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
_patch_config(monkeypatch, {
|
||||
"post_processing.retry_exhaustive": True,
|
||||
"post_processing.retries_per_query": 5,
|
||||
})
|
||||
|
||||
# query_count=2 → budget for source 'hifi' = 2 * 5 = 10; first failure retries.
|
||||
task, completion, _ = _run_wrapper_with_quarantine(
|
||||
monkeypatch, _acoustid_quarantine, task_extra={"query_count": 2},
|
||||
)
|
||||
|
||||
assert task["status"] == "searching"
|
||||
# Per-source budget tracked separately from the legacy global counter.
|
||||
assert task["quarantine_retry_counts_by_source"] == {"hifi": 1}
|
||||
assert task["quarantine_retry_count"] == 1
|
||||
assert "hifi_123||A - B" in task["used_sources"]
|
||||
assert submitted == [("rtask", "rbatch")]
|
||||
assert completion == []
|
||||
|
||||
|
||||
def test_exhaustive_source_budget_exhausted_fails(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
_patch_config(monkeypatch, {
|
||||
"post_processing.retry_exhaustive": True,
|
||||
"post_processing.retries_per_query": 5,
|
||||
})
|
||||
|
||||
# hifi already at its full budget (query_count 2 * 5 = 10) → fail, no retry.
|
||||
task, completion, _ = _run_wrapper_with_quarantine(
|
||||
monkeypatch, _acoustid_quarantine,
|
||||
task_extra={"query_count": 2, "quarantine_retry_counts_by_source": {"hifi": 10}},
|
||||
)
|
||||
|
||||
assert task["status"] == "failed"
|
||||
assert submitted == []
|
||||
assert completion == [("rbatch", "rtask", False)]
|
||||
|
||||
|
||||
def test_exhaustive_budget_is_separate_per_source(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
_patch_config(monkeypatch, {
|
||||
"post_processing.retry_exhaustive": True,
|
||||
"post_processing.retries_per_query": 5,
|
||||
})
|
||||
|
||||
# soulseek is already maxed, but the failing download is on hifi — hifi has
|
||||
# its own fresh budget, so the task still retries.
|
||||
task, completion, _ = _run_wrapper_with_quarantine(
|
||||
monkeypatch, _acoustid_quarantine,
|
||||
task_extra={"query_count": 1, "quarantine_retry_counts_by_source": {"soulseek": 5}},
|
||||
)
|
||||
|
||||
assert task["status"] == "searching"
|
||||
assert task["quarantine_retry_counts_by_source"] == {"soulseek": 5, "hifi": 1}
|
||||
assert submitted == [("rtask", "rbatch")]
|
||||
|
||||
|
||||
def test_exhaustive_soulseek_peer_resolves_to_soulseek(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
_patch_config(monkeypatch, {
|
||||
"post_processing.retry_exhaustive": True,
|
||||
"post_processing.retries_per_query": 5,
|
||||
})
|
||||
|
||||
# A Soulseek peer name (not a streaming source) is bucketed under 'soulseek'.
|
||||
task, completion, _ = _run_wrapper_with_quarantine(
|
||||
monkeypatch, _acoustid_quarantine,
|
||||
task_extra={"username": "DjPeer", "filename": "f.flac", "query_count": 1},
|
||||
)
|
||||
|
||||
assert task["status"] == "searching"
|
||||
assert task["quarantine_retry_counts_by_source"] == {"soulseek": 1}
|
||||
|
||||
|
||||
def test_exhaustive_budget_defaults_query_count_to_one(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
_patch_config(monkeypatch, {
|
||||
"post_processing.retry_exhaustive": True,
|
||||
"post_processing.retries_per_query": 1,
|
||||
})
|
||||
|
||||
# No query_count on the task → budget defaults to 1 * 1 = 1; hifi already at 1.
|
||||
task, completion, _ = _run_wrapper_with_quarantine(
|
||||
monkeypatch, _acoustid_quarantine,
|
||||
task_extra={"quarantine_retry_counts_by_source": {"hifi": 1}},
|
||||
)
|
||||
|
||||
assert task["status"] == "failed"
|
||||
assert submitted == []
|
||||
|
||||
|
||||
def test_exhaustive_absolute_ceiling_guards_runaway(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
import core.downloads.monitor as monitor
|
||||
monkeypatch.setattr(monitor, "MAX_TOTAL_QUARANTINE_RETRIES", 3)
|
||||
_patch_config(monkeypatch, {
|
||||
"post_processing.retry_exhaustive": True,
|
||||
"post_processing.retries_per_query": 1000, # per-source budget effectively unbounded
|
||||
})
|
||||
|
||||
# Per-source budget is huge, but the absolute total ceiling (3) still fires.
|
||||
task, completion, _ = _run_wrapper_with_quarantine(
|
||||
monkeypatch, _acoustid_quarantine,
|
||||
task_extra={"query_count": 1, "quarantine_retry_count": 3,
|
||||
"quarantine_retry_counts_by_source": {"hifi": 0}},
|
||||
)
|
||||
|
||||
assert task["status"] == "failed"
|
||||
assert submitted == []
|
||||
|
||||
|
||||
def _wire_orchestrator(monkeypatch, mode, hybrid_order):
|
||||
"""Wire monitor's download_orchestrator so per-source budget exhaustion can
|
||||
decide whether another source remains to fall back to."""
|
||||
import core.downloads.monitor as monitor
|
||||
orch = types.SimpleNamespace(mode=mode, hybrid_order=list(hybrid_order))
|
||||
monkeypatch.setattr(monitor, "download_orchestrator", orch)
|
||||
return orch
|
||||
|
||||
|
||||
def test_exhaustive_exhausted_source_switches_in_hybrid(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
_wire_orchestrator(monkeypatch, "hybrid", ["soulseek", "hifi"])
|
||||
_patch_config(monkeypatch, {
|
||||
"post_processing.retry_exhaustive": True,
|
||||
"post_processing.retries_per_query": 5,
|
||||
})
|
||||
|
||||
# soulseek's budget (query_count 2 * 5 = 10) is spent. In hybrid mode the
|
||||
# task switches to the next source instead of failing the whole track.
|
||||
task, completion, _ = _run_wrapper_with_quarantine(
|
||||
monkeypatch, _acoustid_quarantine,
|
||||
task_extra={"username": "DjPeer", "query_count": 2,
|
||||
"quarantine_retry_counts_by_source": {"soulseek": 10}},
|
||||
)
|
||||
|
||||
assert task["status"] == "searching"
|
||||
# The spent source is flagged so the worker excludes it from the next search.
|
||||
assert task["exhausted_download_sources"] == {"soulseek"}
|
||||
# Its per-source counter is NOT pushed past budget — the source is simply done.
|
||||
assert task["quarantine_retry_counts_by_source"]["soulseek"] == 10
|
||||
assert submitted == [("rtask", "rbatch")]
|
||||
assert completion == []
|
||||
|
||||
|
||||
def test_exhaustive_all_sources_exhausted_fails_in_hybrid(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
_wire_orchestrator(monkeypatch, "hybrid", ["soulseek", "hifi"])
|
||||
_patch_config(monkeypatch, {
|
||||
"post_processing.retry_exhaustive": True,
|
||||
"post_processing.retries_per_query": 5,
|
||||
})
|
||||
|
||||
# soulseek was exhausted on an earlier attempt; now hifi spends its last
|
||||
# budget too — no fallback source remains, so the task finally fails.
|
||||
task, completion, _ = _run_wrapper_with_quarantine(
|
||||
monkeypatch, _acoustid_quarantine,
|
||||
task_extra={"username": "hifi", "query_count": 2,
|
||||
"exhausted_download_sources": {"soulseek"},
|
||||
"quarantine_retry_counts_by_source": {"hifi": 10}},
|
||||
)
|
||||
|
||||
assert task["status"] == "failed"
|
||||
assert submitted == []
|
||||
assert completion == [("rbatch", "rtask", False)]
|
||||
|
||||
|
||||
def test_exhaustive_single_source_exhausted_fails(monkeypatch):
|
||||
submitted = _wire_retry_engine(monkeypatch)
|
||||
# Single-source mode: nothing to fall back to once the budget is spent.
|
||||
_wire_orchestrator(monkeypatch, "soulseek", [])
|
||||
_patch_config(monkeypatch, {
|
||||
"post_processing.retry_exhaustive": True,
|
||||
"post_processing.retries_per_query": 5,
|
||||
})
|
||||
|
||||
task, completion, _ = _run_wrapper_with_quarantine(
|
||||
monkeypatch, _acoustid_quarantine,
|
||||
task_extra={"username": "DjPeer", "query_count": 2,
|
||||
"quarantine_retry_counts_by_source": {"soulseek": 10}},
|
||||
)
|
||||
|
||||
assert task["status"] == "failed"
|
||||
assert submitted == []
|
||||
assert completion == [("rbatch", "rtask", False)]
|
||||
|
|
|
|||
|
|
@ -504,7 +504,8 @@ def test_process_single_import_file_resolves_and_posts_context(tmp_path):
|
|||
assert outcome == ("ok", "Song")
|
||||
assert len(post_calls) == 1
|
||||
assert post_calls[0][0].startswith("import_single_")
|
||||
assert post_calls[0][1] == {"track": {"name": "Song"}, "artist": {"name": "Artist"}}
|
||||
assert post_calls[0][1] == {"track": {"name": "Song"}, "artist": {"name": "Artist"},
|
||||
"is_local_import": True}
|
||||
assert post_calls[0][2] == str(audio_file)
|
||||
|
||||
|
||||
|
|
|
|||
207
tests/imports/test_version_mismatch_fallback.py
Normal file
207
tests/imports/test_version_mismatch_fallback.py
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
"""Last-resort acceptance of a version-mismatched quarantine candidate.
|
||||
|
||||
When a track's retries are fully exhausted and EVERY quarantined candidate for
|
||||
it failed the same way (same wrong version, e.g. all instrumental), the only
|
||||
available version is that one — accept the best (first-tried) one instead of
|
||||
leaving the track missing. Strict guards: version-mismatch only, all the same
|
||||
matched version, and a minimum count.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.imports.version_mismatch_fallback import (
|
||||
select_version_mismatch_fallback,
|
||||
try_accept_version_mismatch_fallback,
|
||||
)
|
||||
|
||||
|
||||
def _entry(eid, reason, *, track="Barricades (Movie Ver.)", artist="Hiroyuki Sawano",
|
||||
ctx=True):
|
||||
return {
|
||||
"id": eid,
|
||||
"reason": reason,
|
||||
"expected_track": track,
|
||||
"expected_artist": artist,
|
||||
"has_full_context": ctx,
|
||||
}
|
||||
|
||||
|
||||
_VM = "Version mismatch: expected 'Barricades (Movie Ver.)' (original) but file is 'Barricades <MOVIEver.> ({v})' ({v})"
|
||||
|
||||
|
||||
def _vm(version):
|
||||
return _VM.format(v=version)
|
||||
|
||||
|
||||
def test_picks_oldest_when_all_same_version_and_count_met():
|
||||
# 3 instrumental mismatches → all same kind → pick first-tried (smallest id).
|
||||
entries = [
|
||||
_entry("20260605_120300", _vm("instrumental")),
|
||||
_entry("20260605_120100", _vm("instrumental")), # oldest = first tried
|
||||
_entry("20260605_120200", _vm("instrumental")),
|
||||
]
|
||||
chosen = select_version_mismatch_fallback(entries, "Barricades (Movie Ver.)",
|
||||
"Hiroyuki Sawano", min_count=2)
|
||||
assert chosen is not None
|
||||
assert chosen["id"] == "20260605_120100"
|
||||
|
||||
|
||||
def test_none_when_below_min_count():
|
||||
entries = [_entry("20260605_120100", _vm("instrumental"))]
|
||||
assert select_version_mismatch_fallback(
|
||||
entries, "Barricades (Movie Ver.)", "Hiroyuki Sawano", min_count=2) is None
|
||||
|
||||
|
||||
def test_none_when_mixed_versions():
|
||||
# instrumental + live → inconsistent → never auto-accept (ambiguous).
|
||||
entries = [
|
||||
_entry("20260605_120100", _vm("instrumental")),
|
||||
_entry("20260605_120200", _vm("live")),
|
||||
]
|
||||
assert select_version_mismatch_fallback(
|
||||
entries, "Barricades (Movie Ver.)", "Hiroyuki Sawano", min_count=2) is None
|
||||
|
||||
|
||||
def test_ignores_non_version_mismatch_reasons():
|
||||
# Audio mismatch (wrong artist/song) and integrity must NOT count.
|
||||
entries = [
|
||||
_entry("20260605_120100",
|
||||
"Audio mismatch: file identified as 'X' by 'Y' (artist=0%)"),
|
||||
_entry("20260605_120200",
|
||||
"Integrity check failed: Duration mismatch: file is 175s, expected 182s"),
|
||||
]
|
||||
assert select_version_mismatch_fallback(
|
||||
entries, "Barricades (Movie Ver.)", "Hiroyuki Sawano", min_count=1) is None
|
||||
|
||||
|
||||
def test_only_counts_entries_for_this_track():
|
||||
entries = [
|
||||
_entry("20260605_120100", _vm("instrumental")),
|
||||
_entry("20260605_120200", _vm("instrumental"), track="Call Your Name (Gv)"),
|
||||
]
|
||||
# Only one entry matches this track → below min_count of 2.
|
||||
assert select_version_mismatch_fallback(
|
||||
entries, "Barricades (Movie Ver.)", "Hiroyuki Sawano", min_count=2) is None
|
||||
|
||||
|
||||
def test_excludes_thin_sidecar_entries_without_context():
|
||||
# Can't approve without embedded context → exclude from the candidate pool.
|
||||
entries = [
|
||||
_entry("20260605_120100", _vm("instrumental"), ctx=False),
|
||||
_entry("20260605_120200", _vm("instrumental"), ctx=False),
|
||||
]
|
||||
assert select_version_mismatch_fallback(
|
||||
entries, "Barricades (Movie Ver.)", "Hiroyuki Sawano", min_count=2) is None
|
||||
|
||||
|
||||
def test_track_match_is_case_and_space_insensitive():
|
||||
entries = [
|
||||
_entry("20260605_120100", _vm("instrumental")),
|
||||
_entry("20260605_120200", _vm("instrumental")),
|
||||
]
|
||||
chosen = select_version_mismatch_fallback(
|
||||
entries, " barricades (movie ver.) ", "HIROYUKI SAWANO", min_count=2)
|
||||
assert chosen is not None
|
||||
assert chosen["id"] == "20260605_120100"
|
||||
|
||||
|
||||
# ── Orchestration (try_accept_version_mismatch_fallback) ──────────────────────
|
||||
|
||||
def _cfg(enabled=True, min_count=2):
|
||||
values = {
|
||||
"post_processing.accept_version_mismatch_fallback": enabled,
|
||||
"post_processing.version_mismatch_min_count": min_count,
|
||||
}
|
||||
return lambda key, default=None: values.get(key, default)
|
||||
|
||||
|
||||
def _two_instrumental_entries():
|
||||
return [
|
||||
_entry("20260605_120100", _vm("instrumental")),
|
||||
_entry("20260605_120200", _vm("instrumental")),
|
||||
]
|
||||
|
||||
|
||||
def test_orchestration_disabled_does_nothing():
|
||||
calls = {"approve": 0, "reprocess": 0}
|
||||
|
||||
def approve(*a, **k):
|
||||
calls["approve"] += 1
|
||||
return ("/restored.flac", {}, "acoustid")
|
||||
|
||||
def reprocess(*a, **k):
|
||||
calls["reprocess"] += 1
|
||||
|
||||
ok = try_accept_version_mismatch_fallback(
|
||||
quarantine_dir="/q", restore_dir="/r",
|
||||
expected_title="Barricades (Movie Ver.)", expected_artist="Hiroyuki Sawano",
|
||||
task_id="t1", batch_id="b1",
|
||||
config_get=_cfg(enabled=False),
|
||||
list_entries=lambda d: _two_instrumental_entries(),
|
||||
approve_entry=approve, reprocess=reprocess,
|
||||
)
|
||||
assert ok is False
|
||||
assert calls == {"approve": 0, "reprocess": 0}
|
||||
|
||||
|
||||
def test_orchestration_accepts_and_reprocesses_with_acoustid_bypass():
|
||||
captured = {}
|
||||
|
||||
def approve(qdir, entry_id, rdir):
|
||||
captured["entry_id"] = entry_id
|
||||
return ("/restored.flac", {"existing": 1}, "acoustid")
|
||||
|
||||
def reprocess(path, context, task_id, batch_id):
|
||||
captured["path"] = path
|
||||
captured["context"] = context
|
||||
captured["task_id"] = task_id
|
||||
|
||||
ok = try_accept_version_mismatch_fallback(
|
||||
quarantine_dir="/q", restore_dir="/r",
|
||||
expected_title="Barricades (Movie Ver.)", expected_artist="Hiroyuki Sawano",
|
||||
task_id="t1", batch_id="b1",
|
||||
config_get=_cfg(),
|
||||
list_entries=lambda d: _two_instrumental_entries(),
|
||||
approve_entry=approve, reprocess=reprocess,
|
||||
)
|
||||
assert ok is True
|
||||
assert captured["entry_id"] == "20260605_120100" # oldest/best
|
||||
assert captured["path"] == "/restored.flac"
|
||||
assert captured["task_id"] == "t1"
|
||||
# Only AcoustID bypassed — integrity/bit-depth gates still run.
|
||||
assert captured["context"]["_skip_quarantine_check"] == "acoustid"
|
||||
assert captured["context"]["_version_mismatch_fallback"] == "instrumental"
|
||||
assert captured["context"]["task_id"] == "t1"
|
||||
assert captured["context"]["batch_id"] == "b1"
|
||||
|
||||
|
||||
def test_orchestration_no_candidate_does_not_reprocess():
|
||||
calls = {"reprocess": 0}
|
||||
|
||||
def reprocess(*a, **k):
|
||||
calls["reprocess"] += 1
|
||||
|
||||
ok = try_accept_version_mismatch_fallback(
|
||||
quarantine_dir="/q", restore_dir="/r",
|
||||
expected_title="Barricades (Movie Ver.)", expected_artist="Hiroyuki Sawano",
|
||||
task_id="t1", batch_id="b1",
|
||||
config_get=_cfg(),
|
||||
list_entries=lambda d: [_entry("20260605_120100", _vm("instrumental"))], # 1 < min 2
|
||||
approve_entry=lambda *a, **k: ("/x", {}, "acoustid"),
|
||||
reprocess=reprocess,
|
||||
)
|
||||
assert ok is False
|
||||
assert calls["reprocess"] == 0
|
||||
|
||||
|
||||
def test_orchestration_approve_failure_returns_false():
|
||||
ok = try_accept_version_mismatch_fallback(
|
||||
quarantine_dir="/q", restore_dir="/r",
|
||||
expected_title="Barricades (Movie Ver.)", expected_artist="Hiroyuki Sawano",
|
||||
task_id="t1", batch_id="b1",
|
||||
config_get=_cfg(),
|
||||
list_entries=lambda d: _two_instrumental_entries(),
|
||||
approve_entry=lambda *a, **k: None, # thin sidecar / move failed
|
||||
reprocess=lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not reprocess")),
|
||||
)
|
||||
assert ok is False
|
||||
113
tests/library/test_direct_id_match.py
Normal file
113
tests/library/test_direct_id_match.py
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
"""Direct-ID manual matching (Ashh: 'just slap MB ID in that search').
|
||||
|
||||
When the right release isn't in the top-8 fuzzy results, the user pastes the
|
||||
exact ID. extract_direct_id detects it (pure); _search_service confirms it
|
||||
via a direct lookup and returns just that entity, falling back to fuzzy
|
||||
search if the paste only looks ID-ish.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from core.library.direct_id import extract_direct_id
|
||||
|
||||
MBID = "1af02ea7-3f00-40ca-804b-41e2dca7e4a9"
|
||||
|
||||
|
||||
# ── pure detector ────────────────────────────────────────────────────────────
|
||||
|
||||
def test_bare_mbid_detected():
|
||||
assert extract_direct_id("musicbrainz", "album", MBID) == MBID
|
||||
assert extract_direct_id("musicbrainz", "artist", MBID.upper()) == MBID # normalized
|
||||
|
||||
|
||||
def test_mbid_in_url_detected():
|
||||
for url in (
|
||||
f"https://musicbrainz.org/release/{MBID}",
|
||||
f"https://musicbrainz.org/release/{MBID}/cover-art",
|
||||
f" https://beta.musicbrainz.org/artist/{MBID} ",
|
||||
):
|
||||
assert extract_direct_id("musicbrainz", "album", url) == MBID
|
||||
|
||||
|
||||
def test_plain_text_query_is_not_an_id():
|
||||
assert extract_direct_id("musicbrainz", "album", "Idols") is None
|
||||
assert extract_direct_id("musicbrainz", "album", "Yungblud Idols") is None
|
||||
assert extract_direct_id("musicbrainz", "album", "") is None
|
||||
assert extract_direct_id("musicbrainz", "album", " ") is None
|
||||
|
||||
|
||||
def test_loose_uuid_without_url_context_is_rejected():
|
||||
# A UUID embedded in free text (not the whole query, no MB URL) is NOT
|
||||
# treated as a direct ID — avoids hijacking a genuine search.
|
||||
assert extract_direct_id("musicbrainz", "album", f"album {MBID} deluxe") is None
|
||||
|
||||
|
||||
def test_non_musicbrainz_services_have_no_direct_id_yet():
|
||||
assert extract_direct_id("spotify", "album", MBID) is None
|
||||
assert extract_direct_id("deezer", "track", "12345") is None
|
||||
|
||||
|
||||
# ── _search_service direct dispatch ──────────────────────────────────────────
|
||||
|
||||
def _wire_mb(monkeypatch, **methods):
|
||||
import core.library.service_search as ss
|
||||
mb_client = MagicMock(**methods)
|
||||
worker = SimpleNamespace(mb_service=SimpleNamespace(mb_client=mb_client))
|
||||
monkeypatch.setattr(ss, "mb_worker", worker)
|
||||
return ss, mb_client
|
||||
|
||||
|
||||
def test_pasted_mbid_returns_single_confirmed_release(monkeypatch):
|
||||
ss, mb_client = _wire_mb(monkeypatch)
|
||||
mb_client.get_release.return_value = {
|
||||
"id": MBID, "title": "Idols", "date": "2025-06-20",
|
||||
"artist-credit": [{"name": "Yungblud"}],
|
||||
}
|
||||
results = ss._search_service("musicbrainz", "album", MBID)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["id"] == MBID
|
||||
assert results[0]["name"] == "Idols"
|
||||
assert "Direct ID match" in results[0]["extra"]
|
||||
assert "Yungblud" in results[0]["extra"]
|
||||
mb_client.get_release.assert_called_once_with(MBID)
|
||||
mb_client.search_release.assert_not_called() # never fuzzy-searched
|
||||
|
||||
|
||||
def test_album_falls_back_to_release_group(monkeypatch):
|
||||
ss, mb_client = _wire_mb(
|
||||
monkeypatch,
|
||||
get_release=lambda mbid: None,
|
||||
get_release_group=lambda mbid: {"id": MBID, "title": "Idols", "artist-credit": []},
|
||||
)
|
||||
results = ss._search_service("musicbrainz", "album", MBID)
|
||||
assert len(results) == 1 and results[0]["name"] == "Idols"
|
||||
|
||||
|
||||
def test_unresolvable_mbid_falls_through_to_fuzzy(monkeypatch):
|
||||
# ID-shaped but doesn't resolve → don't dead-end; run the normal search.
|
||||
ss, mb_client = _wire_mb(
|
||||
monkeypatch,
|
||||
get_release=lambda mbid: None,
|
||||
get_release_group=lambda mbid: None,
|
||||
search_release=lambda q, limit=8, strict=False: [
|
||||
{"id": "other", "title": "Idols (fuzzy)", "artist-credit": [], "date": "", "score": 90},
|
||||
],
|
||||
)
|
||||
results = ss._search_service("musicbrainz", "album", MBID)
|
||||
assert len(results) == 1 and results[0]["id"] == "other" # fuzzy result
|
||||
|
||||
|
||||
def test_plain_query_skips_direct_lookup(monkeypatch):
|
||||
ss, mb_client = _wire_mb(
|
||||
monkeypatch,
|
||||
search_release=lambda q, limit=8, strict=False: [
|
||||
{"id": "r1", "title": "Idols", "artist-credit": [], "date": "", "score": 100},
|
||||
],
|
||||
)
|
||||
results = ss._search_service("musicbrainz", "album", "Idols")
|
||||
assert results[0]["id"] == "r1"
|
||||
mb_client.get_release.assert_not_called() # no wasted direct lookup
|
||||
91
tests/library/test_expired_cleanup.py
Normal file
91
tests/library/test_expired_cleanup.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
"""Pure expiry decision for the Expired Download Cleaner."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from core.library.expired_cleanup import (
|
||||
retention_cutoff,
|
||||
is_expired,
|
||||
select_expired,
|
||||
)
|
||||
|
||||
NOW = datetime(2026, 6, 7, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _entry(origin="playlist", days_old=100, play_count=0, protected=False, eid=1):
|
||||
return {
|
||||
"id": eid, "origin": origin, "play_count": play_count, "protected": protected,
|
||||
"created_at": (NOW - timedelta(days=days_old)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
|
||||
|
||||
def _check(entry, wl="off", pl="2mo", min_plays=2):
|
||||
return is_expired(entry, watchlist_retention=wl, playlist_retention=pl,
|
||||
min_plays=min_plays, now=NOW)
|
||||
|
||||
|
||||
# ── retention windows ────────────────────────────────────────────────────────
|
||||
|
||||
def test_retention_cutoff_maps_durations():
|
||||
assert retention_cutoff("2mo", NOW) == NOW - timedelta(days=60)
|
||||
assert retention_cutoff("1w", NOW) == NOW - timedelta(days=7)
|
||||
assert retention_cutoff("off", NOW) is None
|
||||
assert retention_cutoff(None, NOW) is None
|
||||
assert retention_cutoff("bogus", NOW) is None
|
||||
|
||||
|
||||
def test_expired_only_past_window():
|
||||
assert _check(_entry(days_old=70), pl="2mo") is True # 70 > 60d
|
||||
assert _check(_entry(days_old=50), pl="2mo") is False # 50 < 60d
|
||||
|
||||
|
||||
def test_off_retention_never_expires():
|
||||
assert _check(_entry(origin="watchlist", days_old=999), wl="off") is False
|
||||
|
||||
|
||||
def test_origin_uses_its_own_window():
|
||||
wl = _entry(origin="watchlist", days_old=30)
|
||||
# watchlist=1w (expired at 30d), playlist=off
|
||||
assert is_expired(wl, watchlist_retention="1w", playlist_retention="off",
|
||||
min_plays=2, now=NOW) is True
|
||||
pl = _entry(origin="playlist", days_old=30)
|
||||
assert is_expired(pl, watchlist_retention="1w", playlist_retention="off",
|
||||
min_plays=2, now=NOW) is False # playlist off
|
||||
|
||||
|
||||
# ── the keep guards ──────────────────────────────────────────────────────────
|
||||
|
||||
def test_protected_kept_even_if_old():
|
||||
assert _check(_entry(days_old=999, protected=True), pl="1w") is False
|
||||
|
||||
|
||||
def test_played_more_than_once_kept():
|
||||
assert _check(_entry(days_old=999, play_count=2), pl="1w", min_plays=2) is False
|
||||
assert _check(_entry(days_old=999, play_count=1), pl="1w", min_plays=2) is True # one play = deletable
|
||||
assert _check(_entry(days_old=999, play_count=0), pl="1w", min_plays=2) is True
|
||||
|
||||
|
||||
def test_min_plays_threshold_configurable():
|
||||
e = _entry(days_old=999, play_count=1)
|
||||
assert _check(e, pl="1w", min_plays=1) is False # keep-if-played-at-least-1
|
||||
assert _check(e, pl="1w", min_plays=3) is True # needs 3 plays to keep
|
||||
|
||||
|
||||
def test_unknown_age_never_deleted():
|
||||
e = _entry(days_old=999)
|
||||
e["created_at"] = "garbage"
|
||||
assert _check(e, pl="1w") is False
|
||||
|
||||
|
||||
# ── select_expired ───────────────────────────────────────────────────────────
|
||||
|
||||
def test_select_expired_filters():
|
||||
entries = [
|
||||
_entry(eid=1, days_old=70, play_count=0), # expired
|
||||
_entry(eid=2, days_old=70, play_count=5), # listened → keep
|
||||
_entry(eid=3, days_old=70, protected=True), # mirrored → keep
|
||||
_entry(eid=4, days_old=10), # too new → keep
|
||||
]
|
||||
out = select_expired(entries, watchlist_retention="off", playlist_retention="2mo")
|
||||
assert [e["id"] for e in out] == [1]
|
||||
|
|
@ -191,6 +191,35 @@ def test_trust_gate_rejects_when_two_high_mb_scores_tie(service):
|
|||
assert aliases == []
|
||||
|
||||
|
||||
def test_trust_gate_uses_mb_score_leader_not_combined_leader(service):
|
||||
# Production case "Sawano Hiroyuki": a same-script DECOY entity leads on
|
||||
# COMBINED score (high local sim, mb_score 83) but sits just under the
|
||||
# 0.85 combined bar, while the genuine cross-script artist has mb_score
|
||||
# 100 and ~0 local sim → lowest combined, sorted last. The mb-only escape
|
||||
# must evaluate the MB-SCORE leader, not scored[0] (the combined leader),
|
||||
# otherwise it inspects mb_score 83 < 95 and wrongly returns [].
|
||||
service._calculate_similarity = (
|
||||
lambda a, b: 0.82 if b == 'SawanoHiroyuki[nZk]' else 0.0
|
||||
)
|
||||
service.mb_client.search_artist.return_value = [
|
||||
{'id': 'mbid-decoy', 'name': 'SawanoHiroyuki[nZk]', 'score': 83},
|
||||
{'id': 'mbid-canonical', 'name': '澤野弘之', 'score': 100},
|
||||
]
|
||||
|
||||
def get_artist(mbid, **kwargs):
|
||||
if mbid == 'mbid-canonical':
|
||||
return {'name': '澤野弘之', 'aliases': [{'name': 'Hiroyuki Sawano'}]}
|
||||
return {'name': 'SawanoHiroyuki[nZk]', 'aliases': []}
|
||||
service.mb_client.get_artist.side_effect = get_artist
|
||||
|
||||
aliases = service.lookup_artist_aliases('Sawano Hiroyuki')
|
||||
# The canonical kanji name must come back (its alias set was fetched).
|
||||
assert '澤野弘之' in aliases
|
||||
# And we must have fetched the MB-score leader, not the decoy.
|
||||
service.mb_client.get_artist.assert_called_once()
|
||||
assert service.mb_client.get_artist.call_args.args[0] == 'mbid-canonical'
|
||||
|
||||
|
||||
def test_trust_gate_passes_combined_score_when_local_sim_strong(service):
|
||||
# Same-script case from #442 — local sim high. Should still pass
|
||||
# (no regression on the existing path).
|
||||
|
|
|
|||
65
tests/matching/test_numeric_release_guard.py
Normal file
65
tests/matching/test_numeric_release_guard.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
"""Sokhi's wrong-cover-art report: 'Vol.4' must never match 'Vol.4.5'.
|
||||
|
||||
Two paths served wrong art for volume-numbered series:
|
||||
- the art picker's _album_matches subset check (Vol.4's tokens are a subset
|
||||
of Vol.4.5's once CJK/punctuation normalizes away) — covered in
|
||||
tests/metadata/test_art_lookup.py
|
||||
- MusicBrainz match_release: 0.97 string similarity let the wrong volume
|
||||
win, and its MBID feeds CAA art with no downstream validation — covered
|
||||
here, plus the shared helper itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from core.text.title_match import numeric_tokens_differ
|
||||
from core.musicbrainz_service import MusicBrainzService
|
||||
|
||||
VOL4 = "B小町 - TVアニメ「【推しの子】」キャラクターソングCD Vol.4"
|
||||
VOL45 = "B小町 - TVアニメ「【推しの子】」キャラクターソングCD Vol.4.5"
|
||||
|
||||
|
||||
def test_helper_volume_and_sequel_differ():
|
||||
assert numeric_tokens_differ(VOL4, VOL45)
|
||||
assert numeric_tokens_differ("Album", "Album 2")
|
||||
assert numeric_tokens_differ("Now 99", "Now 100")
|
||||
|
||||
|
||||
def test_helper_shared_or_no_digits_match():
|
||||
assert not numeric_tokens_differ("1989", "1989 (Deluxe)")
|
||||
assert not numeric_tokens_differ(VOL4, VOL4)
|
||||
assert not numeric_tokens_differ("IGOR", "IGOR (Deluxe)")
|
||||
assert not numeric_tokens_differ("", "")
|
||||
|
||||
|
||||
def _service_with_results(results):
|
||||
svc = MusicBrainzService.__new__(MusicBrainzService)
|
||||
svc.mb_client = MagicMock()
|
||||
svc.mb_client.search_release.return_value = results
|
||||
svc._check_cache = lambda *a, **k: None
|
||||
svc._save_to_cache = lambda *a, **k: None
|
||||
return svc
|
||||
|
||||
|
||||
def _result(title, score=100, mbid="mb-x"):
|
||||
return {
|
||||
'id': mbid, 'title': title, 'score': score,
|
||||
'artist-credit': [{'artist': {'name': 'B小町'}}],
|
||||
}
|
||||
|
||||
|
||||
def test_match_release_rejects_wrong_volume():
|
||||
"""Only the wrong volume is returned by search → no match at all is
|
||||
better than caching Vol.4.5's MBID (CAA would serve its art unvalidated)."""
|
||||
svc = _service_with_results([_result(VOL45, score=100, mbid='mb-wrong')])
|
||||
assert svc.match_release(VOL4, 'B小町') is None
|
||||
|
||||
|
||||
def test_match_release_prefers_exact_volume_over_near_identical():
|
||||
svc = _service_with_results([
|
||||
_result(VOL45, score=100, mbid='mb-wrong'),
|
||||
_result(VOL4, score=90, mbid='mb-right'),
|
||||
])
|
||||
got = svc.match_release(VOL4, 'B小町')
|
||||
assert got and got['mbid'] == 'mb-right'
|
||||
|
|
@ -181,6 +181,27 @@ def test_album_matches_unknown_requested_artist_allows_album_match():
|
|||
assert art_lookup._album_matches("", "1989", "Taylor Swift", "1989")
|
||||
|
||||
|
||||
def test_album_matches_rejects_numeric_difference():
|
||||
"""Sokhi: same series, different volume number. CJK strips to latin
|
||||
tokens, so Vol.4 was a token-subset of Vol.4.5 and inherited its art.
|
||||
A number on only one side = a different release, never a suffix."""
|
||||
A = "B小町"
|
||||
assert not art_lookup._album_matches(
|
||||
A, "B小町 - TVアニメ「【推しの子】」キャラクターソングCD Vol.4",
|
||||
A, "B小町 - TVアニメ「【推しの子】」キャラクターソングCD Vol.4.5")
|
||||
assert not art_lookup._album_matches(
|
||||
A, "B小町 - TVアニメ「【推しの子】」キャラクターソングCD Vol.2",
|
||||
A, "B小町 - TVアニメ「【推しの子】」キャラクターソングCD Vol.2.5")
|
||||
# Sequels are different albums too.
|
||||
assert not art_lookup._album_matches("Artist", "Album", "Artist", "Album 2")
|
||||
# Identical volume numbers still match.
|
||||
assert art_lookup._album_matches(
|
||||
A, "B小町 - TVアニメ「【推しの子】」キャラクターソングCD Vol.4",
|
||||
A, "B小町 - TVアニメ「【推しの子】」キャラクターソングCD Vol.4")
|
||||
# Numeric token shared by BOTH sides keeps non-numeric suffix tolerance.
|
||||
assert art_lookup._album_matches("Taylor Swift", "1989", "Taylor Swift", "1989 (Deluxe)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build_art_lookup — caching + guarding
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -45,26 +45,24 @@ class TestUpgradeArtUrl:
|
|||
url = 'https://cdn-images.dzcdn.net/images/cover/abc/1000x1000-000000-80-0-0.jpg'
|
||||
assert '1900x1900' in _upgrade_art_url(url)
|
||||
|
||||
def test_caa_thumbnail_upgraded_to_1200(self):
|
||||
# MusicBrainz art arrives as the /front-250 thumbnail; upgrade to the
|
||||
# 1200px CDN thumbnail (NOT the flaky bare /front original).
|
||||
def test_caa_thumbnail_upgraded_to_native_original(self):
|
||||
# #806: MusicBrainz art arrives as the /front-250 thumbnail; upgrade
|
||||
# to the bare /front ORIGINAL (native res, frequently 3000px+).
|
||||
# Flakiness of the original is handled by _fetch_art_bytes' 1200px
|
||||
# midpoint fallback, not by capping the URL here.
|
||||
url = 'https://coverartarchive.org/release/abc-123/front-250'
|
||||
assert _upgrade_art_url(url) == 'https://coverartarchive.org/release/abc-123/front-1200'
|
||||
assert _upgrade_art_url(url) == 'https://coverartarchive.org/release/abc-123/front'
|
||||
|
||||
def test_caa_500_scope_and_idempotent(self):
|
||||
def test_caa_all_sizes_and_scopes_upgrade_to_original(self):
|
||||
assert _upgrade_art_url('https://coverartarchive.org/release/x/front-500') \
|
||||
== 'https://coverartarchive.org/release/x/front-1200'
|
||||
# release-group scope works the same.
|
||||
== 'https://coverartarchive.org/release/x/front'
|
||||
assert _upgrade_art_url('https://coverartarchive.org/release/x/front-1200') \
|
||||
== 'https://coverartarchive.org/release/x/front'
|
||||
# release-group scope works the same (the URL shape from #806).
|
||||
assert _upgrade_art_url('https://coverartarchive.org/release-group/y/front-250') \
|
||||
== 'https://coverartarchive.org/release-group/y/front-1200'
|
||||
# Idempotent — an already-1200 URL stays put.
|
||||
assert _upgrade_art_url('https://coverartarchive.org/release-group/y/front-1200') \
|
||||
== 'https://coverartarchive.org/release-group/y/front-1200'
|
||||
== 'https://coverartarchive.org/release-group/y/front'
|
||||
|
||||
def test_caa_bare_front_left_alone(self):
|
||||
# The bare /front original is intentionally NOT what we want; the
|
||||
# sized-thumbnail regex doesn't touch it (and it never reaches the
|
||||
# helper in practice — _cover_art_url always emits /front-250).
|
||||
def test_caa_bare_front_idempotent(self):
|
||||
url = 'https://coverartarchive.org/release/abc/front'
|
||||
assert _upgrade_art_url(url) == url
|
||||
|
||||
|
|
@ -163,3 +161,92 @@ class TestFetchArtBytes:
|
|||
def test_empty_url_returns_none(self):
|
||||
assert _fetch_art_bytes('') == (None, None)
|
||||
assert _fetch_art_bytes(None) == (None, None)
|
||||
|
||||
# ── #806: CAA native-res chain — original → 1200px midpoint → thumbnail ──
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_caa_cooldown(self, monkeypatch):
|
||||
# The negative cache persists module-globally; isolate every test.
|
||||
import core.metadata.artwork as aw
|
||||
monkeypatch.setattr(aw, '_caa_original_down_until', 0.0)
|
||||
|
||||
def test_caa_fetches_native_original_first(self, monkeypatch):
|
||||
calls = []
|
||||
|
||||
def fake_urlopen(url, timeout=None):
|
||||
calls.append(url)
|
||||
return _FakeResponse(b'native-3000px')
|
||||
|
||||
monkeypatch.setattr('core.metadata.artwork.urllib.request.urlopen', fake_urlopen)
|
||||
|
||||
data, _ = _fetch_art_bytes('https://coverartarchive.org/release/r1/front-250')
|
||||
|
||||
assert data == b'native-3000px'
|
||||
assert calls == ['https://coverartarchive.org/release/r1/front']
|
||||
|
||||
def test_caa_flaky_original_degrades_to_1200_not_250(self, monkeypatch):
|
||||
"""archive.org flakiness lands on the old 1200px behavior — the
|
||||
midpoint sits BEFORE the tiny original thumbnail in the chain."""
|
||||
calls = []
|
||||
|
||||
def fake_urlopen(url, timeout=None):
|
||||
calls.append(url)
|
||||
if url.endswith('/front'):
|
||||
raise Exception('503 archive.org is having a day')
|
||||
return _FakeResponse(b'cdn-1200px')
|
||||
|
||||
monkeypatch.setattr('core.metadata.artwork.urllib.request.urlopen', fake_urlopen)
|
||||
|
||||
data, _ = _fetch_art_bytes('https://coverartarchive.org/release/r1/front-250')
|
||||
|
||||
assert data == b'cdn-1200px'
|
||||
assert calls == [
|
||||
'https://coverartarchive.org/release/r1/front',
|
||||
'https://coverartarchive.org/release/r1/front-1200',
|
||||
]
|
||||
|
||||
def test_caa_full_chain_ends_at_the_original_thumbnail(self, monkeypatch):
|
||||
calls = []
|
||||
|
||||
def fake_urlopen(url, timeout=None):
|
||||
calls.append(url)
|
||||
if url.endswith('/front') or url.endswith('-1200'):
|
||||
raise Exception('refused')
|
||||
return _FakeResponse(b'tiny-250px')
|
||||
|
||||
monkeypatch.setattr('core.metadata.artwork.urllib.request.urlopen', fake_urlopen)
|
||||
|
||||
data, _ = _fetch_art_bytes('https://coverartarchive.org/release/r1/front-250')
|
||||
|
||||
assert data == b'tiny-250px'
|
||||
assert calls == [
|
||||
'https://coverartarchive.org/release/r1/front',
|
||||
'https://coverartarchive.org/release/r1/front-1200',
|
||||
'https://coverartarchive.org/release/r1/front-250',
|
||||
]
|
||||
|
||||
def test_caa_outage_cooldown_skips_originals_for_subsequent_fetches(self, monkeypatch):
|
||||
"""One archive.org failure puts originals on cooldown: the NEXT track's
|
||||
fetch goes straight to the 1200px CDN — no 10s timeout per track during
|
||||
an outage (art is fetched per track)."""
|
||||
calls = []
|
||||
|
||||
def fake_urlopen(url, timeout=None):
|
||||
calls.append(url)
|
||||
if url.endswith('/front'):
|
||||
raise Exception('archive.org down')
|
||||
return _FakeResponse(b'cdn-1200px')
|
||||
|
||||
monkeypatch.setattr('core.metadata.artwork.urllib.request.urlopen', fake_urlopen)
|
||||
|
||||
# Track 1: pays the failed original once, falls back to 1200.
|
||||
_fetch_art_bytes('https://coverartarchive.org/release/r1/front-250')
|
||||
# Track 2: cooldown active — never touches the original.
|
||||
data, _ = _fetch_art_bytes('https://coverartarchive.org/release/r2/front-250')
|
||||
|
||||
assert data == b'cdn-1200px'
|
||||
assert calls == [
|
||||
'https://coverartarchive.org/release/r1/front', # track 1 pays once
|
||||
'https://coverartarchive.org/release/r1/front-1200', # and falls back
|
||||
'https://coverartarchive.org/release/r2/front-1200', # track 2 skips straight to CDN
|
||||
]
|
||||
|
|
|
|||
|
|
@ -494,3 +494,70 @@ class TestDeezerContributorsUpgrade:
|
|||
|
||||
assert meta["_artists_list"] == ["Primary"]
|
||||
fake_deezer.get_track_details.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Netti93 follow-up: the DIRECT download flow (Search with Deezer ->
|
||||
# Download Now via Tidal). Deezer /search returns ONE artist; the full
|
||||
# contributors list only exists on /track/<id>. The download context must
|
||||
# trigger the contributors upgrade and then honor feat_in_title /
|
||||
# artist_separator — pre-fix this only worked after a manual retag.
|
||||
# Verified live against Deezer's API for the reported track
|
||||
# ('VERLIEBT IN MICH', FAYAN feat. Dalton, id 3526028401).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _netti_context():
|
||||
"""The exact shape the search page's Deezer result produces, riding a
|
||||
Tidal download (provider identity lives on the candidate, not here)."""
|
||||
return {
|
||||
"track_info": {
|
||||
"id": 3526028401,
|
||||
"name": "VERLIEBT IN MICH",
|
||||
"artists": ["FAYAN"], # /search: primary only
|
||||
"album": "VERLIEBT IN MICH",
|
||||
"source": "deezer",
|
||||
},
|
||||
"original_search_result": {
|
||||
"username": "tidal", "filename": "x.flac",
|
||||
"title": "VERLIEBT IN MICH", "artist": "FAYAN",
|
||||
},
|
||||
"task_id": "t1",
|
||||
}
|
||||
|
||||
|
||||
def _call_extract_netti(cfg_overrides):
|
||||
import core.metadata as metadata_pkg
|
||||
from core.metadata import source as src_module
|
||||
|
||||
deezer = MagicMock()
|
||||
deezer.get_track_details.return_value = {
|
||||
"id": 3526028401,
|
||||
"artists": ["FAYAN", "Dalton"], # /track/<id> contributors
|
||||
}
|
||||
with patch.object(src_module, "get_config_manager", return_value=_make_cfg(cfg_overrides)), \
|
||||
patch.object(metadata_pkg, "get_deezer_client", return_value=deezer):
|
||||
return src_module.extract_source_metadata(
|
||||
_netti_context(), {"name": "FAYAN"}, {"name": "VERLIEBT IN MICH"})
|
||||
|
||||
|
||||
class TestDeezerDirectDownloadFlow:
|
||||
def test_contributors_upgrade_plus_feat_in_title(self):
|
||||
meta = _call_extract_netti({
|
||||
"metadata_enhancement.tags.write_multi_artist": True,
|
||||
"metadata_enhancement.tags.feat_in_title": True,
|
||||
"metadata_enhancement.tags.artist_separator": ";",
|
||||
})
|
||||
assert meta["_artists_list"] == ["FAYAN", "Dalton"]
|
||||
assert meta["artist"] == "FAYAN" # primary only
|
||||
assert meta["title"] == "VERLIEBT IN MICH (feat. Dalton)"
|
||||
|
||||
def test_contributors_upgrade_with_separator_join(self):
|
||||
meta = _call_extract_netti({
|
||||
"metadata_enhancement.tags.write_multi_artist": True,
|
||||
"metadata_enhancement.tags.feat_in_title": False,
|
||||
"metadata_enhancement.tags.artist_separator": ";",
|
||||
})
|
||||
assert meta["_artists_list"] == ["FAYAN", "Dalton"]
|
||||
assert meta["artist"] == "FAYAN;Dalton"
|
||||
assert meta["title"] == "VERLIEBT IN MICH"
|
||||
|
|
|
|||
|
|
@ -200,3 +200,18 @@ def test_worker_drives_a_real_stream_session(tmp_path):
|
|||
assert session['status'] == 'error'
|
||||
assert 'Failed to initiate' in session['error_message']
|
||||
assert session['track_info'] == {'username': 'u', 'filename': 'song.flac', 'size': 1}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Observability: prep logs must actually reach app.log
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_prepare_logger_is_in_soulsync_namespace():
|
||||
"""Handlers only attach to the soulsync.* hierarchy. A bare
|
||||
getLogger(__name__) gave this module a 'core.streaming.prepare' logger with
|
||||
no handler — every prep log (including failures) vanished, which made the
|
||||
broken-stream report undebuggable from app.log. Lock the namespace."""
|
||||
assert sp.logger.name.startswith('soulsync.'), (
|
||||
f"prepare logger '{sp.logger.name}' is outside the soulsync.* namespace "
|
||||
"— its output never reaches app.log"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -145,3 +145,68 @@ def test_apply_counts_failures_without_raising(tmp_path, monkeypatch):
|
|||
res = aa.apply_art_to_album_files([str(f1)], {'album': 'B'}, {'album_name': 'B'}, folder=str(tmp_path))
|
||||
assert res['embedded'] == 0
|
||||
assert res['failed'] == 1 # save() raised (read-only) — counted, not crashed
|
||||
|
||||
|
||||
# ── read-only filesystem detection (Tim: docker ':ro' mount, Errno 30) ──
|
||||
|
||||
|
||||
def test_apply_does_not_trust_statvfs_writable_fs_succeeds(tmp_path, monkeypatch):
|
||||
"""Regression (Sokhi): a WRITABLE union/FUSE/NFS mount can misreport
|
||||
ST_RDONLY in statvfs. The apply must NOT use statvfs — it writes anyway,
|
||||
and only the actual write decides. Here the embed succeeds even though
|
||||
statvfs (if it were consulted) would claim read-only."""
|
||||
f = tmp_path / 'a.mp3'
|
||||
f.write_bytes(b'')
|
||||
saved = []
|
||||
audio = SimpleNamespace(pictures=[], tags=None, add_tags=lambda: None,
|
||||
save=lambda: saved.append(True))
|
||||
monkeypatch.setattr(aa, 'get_mutagen_symbols', lambda: _fake_symbols(audio))
|
||||
monkeypatch.setattr(aa, 'embed_album_art_metadata', lambda *a, **k: True)
|
||||
monkeypatch.setattr(aa, 'download_cover_art', lambda *a, **k: None)
|
||||
# Even if statvfs lies and says read-only, the write still happens.
|
||||
monkeypatch.setattr(aa.os, 'statvfs',
|
||||
lambda p: SimpleNamespace(f_flag=getattr(aa.os, 'ST_RDONLY', 1)),
|
||||
raising=False)
|
||||
|
||||
res = aa.apply_art_to_album_files([str(f)], {}, {}, folder=str(tmp_path))
|
||||
|
||||
assert res['embedded'] == 1 and saved == [True]
|
||||
assert res['read_only_fs'] is False # statvfs ignored; write succeeded
|
||||
|
||||
|
||||
def test_apply_flags_erofs_from_actual_write(tmp_path, monkeypatch):
|
||||
"""read-only is detected from a real EROFS on write (the only honest test),
|
||||
and it fast-fails the remaining files."""
|
||||
import errno as _errno
|
||||
f1 = tmp_path / 'a.mp3'; f1.write_bytes(b'')
|
||||
f2 = tmp_path / 'b.mp3'; f2.write_bytes(b'')
|
||||
|
||||
def _boom():
|
||||
raise OSError(_errno.EROFS, 'Read-only file system')
|
||||
audio = SimpleNamespace(pictures=[], tags={'ok': 1}, save=_boom)
|
||||
monkeypatch.setattr(aa, 'get_mutagen_symbols', lambda: _fake_symbols(audio))
|
||||
monkeypatch.setattr(aa, 'embed_album_art_metadata', lambda *a, **k: True)
|
||||
monkeypatch.setattr(aa, 'download_cover_art', lambda *a, **k: None)
|
||||
|
||||
res = aa.apply_art_to_album_files([str(f1), str(f2)], {}, {}, folder=str(tmp_path))
|
||||
|
||||
assert res['read_only_fs'] is True
|
||||
assert res['failed'] == 2 # first EROFS fails it + bails the rest
|
||||
|
||||
|
||||
def test_apply_normal_failure_not_flagged_read_only(tmp_path, monkeypatch):
|
||||
def _boom():
|
||||
raise PermissionError(13, 'Permission denied')
|
||||
f = tmp_path / 'a.mp3'
|
||||
f.write_bytes(b'')
|
||||
audio = SimpleNamespace(pictures=[], tags={'ok': 1}, save=_boom)
|
||||
monkeypatch.setattr(aa, 'get_mutagen_symbols', lambda: _fake_symbols(audio))
|
||||
monkeypatch.setattr(aa, 'embed_album_art_metadata', lambda *a, **k: True)
|
||||
monkeypatch.setattr(aa, 'download_cover_art', lambda *a, **k: None)
|
||||
|
||||
res = aa.apply_art_to_album_files([str(f)], {}, {}, folder=str(tmp_path))
|
||||
|
||||
assert res['read_only_fs'] is False # EACCES is chmod-able, EROFS is not
|
||||
assert res['failed'] == 1
|
||||
|
||||
|
||||
|
|
|
|||
84
tests/test_artist_albums_cache_poisoning.py
Normal file
84
tests/test_artist_albums_cache_poisoning.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""Artist album-list cache poisoning (Boulder: 'Taylor Swift has 8 albums,
|
||||
nothing before 2022').
|
||||
|
||||
get_artist_albums caches its result under an UNQUALIFIED key (no limit/page
|
||||
info). The watchlist's new-release probe (limit=5, max_pages=1) stored its
|
||||
truncated page in that slot, so the artist detail page — which reads the
|
||||
cache — showed only the newest handful of releases for every watchlist
|
||||
artist. The writer must never cache a fetch that stopped while more pages
|
||||
existed; complete fetches (even small real discographies) stay cacheable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import core.spotify_client as sc
|
||||
from core.spotify_client import SpotifyClient
|
||||
|
||||
|
||||
def _album(i):
|
||||
return {
|
||||
'id': f'al{i}', 'name': f'Album {i}', 'album_type': 'album',
|
||||
'artists': [{'id': 'ar1', 'name': 'Taylor Swift'}],
|
||||
'release_date': '2024-01-01', 'total_tracks': 12, 'images': [],
|
||||
'external_urls': {},
|
||||
}
|
||||
|
||||
|
||||
def _client(monkeypatch, pages):
|
||||
"""Fake sp.artist_albums + sp.next over a list of page dicts."""
|
||||
client = SpotifyClient.__new__(SpotifyClient)
|
||||
fake = MagicMock()
|
||||
fake.artist_albums.return_value = pages[0]
|
||||
fake.next.side_effect = pages[1:]
|
||||
client.sp = fake
|
||||
monkeypatch.setattr(client, 'is_spotify_authenticated', lambda: True)
|
||||
monkeypatch.setattr(sc, '_last_api_call_time', 0)
|
||||
store_calls = []
|
||||
cache = MagicMock()
|
||||
cache.get_entity.return_value = None
|
||||
cache.store_entity.side_effect = lambda *a, **k: store_calls.append(a)
|
||||
monkeypatch.setattr(sc, 'get_metadata_cache', lambda: cache)
|
||||
return client, store_calls
|
||||
|
||||
|
||||
def test_truncated_fetch_is_not_cached(monkeypatch):
|
||||
# Two pages exist; max_pages=1 stops with a 'next' pending -> truncated.
|
||||
pages = [
|
||||
{'items': [_album(1), _album(2)], 'next': 'page2-url'},
|
||||
{'items': [_album(3)], 'next': None},
|
||||
]
|
||||
client, stores = _client(monkeypatch, pages)
|
||||
|
||||
albums = client.get_artist_albums('ar1', limit=5, skip_cache=True, max_pages=1)
|
||||
|
||||
assert len(albums) == 2 # the probe still works
|
||||
album_list_stores = [s for s in stores if s[1] == 'artist']
|
||||
assert album_list_stores == [] # but never poisons the slot
|
||||
|
||||
|
||||
def test_complete_fetch_is_cached(monkeypatch):
|
||||
pages = [
|
||||
{'items': [_album(1), _album(2)], 'next': 'page2-url'},
|
||||
{'items': [_album(3)], 'next': None},
|
||||
]
|
||||
client, stores = _client(monkeypatch, pages)
|
||||
|
||||
albums = client.get_artist_albums('ar1', limit=50, skip_cache=True, max_pages=0)
|
||||
|
||||
assert len(albums) == 3
|
||||
album_list_stores = [s for s in stores if s[1] == 'artist']
|
||||
assert len(album_list_stores) == 1 # full discography cached
|
||||
|
||||
|
||||
def test_small_real_discography_with_page_cap_still_cached(monkeypatch):
|
||||
# Artist genuinely has one page; max_pages=1 didn't truncate anything.
|
||||
pages = [{'items': [_album(1)], 'next': None}]
|
||||
client, stores = _client(monkeypatch, pages)
|
||||
|
||||
albums = client.get_artist_albums('ar1', limit=5, skip_cache=True, max_pages=1)
|
||||
|
||||
assert len(albums) == 1
|
||||
album_list_stores = [s for s in stores if s[1] == 'artist']
|
||||
assert len(album_list_stores) == 1 # complete -> cacheable
|
||||
106
tests/test_canonical_alternates.py
Normal file
106
tests/test_canonical_alternates.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
"""Tests for #767-2 alternate-edition fetching: the pure same-release name
|
||||
matcher, and the production default_fetch_alternates wired over fake source APIs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import core.metadata.canonical_resolver as cr
|
||||
from core.metadata.canonical_resolver import (
|
||||
_release_name_key,
|
||||
_same_release,
|
||||
default_fetch_alternates,
|
||||
)
|
||||
|
||||
|
||||
# ── pure same-release name matching ───────────────────────────────────────────
|
||||
|
||||
def test_release_name_key_strips_editions_and_punctuation():
|
||||
assert _release_name_key("Scatterbrain") == "scatterbrain"
|
||||
assert _release_name_key("Scatterbrain (Deluxe Edition)") == "scatterbrain"
|
||||
assert _release_name_key("Scatterbrain - Single") == "scatterbrain"
|
||||
assert _release_name_key("Scatterbrain [Remastered]") == "scatterbrain"
|
||||
assert _release_name_key("Scatterbrain (Expanded)") == "scatterbrain"
|
||||
|
||||
|
||||
def test_same_release_matches_editions_of_one_album():
|
||||
assert _same_release("Scatterbrain", "Scatterbrain (Deluxe)")
|
||||
assert _same_release("Scatterbrain - Single", "Scatterbrain (Deluxe Edition)")
|
||||
assert _same_release("The Wall", "The Wall [Remastered]")
|
||||
|
||||
|
||||
def test_same_release_rejects_different_albums():
|
||||
assert not _same_release("Scatterbrain", "Brain Scatter")
|
||||
assert not _same_release("Yellow", "Parachutes")
|
||||
assert not _same_release("", "Anything") # empty key never matches
|
||||
|
||||
|
||||
# ── production fetcher over fake source APIs ──────────────────────────────────
|
||||
|
||||
SINGLE = [{"name": "Scatterbrain", "track_number": 1, "duration_ms": 129_000}]
|
||||
DELUXE = [{"name": "Intro", "track_number": 1, "duration_ms": 200_000}] + [
|
||||
{"name": f"Track {i}", "track_number": i + 1, "duration_ms": 180_000}
|
||||
for i in range(1, 10)
|
||||
]
|
||||
|
||||
|
||||
def _install_fake_apis(monkeypatch, *, artist_albums, tracklists, album_meta=None):
|
||||
"""Patch the album_tracks module functions default_fetch_alternates imports."""
|
||||
import core.metadata.album_tracks as at
|
||||
|
||||
monkeypatch.setattr(at, "get_album_for_source",
|
||||
lambda s, aid: (album_meta or {}).get(aid), raising=True)
|
||||
monkeypatch.setattr(at, "get_artist_albums_for_source",
|
||||
lambda s, a_id, a_name, **kw: artist_albums, raising=True)
|
||||
# default_fetch_alternates pulls per-edition tracklists via default_fetch_tracklist,
|
||||
# which calls get_album_tracks_for_source in the metadata_service module.
|
||||
monkeypatch.setattr(
|
||||
"core.metadata_service.get_album_tracks_for_source",
|
||||
lambda s, aid: tracklists.get(aid), raising=True,
|
||||
)
|
||||
|
||||
|
||||
def test_default_fetch_alternates_finds_the_single(monkeypatch):
|
||||
artist_albums = [
|
||||
{"id": "sp_deluxe", "name": "Scatterbrain (Deluxe)"},
|
||||
{"id": "sp_single", "name": "Scatterbrain - Single"},
|
||||
{"id": "other", "name": "A Different Album"},
|
||||
]
|
||||
tracklists = {"sp_deluxe": DELUXE, "sp_single": SINGLE, "other": SINGLE}
|
||||
_install_fake_apis(monkeypatch, artist_albums=artist_albums, tracklists=tracklists)
|
||||
|
||||
out = default_fetch_alternates(
|
||||
"spotify", "sp_deluxe",
|
||||
artist_id="art1", artist_name="The Band", album_title="Scatterbrain",
|
||||
)
|
||||
ids = {e["album_id"] for e in out}
|
||||
assert ids == {"sp_deluxe", "sp_single"} # the unrelated album is excluded
|
||||
single = next(e for e in out if e["album_id"] == "sp_single")
|
||||
assert len(single["tracks"]) == 1 and single["tracks"][0]["duration_ms"] == 129_000
|
||||
|
||||
|
||||
def test_default_fetch_alternates_discovers_artist_from_album_meta(monkeypatch):
|
||||
# No artist context supplied -> it must call get_album_for_source to learn it.
|
||||
album_meta = {"sp_deluxe": {"title": "Scatterbrain", "artist_id": "art1", "artist": "The Band"}}
|
||||
artist_albums = [{"id": "sp_single", "name": "Scatterbrain (Single)"}]
|
||||
tracklists = {"sp_single": SINGLE}
|
||||
_install_fake_apis(monkeypatch, artist_albums=artist_albums,
|
||||
tracklists=tracklists, album_meta=album_meta)
|
||||
|
||||
out = default_fetch_alternates("spotify", "sp_deluxe")
|
||||
assert [e["album_id"] for e in out] == ["sp_single"]
|
||||
|
||||
|
||||
def test_default_fetch_alternates_empty_when_no_artist(monkeypatch):
|
||||
_install_fake_apis(monkeypatch, artist_albums=[], tracklists={})
|
||||
out = default_fetch_alternates("spotify", "x", album_title="Scatterbrain")
|
||||
assert out == [] # no artist id/name and no album meta -> nothing to search
|
||||
|
||||
|
||||
def test_default_fetch_alternates_caps_editions(monkeypatch):
|
||||
# 10 same-release editions, cap is 6.
|
||||
artist_albums = [{"id": f"e{i}", "name": "Scatterbrain (Version %d)" % i} for i in range(10)]
|
||||
tracklists = {f"e{i}": SINGLE for i in range(10)}
|
||||
_install_fake_apis(monkeypatch, artist_albums=artist_albums, tracklists=tracklists)
|
||||
out = default_fetch_alternates(
|
||||
"spotify", "e0", artist_id="a", album_title="Scatterbrain", max_editions=6,
|
||||
)
|
||||
assert len(out) == 6
|
||||
|
|
@ -193,3 +193,103 @@ def test_score_is_rounded():
|
|||
source_priority=PRIORITY,
|
||||
)
|
||||
assert out["score"] == round(out["score"], 4)
|
||||
|
||||
|
||||
# ── #767-2: expand to alternate editions when the linked one clearly misfits ──
|
||||
#
|
||||
# The reported bug: a 1-track single was enriched against the *deluxe* album, so
|
||||
# every linked source ID points at the 10-track deluxe. The single is never a
|
||||
# candidate, so it can never be chosen — and the deluxe scores so badly (0.1,
|
||||
# below the 0.5 floor) that nothing gets pinned and the organizer falls back to
|
||||
# the stored deluxe ID. The fix: when no LINKED edition clears the floor, fetch
|
||||
# the source's other editions of the same release and score those too.
|
||||
|
||||
SINGLE_FILE = [{"duration_ms": 129_000, "title": "Scatterbrain"}] # owns the 2:09 single
|
||||
# 10-track deluxe; "Scatterbrain" sits at track 2 (2:10, within ±3s of the file).
|
||||
DELUXE_10 = (
|
||||
[{"duration_ms": 200_000, "title": "Intro"}]
|
||||
+ [{"duration_ms": 130_000, "title": "Scatterbrain"}]
|
||||
+ [{"duration_ms": 180_000 + i * 5_000, "title": f"Deluxe {i+1}"} for i in range(8)]
|
||||
)
|
||||
SINGLE_RELEASE = [{"duration_ms": 129_000, "title": "Scatterbrain", "track_number": 1}]
|
||||
|
||||
|
||||
def _alternates(table):
|
||||
"""fetch_alternates backed by a {(source, album_id): [release, ...]} table.
|
||||
Each release is {'album_id', 'tracks'} — a sibling edition from that source."""
|
||||
def fetch(source, album_id):
|
||||
return table.get((source, album_id)) or []
|
||||
return fetch
|
||||
|
||||
|
||||
def test_expands_to_alternate_edition_when_linked_misfits():
|
||||
# Only linked id is the deluxe; user actually owns the single.
|
||||
alt_table = {
|
||||
("spotify", "sp_deluxe"): [
|
||||
{"album_id": "sp_single", "tracks": SINGLE_RELEASE},
|
||||
{"album_id": "sp_deluxe", "tracks": DELUXE_10}, # the edition we already have
|
||||
],
|
||||
}
|
||||
out = resolve_canonical_for_album(
|
||||
album_source_ids={"spotify": "sp_deluxe"},
|
||||
file_tracks=list(SINGLE_FILE),
|
||||
fetch_tracklist=_fetcher({("spotify", "sp_deluxe"): DELUXE_10}),
|
||||
fetch_alternates=_alternates(alt_table),
|
||||
source_priority=PRIORITY,
|
||||
)
|
||||
assert out is not None, "deluxe alone scores 0.1 (below floor) — must expand"
|
||||
assert out["source"] == "spotify" and out["album_id"] == "sp_single"
|
||||
assert out["score"] > 0.9
|
||||
|
||||
|
||||
def test_does_not_expand_when_linked_edition_fits():
|
||||
# Linked edition fits the files (score ~1.0) -> NEVER fetch alternates (cost
|
||||
# guard + zero behaviour change for the 95% common case).
|
||||
calls = []
|
||||
|
||||
def spy(source, album_id):
|
||||
calls.append((source, album_id))
|
||||
return [{"album_id": "sp_other", "tracks": DELUXE_10}]
|
||||
|
||||
out = resolve_canonical_for_album(
|
||||
album_source_ids={"spotify": "sp_std"},
|
||||
file_tracks=list(STD),
|
||||
fetch_tracklist=_fetcher({("spotify", "sp_std"): STD}),
|
||||
fetch_alternates=spy,
|
||||
source_priority=PRIORITY,
|
||||
)
|
||||
assert out["source"] == "spotify" and out["album_id"] == "sp_std"
|
||||
assert calls == [], "must not fetch alternates when the linked edition already fits"
|
||||
|
||||
|
||||
def test_expansion_dedupes_alternates_against_linked():
|
||||
# The alternates list re-offers the linked deluxe; it must not be double-scored
|
||||
# and must not crash. The exact-fit single still wins.
|
||||
alt_table = {
|
||||
("spotify", "sp_deluxe"): [
|
||||
{"album_id": "sp_deluxe", "tracks": DELUXE_10}, # dup of the linked one
|
||||
{"album_id": "sp_single", "tracks": SINGLE_RELEASE},
|
||||
],
|
||||
}
|
||||
out = resolve_canonical_for_album(
|
||||
album_source_ids={"spotify": "sp_deluxe"},
|
||||
file_tracks=list(SINGLE_FILE),
|
||||
fetch_tracklist=_fetcher({("spotify", "sp_deluxe"): DELUXE_10}),
|
||||
fetch_alternates=_alternates(alt_table),
|
||||
source_priority=PRIORITY,
|
||||
)
|
||||
assert out["album_id"] == "sp_single"
|
||||
ids = [c["album_id"] for c in out["candidates"]]
|
||||
assert ids.count("sp_deluxe") == 1, "linked edition must be scored exactly once"
|
||||
|
||||
|
||||
def test_no_expansion_when_alternates_not_provided():
|
||||
# Backward-compat: without a fetch_alternates callable, behaviour is exactly
|
||||
# as before — the misfit deluxe stays unresolved (None).
|
||||
out = resolve_canonical_for_album(
|
||||
album_source_ids={"spotify": "sp_deluxe"},
|
||||
file_tracks=list(SINGLE_FILE),
|
||||
fetch_tracklist=_fetcher({("spotify", "sp_deluxe"): DELUXE_10}),
|
||||
source_priority=PRIORITY,
|
||||
)
|
||||
assert out is None
|
||||
|
|
|
|||
104
tests/test_context_qualifier_match.py
Normal file
104
tests/test_context_qualifier_match.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
"""#808: parenthetical qualifiers that restate album context must not block
|
||||
library-presence matching.
|
||||
|
||||
carlosjfcasero's case: the wishlist held 'Champagne Supernova (OurVinyl
|
||||
Sessions)' (Deezer/iTunes title) while the library track was on the album
|
||||
'Champagne Supernova (OurVinyl Sessions)'. When one side's title carries the
|
||||
qualifier and the other doesn't, the length-ratio penalty crushed the pair to
|
||||
~0.17 — wishlist cleanup never recognised the owned edition and the track
|
||||
re-appeared every cycle. The qualifier appearing in the (db) album title
|
||||
proves it's album context, not a different version.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from core.text.title_match import strip_redundant_context_qualifiers
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
|
||||
# ── the pure helper ──────────────────────────────────────────────────────────
|
||||
|
||||
def test_qualifier_confirmed_by_album_is_stripped():
|
||||
out = strip_redundant_context_qualifiers(
|
||||
'champagne supernova (ourvinyl sessions)',
|
||||
'champagne supernova (ourvinyl sessions)', # db album title
|
||||
)
|
||||
assert out == 'champagne supernova'
|
||||
|
||||
|
||||
def test_version_marker_on_unrelated_album_is_kept():
|
||||
assert strip_redundant_context_qualifiers('song (live)', 'studio album') == 'song (live)'
|
||||
assert strip_redundant_context_qualifiers('song (remix)', 'the album') == 'song (remix)'
|
||||
|
||||
|
||||
def test_version_marker_confirmed_by_album_is_stripped():
|
||||
# Owning 'Song (Live)' on the album 'Live at Wembley' IS owning that cut.
|
||||
assert strip_redundant_context_qualifiers('song (live)', 'live at wembley') == 'song'
|
||||
|
||||
|
||||
def test_word_boundary_containment():
|
||||
# 'live' inside 'alive' must NOT count as context confirmation.
|
||||
assert strip_redundant_context_qualifiers('song (live)', 'alive and well') == 'song (live)'
|
||||
|
||||
|
||||
def test_no_context_or_title_untouched():
|
||||
assert strip_redundant_context_qualifiers('plain title', 'anything') == 'plain title'
|
||||
assert strip_redundant_context_qualifiers('', 'ctx') == ''
|
||||
assert strip_redundant_context_qualifiers('song (x)') == 'song (x)'
|
||||
|
||||
|
||||
# ── end to end through check_track_exists (the wishlist-cleanup contract) ────
|
||||
|
||||
@pytest.fixture()
|
||||
def lib_db(tmp_path):
|
||||
db = MusicDatabase(str(tmp_path / 'm.db'))
|
||||
conn = db._get_connection()
|
||||
c = conn.cursor()
|
||||
c.execute("INSERT INTO artists (id, name, server_source) VALUES ('a1', 'Jillette Johnson', 'plex')")
|
||||
c.execute("""INSERT INTO albums (id, title, artist_id, server_source)
|
||||
VALUES ('al1', 'Champagne Supernova (OurVinyl Sessions)', 'a1', 'plex')""")
|
||||
c.execute("""INSERT INTO tracks (id, album_id, artist_id, title, file_path, server_source)
|
||||
VALUES ('t1', 'al1', 'a1', 'Champagne Supernova', '/m/cs.mp3', 'plex')""")
|
||||
# Version-safety control: a live cut on a studio-named album.
|
||||
c.execute("""INSERT INTO albums (id, title, artist_id, server_source)
|
||||
VALUES ('al2', 'Water In A Whale', 'a1', 'plex')""")
|
||||
c.execute("""INSERT INTO tracks (id, album_id, artist_id, title, file_path, server_source)
|
||||
VALUES ('t2', 'al2', 'a1', 'Cameron', '/m/c.mp3', 'plex')""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return db
|
||||
|
||||
|
||||
def test_808_qualified_search_matches_bare_library_track(lib_db):
|
||||
"""The reported direction: source/wishlist title carries the qualifier,
|
||||
library title is bare, the library ALBUM carries the qualifier."""
|
||||
match, conf = lib_db.check_track_exists(
|
||||
'Champagne Supernova (OurVinyl Sessions)', 'Jillette Johnson',
|
||||
confidence_threshold=0.7, server_source='plex',
|
||||
album='Jillette Johnson | OurVinyl Sessions',
|
||||
)
|
||||
assert match is not None and conf >= 0.7
|
||||
|
||||
|
||||
def test_version_marker_still_blocks_without_album_confirmation(lib_db):
|
||||
"""'Cameron (Live)' must NOT match the studio 'Cameron' — the qualifier
|
||||
appears in no album context, so the mismatch penalty stands."""
|
||||
match, conf = lib_db.check_track_exists(
|
||||
'Cameron (Live)', 'Jillette Johnson',
|
||||
confidence_threshold=0.7, server_source='plex',
|
||||
)
|
||||
assert conf < 0.7
|
||||
|
||||
|
||||
def test_different_song_prefix_still_blocked(lib_db):
|
||||
"""'Champagne' alone is a different (hypothetical) song — the length
|
||||
penalty on the reduced forms still applies."""
|
||||
match, conf = lib_db.check_track_exists(
|
||||
'Champagne', 'Jillette Johnson',
|
||||
confidence_threshold=0.7, server_source='plex',
|
||||
)
|
||||
assert conf < 0.7
|
||||
130
tests/test_cover_art_targets.py
Normal file
130
tests/test_cover_art_targets.py
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
"""Per-target cover-art apply (Pache711: 'select one or the other to fix').
|
||||
|
||||
A missing-cover-art finding now offers album art AND artist art as
|
||||
independently applyable targets. _fix_missing_cover_art routes on _fix_action:
|
||||
'album' (default), 'artist', or 'both'. Verified against a real SQLite DB so
|
||||
the UPDATE statements are exercised.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
|
||||
if "spotipy" not in sys.modules:
|
||||
spotipy = types.ModuleType("spotipy")
|
||||
spotipy.Spotify = type("S", (), {})
|
||||
oauth2 = types.ModuleType("spotipy.oauth2")
|
||||
oauth2.SpotifyOAuth = oauth2.SpotifyClientCredentials = type("O", (), {})
|
||||
spotipy.oauth2 = oauth2
|
||||
sys.modules["spotipy"] = spotipy
|
||||
sys.modules["spotipy.oauth2"] = oauth2
|
||||
|
||||
if "config.settings" not in sys.modules:
|
||||
config_pkg = types.ModuleType("config")
|
||||
settings_mod = types.ModuleType("config.settings")
|
||||
|
||||
class _Cfg:
|
||||
def get(self, key, default=None):
|
||||
return default
|
||||
|
||||
def get_active_media_server(self):
|
||||
return "plex"
|
||||
|
||||
settings_mod.config_manager = _Cfg()
|
||||
config_pkg.settings = settings_mod
|
||||
sys.modules["config"] = config_pkg
|
||||
sys.modules["config.settings"] = settings_mod
|
||||
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from core.repair_worker import RepairWorker
|
||||
|
||||
|
||||
class _DB:
|
||||
def __init__(self, path):
|
||||
self.path = str(path)
|
||||
conn = self._get_connection()
|
||||
c = conn.cursor()
|
||||
c.execute("CREATE TABLE artists (id TEXT PRIMARY KEY, name TEXT, thumb_url TEXT, updated_at TEXT)")
|
||||
c.execute("CREATE TABLE albums (id TEXT PRIMARY KEY, title TEXT, artist_id TEXT, thumb_url TEXT, musicbrainz_release_id TEXT, updated_at TEXT)")
|
||||
c.execute("CREATE TABLE tracks (id TEXT PRIMARY KEY, album_id TEXT, file_path TEXT)")
|
||||
c.execute("INSERT INTO artists VALUES ('ar1', 'Forre Sterra', 'http://old/artist.jpg', NULL)")
|
||||
c.execute("INSERT INTO albums VALUES ('al1', 'For You', 'ar1', NULL, NULL, NULL)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def _get_connection(self):
|
||||
return sqlite3.connect(self.path)
|
||||
|
||||
|
||||
def _worker(tmp_path):
|
||||
w = RepairWorker.__new__(RepairWorker)
|
||||
w.db = _DB(tmp_path / "m.db")
|
||||
w.transfer_folder = str(tmp_path)
|
||||
w._config_manager = None
|
||||
return w
|
||||
|
||||
|
||||
def _thumbs(w):
|
||||
conn = w.db._get_connection()
|
||||
c = conn.cursor()
|
||||
alb = c.execute("SELECT thumb_url FROM albums WHERE id='al1'").fetchone()[0]
|
||||
art = c.execute("SELECT thumb_url FROM artists WHERE id='ar1'").fetchone()[0]
|
||||
conn.close()
|
||||
return alb, art
|
||||
|
||||
|
||||
DETAILS = {
|
||||
'album_id': 'al1', 'album_title': 'For You', 'artist': 'Forre Sterra',
|
||||
'found_artwork_url': 'http://new/album.jpg',
|
||||
'found_artist_url': 'http://new/artist.jpg',
|
||||
}
|
||||
|
||||
|
||||
def test_artist_only_sets_artist_leaves_album(tmp_path):
|
||||
w = _worker(tmp_path)
|
||||
res = w._fix_missing_cover_art('album', 'al1', None, {**DETAILS, '_fix_action': 'artist'})
|
||||
assert res['success'] and res['action'] == 'applied_artist_art'
|
||||
album_thumb, artist_thumb = _thumbs(w)
|
||||
assert artist_thumb == 'http://new/artist.jpg' # artist updated
|
||||
assert album_thumb is None # album untouched
|
||||
|
||||
|
||||
def test_album_only_sets_album_leaves_artist(tmp_path):
|
||||
w = _worker(tmp_path)
|
||||
res = w._fix_missing_cover_art('album', 'al1', None, {**DETAILS, '_fix_action': 'album'})
|
||||
assert res['success']
|
||||
album_thumb, artist_thumb = _thumbs(w)
|
||||
assert album_thumb == 'http://new/album.jpg' # album updated
|
||||
assert artist_thumb == 'http://old/artist.jpg' # artist left as-is
|
||||
|
||||
|
||||
def test_default_action_is_album_only(tmp_path):
|
||||
# No _fix_action → behaves exactly like the old "Apply Art" (album only).
|
||||
w = _worker(tmp_path)
|
||||
w._fix_missing_cover_art('album', 'al1', None, dict(DETAILS))
|
||||
album_thumb, artist_thumb = _thumbs(w)
|
||||
assert album_thumb == 'http://new/album.jpg'
|
||||
assert artist_thumb == 'http://old/artist.jpg'
|
||||
|
||||
|
||||
def test_both_sets_album_and_artist(tmp_path):
|
||||
w = _worker(tmp_path)
|
||||
res = w._fix_missing_cover_art('album', 'al1', None, {**DETAILS, '_fix_action': 'both'})
|
||||
assert res['success']
|
||||
album_thumb, artist_thumb = _thumbs(w)
|
||||
assert album_thumb == 'http://new/album.jpg'
|
||||
assert artist_thumb == 'http://new/artist.jpg'
|
||||
assert 'artist image' in res['message']
|
||||
|
||||
|
||||
def test_artist_action_without_found_artist_url_fails_cleanly(tmp_path):
|
||||
w = _worker(tmp_path)
|
||||
res = w._fix_missing_cover_art('album', 'al1', None,
|
||||
{**DETAILS, 'found_artist_url': None, '_fix_action': 'artist'})
|
||||
assert res['success'] is False
|
||||
album_thumb, artist_thumb = _thumbs(w)
|
||||
assert artist_thumb == 'http://old/artist.jpg' # nothing changed
|
||||
116
tests/test_download_origins.py
Normal file
116
tests/test_download_origins.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
"""Download-origin provenance: the deriver + the library_history persistence.
|
||||
|
||||
Feature: the origin-history modal (watchlist page / sync page) lists which
|
||||
downloads were triggered by a watchlist scan vs a playlist sync, and lets the
|
||||
user delete them. The trigger is derived once at the import chokepoint and
|
||||
stored on the library_history row.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from core.downloads.origin import derive_download_origin
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
|
||||
# ── deriver ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_explicit_stamp_wins():
|
||||
ctx = {'track_info': {
|
||||
'_dl_origin': 'playlist', '_dl_origin_context': 'Discover Weekly',
|
||||
'source_info': {'watchlist_artist_name': 'Drake'}, # would say watchlist
|
||||
}}
|
||||
assert derive_download_origin(ctx) == ('playlist', 'Discover Weekly')
|
||||
|
||||
|
||||
def test_watchlist_provenance_from_wishlist_source_info():
|
||||
# The exact shape watchlist_scanner writes into the wishlist row, which
|
||||
# rides into track_info when the wishlist worker downloads the item.
|
||||
ctx = {'track_info': {'source_info': {
|
||||
'watchlist_artist_name': 'Kendrick Lamar',
|
||||
'watchlist_artist_id': 'spot123',
|
||||
'album_name': 'GNX',
|
||||
}}}
|
||||
assert derive_download_origin(ctx) == ('watchlist', 'Kendrick Lamar')
|
||||
|
||||
|
||||
def test_playlist_provenance_from_source_info_and_json_string():
|
||||
ctx = {'track_info': {'source_info': {'playlist_name': 'Release Radar'}}}
|
||||
assert derive_download_origin(ctx) == ('playlist', 'Release Radar')
|
||||
# source_info sometimes survives as a JSON string — parse it.
|
||||
ctx2 = {'track_info': {'source_info': json.dumps({'playlist_name': 'RapCaviar'})}}
|
||||
assert derive_download_origin(ctx2) == ('playlist', 'RapCaviar')
|
||||
|
||||
|
||||
def test_playlist_folder_mode_thread():
|
||||
ctx = {'track_info': {'_playlist_name': 'Today’s Top Hits'}}
|
||||
assert derive_download_origin(ctx) == ('playlist', 'Today’s Top Hits')
|
||||
|
||||
|
||||
def test_manual_and_garbage_derive_none():
|
||||
assert derive_download_origin({'track_info': {'name': 'Song'}}) == (None, '')
|
||||
assert derive_download_origin({}) == (None, '')
|
||||
assert derive_download_origin({'track_info': 'not-a-dict'}) == (None, '')
|
||||
# invalid explicit origin is ignored, not trusted
|
||||
assert derive_download_origin({'track_info': {'_dl_origin': 'aliens'}}) == (None, '')
|
||||
|
||||
|
||||
# ── persistence ──────────────────────────────────────────────────────────────
|
||||
|
||||
def _seed(db):
|
||||
db.add_library_history_entry(
|
||||
event_type='download', title='Squabble Up', artist_name='Kendrick Lamar',
|
||||
album_name='GNX', file_path='/music/k/squabble.flac',
|
||||
origin='watchlist', origin_context='Kendrick Lamar')
|
||||
db.add_library_history_entry(
|
||||
event_type='download', title='Opalite', artist_name='Taylor Swift',
|
||||
album_name='Showgirl', file_path='/music/t/opalite.flac',
|
||||
origin='playlist', origin_context='Release Radar')
|
||||
db.add_library_history_entry( # manual download — no origin
|
||||
event_type='download', title='Random', artist_name='Someone',
|
||||
file_path='/music/r/random.flac')
|
||||
|
||||
|
||||
def test_origin_entries_filtered_and_counted(tmp_path):
|
||||
db = MusicDatabase(str(tmp_path / 'm.db'))
|
||||
_seed(db)
|
||||
|
||||
wl, wl_total = db.get_download_origin_entries('watchlist')
|
||||
pl, pl_total = db.get_download_origin_entries('playlist')
|
||||
|
||||
assert wl_total == 1 and wl[0]['title'] == 'Squabble Up'
|
||||
assert wl[0]['origin_context'] == 'Kendrick Lamar'
|
||||
assert pl_total == 1 and pl[0]['title'] == 'Opalite'
|
||||
assert pl[0]['origin_context'] == 'Release Radar'
|
||||
|
||||
|
||||
def test_history_rows_fetch_and_delete(tmp_path):
|
||||
db = MusicDatabase(str(tmp_path / 'm.db'))
|
||||
_seed(db)
|
||||
entries, _ = db.get_download_origin_entries('watchlist')
|
||||
ids = [e['id'] for e in entries]
|
||||
|
||||
rows = db.get_library_history_rows_by_ids(ids)
|
||||
assert rows and rows[0]['file_path'] == '/music/k/squabble.flac'
|
||||
|
||||
assert db.delete_library_history_rows(ids) == 1
|
||||
assert db.get_download_origin_entries('watchlist')[1] == 0
|
||||
# the other origin untouched
|
||||
assert db.get_download_origin_entries('playlist')[1] == 1
|
||||
|
||||
|
||||
def test_delete_track_by_file_path(tmp_path):
|
||||
db = MusicDatabase(str(tmp_path / 'm.db'))
|
||||
conn = db._get_connection()
|
||||
cur = conn.cursor()
|
||||
cur.execute("INSERT INTO artists (id, name) VALUES ('a1', 'A')")
|
||||
cur.execute("INSERT INTO albums (id, title, artist_id) VALUES ('al1', 'Al', 'a1')")
|
||||
cur.execute("""INSERT INTO tracks (id, album_id, artist_id, title, file_path)
|
||||
VALUES ('t1', 'al1', 'a1', 'Song', '/music/k/squabble.flac')""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
assert db.delete_track_by_file_path('/music/k/squabble.flac') == 1
|
||||
assert db.delete_track_by_file_path('/music/k/squabble.flac') == 0
|
||||
assert db.delete_track_by_file_path('') == 0
|
||||
145
tests/test_expired_download_cleaner.py
Normal file
145
tests/test_expired_download_cleaner.py
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
"""Expired Download Cleaner job: scan protection + findings vs auto-delete,
|
||||
and the shared delete helper.
|
||||
|
||||
The pure expiry logic is tested in tests/library/test_expired_cleanup.py; this
|
||||
covers the job's fact-gathering (play_count, active-mirror/watch protection)
|
||||
and the two modes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from core.repair_jobs.expired_download_cleaner import (
|
||||
ExpiredDownloadCleanerJob,
|
||||
delete_origin_download,
|
||||
)
|
||||
|
||||
OLD = (datetime.now(timezone.utc) - timedelta(days=120)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
NEW = (datetime.now(timezone.utc) - timedelta(days=2)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
class _DB:
|
||||
def __init__(self, candidates, mirrored=None, watched=None):
|
||||
self._candidates = candidates
|
||||
self._mirrored = mirrored or []
|
||||
self._watched = watched or []
|
||||
self.deleted_paths = []
|
||||
self.deleted_history = []
|
||||
|
||||
def get_origin_cleanup_candidates(self):
|
||||
return [dict(c) for c in self._candidates]
|
||||
|
||||
def get_mirrored_playlists(self, profile_id=1):
|
||||
return [{'name': n} for n in self._mirrored]
|
||||
|
||||
def get_watchlist_artists(self, profile_id=1):
|
||||
return [SimpleNamespace(artist_name=n) for n in self._watched]
|
||||
|
||||
def delete_track_by_file_path(self, p):
|
||||
self.deleted_paths.append(p)
|
||||
return 1
|
||||
|
||||
def delete_library_history_rows(self, ids):
|
||||
self.deleted_history.extend(ids)
|
||||
return len(ids)
|
||||
|
||||
|
||||
def _ctx(db, settings, findings):
|
||||
return SimpleNamespace(
|
||||
db=db,
|
||||
config_manager=SimpleNamespace(get=lambda k, d=None: settings if k.endswith('.settings') else d),
|
||||
check_stop=lambda: False, wait_if_paused=lambda: False,
|
||||
update_progress=lambda *a, **k: None, report_progress=lambda *a, **k: None,
|
||||
create_finding=lambda **kw: (findings.append(kw) or True),
|
||||
)
|
||||
|
||||
|
||||
def _cand(eid, origin="playlist", created=OLD, play_count=0, ctx="Some Playlist", path=None):
|
||||
return {"id": eid, "origin": origin, "origin_context": ctx, "created_at": created,
|
||||
"file_path": path or f"/music/{eid}.flac", "title": f"T{eid}",
|
||||
"artist_name": "Artist", "play_count": play_count}
|
||||
|
||||
|
||||
# ── scan: findings mode + protections ────────────────────────────────────────
|
||||
|
||||
def test_scan_noop_when_both_retentions_off():
|
||||
db = _DB([_cand(1)])
|
||||
findings = []
|
||||
res = ExpiredDownloadCleanerJob().scan(_ctx(db, {}, findings)) # defaults: both off
|
||||
assert res.findings_created == 0 and findings == []
|
||||
|
||||
|
||||
def test_scan_creates_findings_for_expired():
|
||||
db = _DB([
|
||||
_cand(1, created=OLD, play_count=0), # expired
|
||||
_cand(2, created=NEW, play_count=0), # too new
|
||||
_cand(3, created=OLD, play_count=5), # listened → keep
|
||||
])
|
||||
findings = []
|
||||
res = ExpiredDownloadCleanerJob().scan(_ctx(
|
||||
db, {'playlist_retention': '2mo', 'keep_if_played_at_least': 2}, findings))
|
||||
assert res.findings_created == 1
|
||||
assert findings[0]['details']['history_id'] == 1
|
||||
assert findings[0]['finding_type'] == 'expired_download'
|
||||
|
||||
|
||||
def test_scan_protects_actively_mirrored_playlist():
|
||||
db = _DB([_cand(1, origin="playlist", ctx="My Mix", created=OLD)],
|
||||
mirrored=["My Mix"])
|
||||
findings = []
|
||||
ExpiredDownloadCleanerJob().scan(_ctx(db, {'playlist_retention': '1w'}, findings))
|
||||
assert findings == [] # still mirrored → protected
|
||||
|
||||
|
||||
def test_scan_protects_watched_artist():
|
||||
db = _DB([_cand(1, origin="watchlist", ctx="Drake", created=OLD)],
|
||||
watched=["Drake"])
|
||||
findings = []
|
||||
ExpiredDownloadCleanerJob().scan(_ctx(db, {'watchlist_retention': '1w'}, findings))
|
||||
assert findings == [] # still watched → protected
|
||||
|
||||
|
||||
def test_scan_dry_run_default_is_findings_only():
|
||||
# No dry_run in settings → defaults to True → findings, never deletes.
|
||||
db = _DB([_cand(1, created=OLD, path="/music/x.flac")])
|
||||
findings = []
|
||||
res = ExpiredDownloadCleanerJob().scan(_ctx(db, {'playlist_retention': '2mo'}, findings))
|
||||
assert res.findings_created == 1 and db.deleted_history == [] # nothing deleted
|
||||
|
||||
|
||||
def test_scan_auto_delete_when_dry_run_off():
|
||||
db = _DB([_cand(1, created=OLD, path="/music/x.flac")])
|
||||
findings = []
|
||||
res = ExpiredDownloadCleanerJob().scan(_ctx(
|
||||
db, {'playlist_retention': '2mo', 'dry_run': False}, findings))
|
||||
assert findings == [] # no findings in auto mode
|
||||
assert res.auto_fixed == 1
|
||||
assert 1 in db.deleted_history # history row removed
|
||||
assert "/music/x.flac" in db.deleted_paths # track row removed
|
||||
|
||||
|
||||
# ── delete helper ────────────────────────────────────────────────────────────
|
||||
|
||||
def test_delete_origin_download_missing_file(tmp_path):
|
||||
# File doesn't exist → still cleans up the history row (orphan), no error.
|
||||
db = _DB([])
|
||||
entry = {"id": 9, "file_path": str(tmp_path / "gone.flac")}
|
||||
cfg = SimpleNamespace(get=lambda k, d=None: d)
|
||||
res = delete_origin_download(db, entry, cfg)
|
||||
assert res["error"] is None and res["file_deleted"] is False
|
||||
assert db.deleted_history == [9]
|
||||
|
||||
|
||||
def test_delete_origin_download_removes_real_file(tmp_path):
|
||||
f = tmp_path / "song.flac"; f.write_bytes(b"x")
|
||||
db = _DB([])
|
||||
entry = {"id": 5, "file_path": str(f)}
|
||||
cfg = SimpleNamespace(get=lambda k, d=None: d)
|
||||
res = delete_origin_download(db, entry, cfg)
|
||||
assert res["file_deleted"] is True and not f.exists()
|
||||
assert db.deleted_history == [5]
|
||||
77
tests/test_genius_backoff.py
Normal file
77
tests/test_genius_backoff.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
"""Genius 429 backoff must be a fail-fast gate, never a sleep.
|
||||
|
||||
The old wrapper slept the backoff (30-120s) in the calling thread — while
|
||||
holding the global API lock, serializing every other Genius caller behind
|
||||
it — and then re-raised anyway. The import pipeline measurably napped
|
||||
2x120s per track ("Genius track lookup took 242.4s") for lookups that
|
||||
still failed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
import core.genius_client as gc
|
||||
|
||||
|
||||
def _fresh(monkeypatch):
|
||||
monkeypatch.setattr(gc, '_rate_limit_until', 0)
|
||||
monkeypatch.setattr(gc, '_rate_limit_backoff', 0)
|
||||
monkeypatch.setattr(gc, '_last_api_call_time', 0)
|
||||
|
||||
|
||||
def test_backoff_window_fails_fast_without_sleeping(monkeypatch):
|
||||
_fresh(monkeypatch)
|
||||
monkeypatch.setattr(gc, '_rate_limit_until', time.time() + 120)
|
||||
|
||||
@gc.rate_limited
|
||||
def call():
|
||||
raise AssertionError('must not reach the API during a backoff window')
|
||||
|
||||
started = time.time()
|
||||
with pytest.raises(gc.GeniusRateLimitedError):
|
||||
call()
|
||||
assert time.time() - started < 0.5 # the old code slept the full window here
|
||||
|
||||
|
||||
def test_429_opens_the_gate_without_sleeping_and_escalates(monkeypatch):
|
||||
_fresh(monkeypatch)
|
||||
|
||||
@gc.rate_limited
|
||||
def call():
|
||||
raise requests.exceptions.HTTPError('429 Client Error: Too Many Requests')
|
||||
|
||||
started = time.time()
|
||||
with pytest.raises(requests.exceptions.HTTPError):
|
||||
call()
|
||||
assert time.time() - started < 0.5 # old code slept 30s+ here
|
||||
assert gc._rate_limit_until > time.time() # the gate is open
|
||||
assert gc._rate_limit_backoff == 30
|
||||
|
||||
# Next 429 (after the window expires) doubles the gate: 30 -> 60
|
||||
monkeypatch.setattr(gc, '_rate_limit_until', 0)
|
||||
monkeypatch.setattr(gc, '_last_api_call_time', 0)
|
||||
with pytest.raises(requests.exceptions.HTTPError):
|
||||
call()
|
||||
assert gc._rate_limit_backoff == 60
|
||||
|
||||
|
||||
def test_rate_limited_error_is_a_request_exception():
|
||||
# The design hinge: existing callers (import source lookups, worker item
|
||||
# guards) catch RequestException and skip — no call-site changes needed.
|
||||
assert issubclass(gc.GeniusRateLimitedError, requests.RequestException)
|
||||
|
||||
|
||||
def test_success_decays_backoff(monkeypatch):
|
||||
_fresh(monkeypatch)
|
||||
monkeypatch.setattr(gc, '_rate_limit_backoff', 30)
|
||||
|
||||
@gc.rate_limited
|
||||
def call():
|
||||
return 'ok'
|
||||
|
||||
assert call() == 'ok'
|
||||
assert gc._rate_limit_backoff == 25
|
||||
22
tests/test_import_tag_preservation.py
Normal file
22
tests/test_import_tag_preservation.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
"""Regression: a metadata-enhancement failure must NOT wipe a clean/matched
|
||||
import's tags (#804 — already-tagged files were blanked into Unknown Artist).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.imports.tag_policy import should_wipe_tags_on_enhancement_failure
|
||||
|
||||
|
||||
def test_clean_matched_import_is_never_wiped_on_failure():
|
||||
# The #804 case: matched import (clean metadata) → preserve existing tags.
|
||||
assert should_wipe_tags_on_enhancement_failure(has_clean_metadata=True) is False
|
||||
|
||||
|
||||
def test_unmatched_download_still_strips_junk_on_failure():
|
||||
# Unchanged behavior for unmatched downloads (likely junk source tags).
|
||||
assert should_wipe_tags_on_enhancement_failure(has_clean_metadata=False) is True
|
||||
|
||||
|
||||
def test_falsey_values_treated_as_unmatched():
|
||||
assert should_wipe_tags_on_enhancement_failure(None) is True
|
||||
assert should_wipe_tags_on_enhancement_failure(0) is True
|
||||
|
|
@ -284,3 +284,113 @@ def test_fix_library_retag_counts_unreachable(tmp_path, monkeypatch):
|
|||
'cover_action': None, 'cover_url': None}
|
||||
res = worker._fix_library_retag('album', '1', None, details)
|
||||
assert res['success'] is False # nothing written (file missing)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cover-art scans on path-mapped setups (the "(0 track(s))" / "No tracks to
|
||||
# re-tag in finding" report): the scan must resolve DB paths the same way the
|
||||
# apply handler does, never emit an empty finding, and give unmatched tracks
|
||||
# the album art.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_scan_resolves_mapped_paths_instead_of_skipping(tmp_path, monkeypatch):
|
||||
"""DB stores a container path the scan process can't see directly; the
|
||||
resolver maps it to the real file. Before the fix the bare isfile() check
|
||||
dropped every track and cover-mode scans produced unappliable 0-track
|
||||
findings."""
|
||||
real = tmp_path / 'track.flac'; real.write_bytes(b'')
|
||||
raw = '/container/music/track.flac' # not a real path here
|
||||
conn = _db_with_album(str(tmp_path / 'm.db'), raw, current_title='Old Title')
|
||||
ctx = _context(conn, {'mode': 'overwrite', 'cover_art': 'replace', 'source': 'spotify'})
|
||||
_patch_source(monkeypatch, {
|
||||
'title': 'Old Title', 'album_artist': 'Real Artist', 'album': 'Real Album',
|
||||
'year': '2021', 'genre': 'Rock', 'track_number': 1, 'disc_number': 1,
|
||||
})
|
||||
monkeypatch.setattr(lr, 'resolve_library_file_path',
|
||||
lambda p, **k: str(real) if p == raw else None)
|
||||
|
||||
result = lr.LibraryRetagJob().scan(ctx)
|
||||
|
||||
assert result.findings_created == 1
|
||||
tracks = ctx.findings[0]['details']['tracks']
|
||||
assert len(tracks) == 1
|
||||
assert tracks[0]['file_path'] == str(real) # plan carries the RESOLVED path
|
||||
|
||||
|
||||
def test_cover_scan_with_no_reachable_tracks_creates_no_finding(tmp_path, monkeypatch):
|
||||
"""Cover action set but no track resolvable: skip the album entirely.
|
||||
The old behavior created a '(0 track(s))' finding whose apply always
|
||||
failed with 'No tracks to re-tag in finding'."""
|
||||
conn = _db_with_album(str(tmp_path / 'm.db'), '/container/music/gone.flac')
|
||||
ctx = _context(conn, {'mode': 'overwrite', 'cover_art': 'replace', 'source': 'spotify'})
|
||||
_patch_source(monkeypatch, {'title': 'Old Title'})
|
||||
monkeypatch.setattr(lr, 'resolve_library_file_path', lambda p, **k: None)
|
||||
|
||||
result = lr.LibraryRetagJob().scan(ctx)
|
||||
|
||||
assert ctx.findings == []
|
||||
assert result.findings_created == 0
|
||||
assert result.skipped == 1
|
||||
|
||||
|
||||
def test_cover_scan_includes_unmatched_tracks_as_art_only(tmp_path, monkeypatch):
|
||||
"""A track with no source match can't be re-tagged, but album cover art
|
||||
still applies to it — cover-mode scans include an art-only plan (empty
|
||||
db_data) and the finding title says 'cover art', not '(0 track(s))'."""
|
||||
track = tmp_path / 'track.flac'; track.write_bytes(b'')
|
||||
conn = _db_with_album(str(tmp_path / 'm.db'), str(track), current_title='Old Title')
|
||||
ctx = _context(conn, {'mode': 'overwrite', 'cover_art': 'replace', 'source': 'spotify'})
|
||||
_patch_source(monkeypatch, {'title': 'Old Title'})
|
||||
# Source tracklist that matches NOTHING in the library.
|
||||
monkeypatch.setattr(lr, 'get_album_tracks_for_source',
|
||||
lambda s, i: [{'name': 'Zzz Unrelated Song', 'track_number': 9,
|
||||
'disc_number': 9, 'id': 'zz'}])
|
||||
|
||||
result = lr.LibraryRetagJob().scan(ctx)
|
||||
|
||||
assert result.findings_created == 1
|
||||
f = ctx.findings[0]
|
||||
assert 'cover art' in f['title']
|
||||
tracks = f['details']['tracks']
|
||||
assert len(tracks) == 1
|
||||
assert not tracks[0]['changes'] and tracks[0]['db_data'] == {} # art-only plan
|
||||
|
||||
|
||||
def test_apply_art_only_plan_embeds_cover(tmp_path, monkeypatch):
|
||||
"""The art-only plans the cover-mode scan now emits (empty db_data) must go
|
||||
through apply_track_plans as a WRITE (cover embed), not a skip/failure."""
|
||||
track = tmp_path / 'track.flac'; track.write_bytes(b'')
|
||||
calls = []
|
||||
monkeypatch.setattr('core.tag_writer.download_cover_art',
|
||||
lambda url: (b'img-bytes', 'image/jpeg'))
|
||||
monkeypatch.setattr('core.tag_writer.write_tags_to_file',
|
||||
lambda fp, db_data, **k: calls.append((fp, db_data, k)) or {'success': True})
|
||||
|
||||
res = lr.apply_track_plans(
|
||||
[{'file_path': str(track), 'db_data': {}}],
|
||||
cover_action='replace', cover_url='http://art/cover.jpg',
|
||||
)
|
||||
|
||||
assert res['written'] == 1 and res['failed'] == 0
|
||||
fp, db_data, kwargs = calls[0]
|
||||
assert db_data == {} and kwargs['embed_cover'] is True
|
||||
assert kwargs['cover_data'] == (b'img-bytes', 'image/jpeg')
|
||||
assert res['cover_written'] is True # cover.jpg written next to the track
|
||||
|
||||
|
||||
def test_apply_art_only_plan_skips_when_cover_download_fails(tmp_path, monkeypatch):
|
||||
"""If the cover can't be downloaded there's nothing to write for an
|
||||
art-only plan — it must count as skipped, never failed."""
|
||||
track = tmp_path / 'track.flac'; track.write_bytes(b'')
|
||||
monkeypatch.setattr('core.tag_writer.download_cover_art',
|
||||
lambda url: (_ for _ in ()).throw(RuntimeError('net down')))
|
||||
monkeypatch.setattr('core.tag_writer.write_tags_to_file',
|
||||
lambda fp, db_data, **k: {'success': True})
|
||||
|
||||
res = lr.apply_track_plans(
|
||||
[{'file_path': str(track), 'db_data': {}}],
|
||||
cover_action='replace', cover_url='http://art/cover.jpg',
|
||||
)
|
||||
|
||||
assert res == {'written': 0, 'failed': 0, 'skipped': 1, 'cover_written': False,
|
||||
'lyrics_written': 0}
|
||||
|
|
|
|||
74
tests/test_library_stream_fallback.py
Normal file
74
tests/test_library_stream_fallback.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""#809: when a library file isn't on SoulSync's disk, play it by proxying the
|
||||
media server's stream API instead of 404-ing.
|
||||
|
||||
Tests the routing helper _build_library_stream_url: Navidrome-only, uses the
|
||||
passed song id or falls back to a DB lookup, returns None otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
web_server = pytest.importorskip("web_server")
|
||||
|
||||
|
||||
class _Client:
|
||||
def build_stream_url(self, song_id, max_bitrate=0):
|
||||
return f"http://nav.example/rest/stream?id={song_id}"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def navidrome(monkeypatch):
|
||||
monkeypatch.setattr(web_server.config_manager, "get_active_media_server", lambda: "navidrome")
|
||||
monkeypatch.setattr(web_server.media_server_engine, "client", lambda name: _Client())
|
||||
|
||||
|
||||
def test_non_navidrome_server_returns_none(monkeypatch):
|
||||
monkeypatch.setattr(web_server.config_manager, "get_active_media_server", lambda: "plex")
|
||||
assert web_server._build_library_stream_url("song1", "/music/x.flac") is None
|
||||
|
||||
|
||||
def test_uses_passed_track_id(navidrome):
|
||||
url = web_server._build_library_stream_url("song-42", "/music/x.flac")
|
||||
assert url == "http://nav.example/rest/stream?id=song-42"
|
||||
|
||||
|
||||
def test_falls_back_to_db_lookup_when_no_id(navidrome, monkeypatch):
|
||||
class _Cur:
|
||||
def execute(self, *a):
|
||||
return self
|
||||
def fetchone(self):
|
||||
return ("song-from-db",)
|
||||
|
||||
class _Conn:
|
||||
def cursor(self):
|
||||
return _Cur()
|
||||
def __enter__(self):
|
||||
return self
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(web_server, "get_database",
|
||||
lambda: type("DB", (), {"_get_connection": lambda self: _Conn()})())
|
||||
url = web_server._build_library_stream_url(None, "/music/x.flac")
|
||||
assert url == "http://nav.example/rest/stream?id=song-from-db"
|
||||
|
||||
|
||||
def test_no_id_no_db_match_returns_none(navidrome, monkeypatch):
|
||||
class _Cur:
|
||||
def execute(self, *a):
|
||||
return self
|
||||
def fetchone(self):
|
||||
return None
|
||||
|
||||
class _Conn:
|
||||
def cursor(self):
|
||||
return _Cur()
|
||||
def __enter__(self):
|
||||
return self
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(web_server, "get_database",
|
||||
lambda: type("DB", (), {"_get_connection": lambda self: _Conn()})())
|
||||
assert web_server._build_library_stream_url(None, "/music/x.flac") is None
|
||||
91
tests/test_liked_songs_playlist.py
Normal file
91
tests/test_liked_songs_playlist.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
"""Liked Songs virtual-playlist resolution.
|
||||
|
||||
wolf39us: the mirrored "Liked" playlist silently failed every refresh with
|
||||
``Error fetching playlist spotify:liked-songs: http status: 400 ... Unsupported
|
||||
URL / URI``. There is no real playlist URI behind a user's liked songs — the
|
||||
web UI invents the virtual id ``spotify:liked-songs`` and Spotify serves the
|
||||
collection via the saved-tracks endpoint. ``get_playlist_by_id`` (what the
|
||||
mirrored refresh path resolves stored ids through) must special-case it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from core.spotify_client import LIKED_SONGS_PLAYLIST_ID, SpotifyClient
|
||||
|
||||
|
||||
def _track(i):
|
||||
return SimpleNamespace(
|
||||
id=f'trk{i}', name=f'Song {i}', artists=['Artist'],
|
||||
album='Album', duration_ms=200_000, image_url=None,
|
||||
popularity=10, external_urls=None, preview_url=None,
|
||||
)
|
||||
|
||||
|
||||
def _client(monkeypatch, saved):
|
||||
client = SpotifyClient.__new__(SpotifyClient)
|
||||
client.sp = MagicMock()
|
||||
monkeypatch.setattr(client, 'is_spotify_authenticated', lambda: True)
|
||||
monkeypatch.setattr(client, 'get_saved_tracks', lambda: list(saved))
|
||||
monkeypatch.setattr(client, 'get_user_info', lambda: {'display_name': 'Wolf'})
|
||||
return client
|
||||
|
||||
|
||||
def test_virtual_liked_songs_id_resolves_from_saved_tracks(monkeypatch):
|
||||
client = _client(monkeypatch, [_track(1), _track(2)])
|
||||
client.sp.playlist.side_effect = AssertionError(
|
||||
'sp.playlist() must not be called for the virtual Liked Songs id')
|
||||
|
||||
pl = client.get_playlist_by_id(LIKED_SONGS_PLAYLIST_ID)
|
||||
|
||||
assert pl is not None
|
||||
assert pl.id == LIKED_SONGS_PLAYLIST_ID
|
||||
assert pl.name == 'Liked Songs' and pl.owner == 'Wolf'
|
||||
assert pl.total_tracks == 2 and len(pl.tracks) == 2
|
||||
client.sp.playlist.assert_not_called()
|
||||
|
||||
|
||||
def test_real_playlist_id_still_uses_playlist_endpoint(monkeypatch):
|
||||
"""Regression guard: normal playlists keep going through sp.playlist()."""
|
||||
client = _client(monkeypatch, [])
|
||||
client.sp.playlist.return_value = {
|
||||
'id': 'pl1', 'name': 'Mix', 'description': '', 'public': True,
|
||||
'collaborative': False, 'owner': {'display_name': 'Wolf'},
|
||||
'tracks': {'total': 0},
|
||||
}
|
||||
monkeypatch.setattr(client, '_get_playlist_tracks', lambda pid: [])
|
||||
|
||||
pl = client.get_playlist_by_id('pl1')
|
||||
|
||||
assert pl is not None and pl.id == 'pl1' and pl.name == 'Mix'
|
||||
client.sp.playlist.assert_called_once_with('pl1')
|
||||
|
||||
|
||||
def test_mirrored_adapter_resolves_liked_songs(monkeypatch):
|
||||
"""The seam that actually failed: SpotifyPlaylistSource.get_playlist — the
|
||||
mirrored-playlist refresh path — with the stored virtual id."""
|
||||
from core.playlists.sources.spotify import SpotifyPlaylistSource
|
||||
|
||||
client = _client(monkeypatch, [_track(1)])
|
||||
client.sp.playlist.side_effect = AssertionError('must not hit the playlist endpoint')
|
||||
src = SpotifyPlaylistSource(lambda: client)
|
||||
|
||||
detail = src.get_playlist(LIKED_SONGS_PLAYLIST_ID)
|
||||
|
||||
assert detail is not None
|
||||
assert detail.meta.name == 'Liked Songs'
|
||||
assert detail.meta.source_playlist_id == LIKED_SONGS_PLAYLIST_ID
|
||||
assert len(detail.tracks) == 1
|
||||
assert detail.tracks[0].track_name == 'Song 1'
|
||||
assert detail.tracks[0].artist_name == 'Artist'
|
||||
|
||||
|
||||
def test_empty_saved_tracks_is_a_failed_refresh_not_an_empty_playlist(monkeypatch):
|
||||
"""get_saved_tracks swallows fetch errors into [] — indistinguishable from
|
||||
'no likes'. A valid-looking EMPTY playlist could make a mirror sync clear
|
||||
the server-side copy, so empty must resolve as a failed refresh (None)."""
|
||||
client = _client(monkeypatch, [])
|
||||
|
||||
assert client.get_playlist_by_id(LIKED_SONGS_PLAYLIST_ID) is None
|
||||
60
tests/test_memory_tracker.py
Normal file
60
tests/test_memory_tracker.py
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
"""Seam tests for the #802 memory-growth diagnostic (core/diagnostics)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import core.diagnostics.memory_tracker as mt
|
||||
|
||||
|
||||
def teardown_function(_fn):
|
||||
# Never leave tracemalloc running across tests — it shadows every
|
||||
# allocation in the process.
|
||||
mt.stop_tracking()
|
||||
|
||||
|
||||
def test_report_without_tracking_is_a_hint_not_an_error():
|
||||
mt.stop_tracking()
|
||||
out = mt.report()
|
||||
assert out['tracking'] is False
|
||||
assert 'start' in out['hint']
|
||||
|
||||
|
||||
def test_start_report_stop_roundtrip_captures_growth():
|
||||
assert mt.start_tracking()['tracking'] is True
|
||||
# Idempotent start
|
||||
assert mt.start_tracking()['already_running'] is True
|
||||
|
||||
# Allocate something measurable after the baseline.
|
||||
# bytearray(1000) allocates at RUNTIME — a constant expression like
|
||||
# 'x' * 1000 gets folded into ONE shared string and traces as ~40KB.
|
||||
hog = [bytearray(1000) for _ in range(5000)] # ~5 MB, genuinely allocated
|
||||
|
||||
out = mt.report(top=10)
|
||||
assert out['tracking'] is True
|
||||
assert out['elapsed_seconds'] is not None
|
||||
assert out['traced_current_mb'] > 0
|
||||
assert isinstance(out['top_growth'], list) and out['top_growth']
|
||||
# The hog must show up as growth attributed to THIS file.
|
||||
top_locations = ' '.join(s['location'] for s in out['top_growth'])
|
||||
assert 'test_memory_tracker.py' in top_locations
|
||||
assert any(s['size_diff_mb'] > 1 for s in out['top_growth'])
|
||||
del hog
|
||||
|
||||
stopped = mt.stop_tracking()
|
||||
assert stopped == {'tracking': False, 'was_tracking': True}
|
||||
assert mt.is_tracking() is False
|
||||
|
||||
|
||||
def test_format_stat_projects_duck_typed_stat():
|
||||
frame = SimpleNamespace(filename='core/foo.py', lineno=42)
|
||||
stat = SimpleNamespace(
|
||||
size=2 * 1024 * 1024, size_diff=1024 * 1024,
|
||||
count=10, count_diff=4,
|
||||
traceback=[frame, SimpleNamespace(filename='core/bar.py', lineno=7)],
|
||||
)
|
||||
out = mt.format_stat(stat)
|
||||
assert out['location'] == 'core/bar.py:7'
|
||||
assert out['trace'] == ['core/foo.py:42', 'core/bar.py:7']
|
||||
assert out['size_mb'] == 2.0 and out['size_diff_mb'] == 1.0
|
||||
assert out['count'] == 10 and out['count_diff'] == 4
|
||||
|
|
@ -189,8 +189,13 @@ def test_missing_cover_art_uses_configured_art_sources(monkeypatch):
|
|||
result = mca.MissingCoverArtJob().scan(context)
|
||||
|
||||
assert result.findings_created == 1
|
||||
# ALBUM art came from the configured order — the album source-priority loop
|
||||
# was skipped (if it had run, the URL would be the fake client's art).
|
||||
assert findings[0]['details']['found_artwork_url'] == 'https://configured/art.jpg'
|
||||
assert consulted == [] # source-priority loop skipped when configured art wins
|
||||
# Artist-art search (Pache711) is a SEPARATE lookup that does consult the
|
||||
# sources; the fake client has no search_artists, so it finds nothing and
|
||||
# no artist target is offered.
|
||||
assert findings[0]['details']['found_artist_url'] is None
|
||||
|
||||
|
||||
def test_missing_cover_art_uses_primary_when_prefer_unset(monkeypatch):
|
||||
|
|
|
|||
241
tests/test_missing_lyrics_job.py
Normal file
241
tests/test_missing_lyrics_job.py
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
"""Missing Lyrics maintenance job + lyrics_client check-only seam (Sokhi).
|
||||
|
||||
Mirrors the Cover Art Filler: scan only flags tracks LRClib actually has
|
||||
lyrics for (Option A — instrumentals never flagged), and applying writes the
|
||||
.lrc via the shared LyricsClient.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from core.lyrics_client import LyricsClient
|
||||
from core.repair_jobs.missing_lyrics import MissingLyricsJob, _has_lrc_sidecar
|
||||
|
||||
|
||||
# ── lyrics_client.has_remote_lyrics (check-only seam) ────────────────────────
|
||||
|
||||
def _client_with_api(api):
|
||||
c = LyricsClient.__new__(LyricsClient)
|
||||
c.api = api
|
||||
return c
|
||||
|
||||
|
||||
def test_has_remote_lyrics_true_when_synced():
|
||||
api = MagicMock()
|
||||
api.get_lyrics.return_value = SimpleNamespace(synced_lyrics="[00:01]hi", plain_lyrics=None)
|
||||
c = _client_with_api(api)
|
||||
assert c.has_remote_lyrics("Song", "Artist", "Album", 200) is True
|
||||
|
||||
|
||||
def test_has_remote_lyrics_true_when_plain_only_via_search():
|
||||
api = MagicMock()
|
||||
api.get_lyrics.return_value = None
|
||||
api.search_lyrics.return_value = [SimpleNamespace(synced_lyrics=None, plain_lyrics="words")]
|
||||
c = _client_with_api(api)
|
||||
assert c.has_remote_lyrics("Song", "Artist") is True
|
||||
|
||||
|
||||
def test_has_remote_lyrics_false_when_none():
|
||||
api = MagicMock()
|
||||
api.get_lyrics.return_value = None
|
||||
api.search_lyrics.return_value = []
|
||||
assert _client_with_api(api).has_remote_lyrics("Instrumental", "Artist") is False
|
||||
|
||||
|
||||
def test_has_remote_lyrics_false_when_no_api():
|
||||
c = LyricsClient.__new__(LyricsClient)
|
||||
c.api = None
|
||||
assert c.has_remote_lyrics("Song", "Artist") is False
|
||||
|
||||
|
||||
# ── sidecar detection ────────────────────────────────────────────────────────
|
||||
|
||||
def test_has_lrc_sidecar(tmp_path):
|
||||
audio = tmp_path / "track.flac"
|
||||
audio.write_bytes(b"x")
|
||||
assert _has_lrc_sidecar(str(audio)) is False
|
||||
(tmp_path / "track.lrc").write_text("[00:01]hi")
|
||||
assert _has_lrc_sidecar(str(audio)) is True
|
||||
|
||||
|
||||
# ── the scan (Option A: only flag fixable tracks) ────────────────────────────
|
||||
|
||||
class _DB:
|
||||
def __init__(self, rows):
|
||||
self._rows = rows
|
||||
|
||||
def _get_connection(self):
|
||||
cur = MagicMock()
|
||||
cur.execute.return_value = None
|
||||
cur.fetchone.return_value = [len(self._rows)]
|
||||
cur.fetchall.return_value = self._rows
|
||||
conn = MagicMock()
|
||||
conn.cursor.return_value = cur
|
||||
return conn
|
||||
|
||||
|
||||
def _ctx(db, findings):
|
||||
return SimpleNamespace(
|
||||
db=db,
|
||||
config_manager=SimpleNamespace(get=lambda k, d=None: d),
|
||||
check_stop=lambda: False, wait_if_paused=lambda: False,
|
||||
update_progress=lambda *a, **k: None, report_progress=lambda *a, **k: None,
|
||||
create_finding=lambda **kw: (findings.append(kw) or True),
|
||||
)
|
||||
|
||||
|
||||
def test_scan_flags_only_tracks_with_available_lyrics(tmp_path, monkeypatch):
|
||||
# Two tracks, neither has a .lrc. LRClib has lyrics for the first, not the second.
|
||||
t1 = tmp_path / "song.flac"; t1.write_bytes(b"x")
|
||||
t2 = tmp_path / "instrumental.flac"; t2.write_bytes(b"x")
|
||||
rows = [
|
||||
(1, "Song", "Artist", "Album", str(t1), 200),
|
||||
(2, "Interlude", "Artist", "Album", str(t2), 60),
|
||||
]
|
||||
fake_client = SimpleNamespace(
|
||||
api=object(),
|
||||
has_remote_lyrics=lambda title, artist, album, dur: title == "Song",
|
||||
)
|
||||
monkeypatch.setattr("core.lyrics_client.lyrics_client", fake_client)
|
||||
|
||||
findings = []
|
||||
result = MissingLyricsJob().scan(_ctx(_DB(rows), findings))
|
||||
|
||||
assert result.findings_created == 1
|
||||
assert findings[0]["entity_type"] == "track"
|
||||
assert findings[0]["finding_type"] == "missing_lyrics"
|
||||
assert findings[0]["details"]["track_title"] == "Song" # the instrumental was skipped
|
||||
|
||||
|
||||
def test_scan_converts_duration_ms_to_seconds(tmp_path, monkeypatch):
|
||||
# tracks.duration is milliseconds; LRClib wants seconds. The scan must
|
||||
# convert before querying (215000ms → 215s) and store seconds in the finding.
|
||||
t1 = tmp_path / "song.flac"; t1.write_bytes(b"x")
|
||||
rows = [(1, "Song", "Artist", "Album", str(t1), 215000)] # 215000 ms = 215 s
|
||||
seen = {}
|
||||
fake_client = SimpleNamespace(
|
||||
api=object(),
|
||||
has_remote_lyrics=lambda title, artist, album, dur: seen.update(dur=dur) or True)
|
||||
monkeypatch.setattr("core.lyrics_client.lyrics_client", fake_client)
|
||||
|
||||
findings = []
|
||||
MissingLyricsJob().scan(_ctx(_DB(rows), findings))
|
||||
assert seen["dur"] == 215 # converted to seconds
|
||||
assert findings[0]["details"]["duration"] == 215
|
||||
|
||||
|
||||
def test_scan_skips_tracks_that_already_have_lrc(tmp_path, monkeypatch):
|
||||
t1 = tmp_path / "song.flac"; t1.write_bytes(b"x")
|
||||
(tmp_path / "song.lrc").write_text("[00:01]hi") # already has lyrics
|
||||
rows = [(1, "Song", "Artist", "Album", str(t1), 200)]
|
||||
fake_client = SimpleNamespace(api=object(),
|
||||
has_remote_lyrics=lambda *a, **k: True)
|
||||
monkeypatch.setattr("core.lyrics_client.lyrics_client", fake_client)
|
||||
|
||||
findings = []
|
||||
result = MissingLyricsJob().scan(_ctx(_DB(rows), findings))
|
||||
assert result.findings_created == 0
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_scan_noops_when_lrclib_disabled(monkeypatch):
|
||||
db = _DB([(1, "Song", "Artist", "Album", "/x.flac", 200)])
|
||||
ctx = _ctx(db, [])
|
||||
ctx.config_manager = SimpleNamespace(
|
||||
get=lambda k, d=None: False if k == 'metadata_enhancement.lrclib_enabled' else d)
|
||||
result = MissingLyricsJob().scan(ctx)
|
||||
assert result.scanned == 0 and result.findings_created == 0
|
||||
|
||||
|
||||
# ── _fix_missing_lyrics apply handler ────────────────────────────────────────
|
||||
|
||||
def test_fix_missing_lyrics_calls_create_lrc(tmp_path, monkeypatch):
|
||||
from core.repair_worker import RepairWorker
|
||||
audio = tmp_path / "song.flac"; audio.write_bytes(b"x")
|
||||
|
||||
w = RepairWorker.__new__(RepairWorker)
|
||||
w.transfer_folder = str(tmp_path)
|
||||
w._config_manager = SimpleNamespace(get=lambda k, d=None: d)
|
||||
|
||||
calls = {}
|
||||
fake_client = SimpleNamespace(
|
||||
create_lrc_file=lambda path, title, artist, album_name=None, duration_seconds=None:
|
||||
calls.update(path=path, title=title, artist=artist) or True)
|
||||
monkeypatch.setattr("core.lyrics_client.lyrics_client", fake_client)
|
||||
# _resolve_file_path: the file is already real, so identity is fine.
|
||||
monkeypatch.setattr("core.repair_worker._resolve_file_path",
|
||||
lambda raw, *a, **k: raw)
|
||||
|
||||
res = w._fix_missing_lyrics("track", "1", None, {
|
||||
"file_path": str(audio), "track_title": "Song", "artist": "Artist",
|
||||
"album_title": "Album", "duration": 200})
|
||||
assert res["success"] is True and res["action"] == "applied_lyrics"
|
||||
assert calls["title"] == "Song" and calls["path"] == str(audio)
|
||||
|
||||
|
||||
def test_fix_missing_lyrics_missing_file(tmp_path, monkeypatch):
|
||||
from core.repair_worker import RepairWorker
|
||||
w = RepairWorker.__new__(RepairWorker)
|
||||
w.transfer_folder = str(tmp_path)
|
||||
w._config_manager = SimpleNamespace(get=lambda k, d=None: d)
|
||||
monkeypatch.setattr("core.repair_worker._resolve_file_path", lambda raw, *a, **k: raw)
|
||||
res = w._fix_missing_lyrics("track", "1", None, {"file_path": str(tmp_path / "gone.flac")})
|
||||
assert res["success"] is False
|
||||
|
||||
|
||||
# ── retag apply_track_plans lyrics_action ────────────────────────────────────
|
||||
|
||||
def test_apply_track_plans_lyrics_action(tmp_path, monkeypatch):
|
||||
from core.repair_jobs import library_retag
|
||||
audio = tmp_path / "t.flac"; audio.write_bytes(b"x")
|
||||
|
||||
monkeypatch.setattr(library_retag, "write_tags_to_file",
|
||||
lambda *a, **k: {"success": True}, raising=False)
|
||||
seen = {}
|
||||
fake_client = SimpleNamespace(
|
||||
create_lrc_file=lambda path, title, artist, album_name=None, duration_seconds=None:
|
||||
seen.update(title=title) or True)
|
||||
monkeypatch.setattr("core.lyrics_client.lyrics_client", fake_client)
|
||||
|
||||
plans = [{"file_path": str(audio), "db_data": {},
|
||||
"lyrics_meta": {"title": "Song", "artist": "Artist", "album": "Album"}}]
|
||||
res = library_retag.apply_track_plans(plans, lyrics_action=True)
|
||||
assert res["lyrics_written"] == 1 and seen["title"] == "Song"
|
||||
|
||||
|
||||
def test_apply_track_plans_lyrics_never_writes_tags(tmp_path, monkeypatch):
|
||||
# The lyrics query must come from lyrics_meta, NOT db_data — so an
|
||||
# unmatched track (db_data={}) gets lyrics fetched but NO tags written.
|
||||
from core.repair_jobs import library_retag
|
||||
audio = tmp_path / "t.flac"; audio.write_bytes(b"x")
|
||||
written = []
|
||||
monkeypatch.setattr("core.tag_writer.write_tags_to_file",
|
||||
lambda fp, db_data, **k: written.append(db_data) or {"success": True})
|
||||
monkeypatch.setattr("core.lyrics_client.lyrics_client",
|
||||
SimpleNamespace(create_lrc_file=lambda *a, **k: True))
|
||||
|
||||
plans = [{"file_path": str(audio), "db_data": {},
|
||||
"lyrics_meta": {"title": "Song", "artist": "Artist", "album": "Al"}}]
|
||||
res = library_retag.apply_track_plans(plans, lyrics_action=True)
|
||||
assert res["lyrics_written"] == 1
|
||||
# write_tags_to_file was called with an EMPTY db_data — no title/artist leaked in.
|
||||
assert written == [{}]
|
||||
|
||||
|
||||
def test_apply_track_plans_no_lyrics_when_disabled(tmp_path, monkeypatch):
|
||||
from core.repair_jobs import library_retag
|
||||
audio = tmp_path / "t.flac"; audio.write_bytes(b"x")
|
||||
monkeypatch.setattr(library_retag, "write_tags_to_file",
|
||||
lambda *a, **k: {"success": True}, raising=False)
|
||||
called = []
|
||||
fake_client = SimpleNamespace(create_lrc_file=lambda *a, **k: called.append(1) or True)
|
||||
monkeypatch.setattr("core.lyrics_client.lyrics_client", fake_client)
|
||||
|
||||
plans = [{"file_path": str(audio), "db_data": {"title": "Song"}}]
|
||||
res = library_retag.apply_track_plans(plans, lyrics_action=False)
|
||||
assert res["lyrics_written"] == 0 and called == []
|
||||
67
tests/test_navidrome_stream_url.py
Normal file
67
tests/test_navidrome_stream_url.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
"""Navidrome stream-URL building (#809): play a library track via the server's
|
||||
Subsonic /rest/stream API so playback works without mounting the music into
|
||||
the SoulSync container.
|
||||
|
||||
Mirrors the cover-art URL tests — token/salt are random per call, so we assert
|
||||
structure + required params, not an exact string.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from core.navidrome_client import NavidromeClient
|
||||
|
||||
|
||||
def _connected_client():
|
||||
c = NavidromeClient()
|
||||
c.base_url = "https://nav.example.com"
|
||||
c.username = "boulder"
|
||||
c.password = "hunter2"
|
||||
return c
|
||||
|
||||
|
||||
def test_builds_authenticated_stream_url():
|
||||
url = _connected_client().build_stream_url("song-42")
|
||||
parts = urlsplit(url)
|
||||
assert parts.scheme == "https"
|
||||
assert parts.netloc == "nav.example.com"
|
||||
assert parts.path == "/rest/stream"
|
||||
q = parse_qs(parts.query)
|
||||
assert q["id"] == ["song-42"]
|
||||
assert q["u"] == ["boulder"]
|
||||
# Subsonic token auth — salted md5, never the raw password.
|
||||
assert q["t"] and q["t"][0] != "hunter2"
|
||||
assert q["s"]
|
||||
assert "hunter2" not in url
|
||||
for required in ("t", "s", "v", "c"):
|
||||
assert required in q
|
||||
|
||||
|
||||
def test_no_transcode_by_default():
|
||||
assert "maxBitRate" not in (_connected_client().build_stream_url("x") or "")
|
||||
|
||||
|
||||
def test_max_bitrate_when_set():
|
||||
assert "maxBitRate=320" in _connected_client().build_stream_url("x", max_bitrate=320)
|
||||
# 0 / falsy → omitted (original file).
|
||||
assert "maxBitRate" not in _connected_client().build_stream_url("x", max_bitrate=0)
|
||||
|
||||
|
||||
def test_song_id_stringified():
|
||||
assert "id=12345" in _connected_client().build_stream_url(12345)
|
||||
|
||||
|
||||
def test_returns_none_when_not_connected():
|
||||
assert NavidromeClient().build_stream_url("song-1") is None
|
||||
|
||||
|
||||
def test_returns_none_for_empty_song_id():
|
||||
assert _connected_client().build_stream_url("") is None
|
||||
assert _connected_client().build_stream_url(None) is None
|
||||
|
||||
|
||||
def test_returns_none_without_credentials():
|
||||
c = NavidromeClient()
|
||||
c.base_url = "https://nav.example.com"
|
||||
assert c.build_stream_url("song-1") is None
|
||||
|
|
@ -68,6 +68,13 @@ class _FakeDatabase:
|
|||
CREATE TABLE discovery_artist_blacklist (
|
||||
artist_name TEXT PRIMARY KEY
|
||||
);
|
||||
-- Unified blocklist — discovery filtering now unions artist bans
|
||||
-- from here too (Phase 1 blocklist). Minimal shape for the subquery.
|
||||
CREATE TABLE blocklist (
|
||||
id INTEGER PRIMARY KEY,
|
||||
entity_type TEXT,
|
||||
name TEXT
|
||||
);
|
||||
-- Minimal `tracks` table: exists so the `exclude_owned`
|
||||
-- subquery in `_select_discovery_tracks` can join. Real
|
||||
-- schema has many more columns; we only need the source-id
|
||||
|
|
|
|||
67
tests/test_release_dates.py
Normal file
67
tests/test_release_dates.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
"""#705 release-date gate: unreleased tracks stay out of hot paths.
|
||||
|
||||
Watchlist scans add announced albums on purpose; the gate keeps their
|
||||
future-dated tracks out of the wishlist search cycle and the Fresh Tape
|
||||
radar until release day. Conservative by design: only a CONFIDENTLY
|
||||
future date gates; bad/missing dates never block anything.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from core.metadata.release_dates import (
|
||||
is_future_release,
|
||||
split_released_unreleased,
|
||||
track_release_date,
|
||||
)
|
||||
|
||||
TODAY = date(2026, 6, 7)
|
||||
|
||||
|
||||
def test_full_dates():
|
||||
assert is_future_release('2026-06-08', today=TODAY) is True
|
||||
assert is_future_release('2026-06-07', today=TODAY) is False # release DAY = released
|
||||
assert is_future_release('2026-06-06', today=TODAY) is False
|
||||
assert is_future_release('2027-01-01', today=TODAY) is True
|
||||
|
||||
|
||||
def test_partial_dates_are_conservative():
|
||||
# Year-only: future only when the YEAR is future.
|
||||
assert is_future_release('2027', today=TODAY) is True
|
||||
assert is_future_release('2026', today=TODAY) is False
|
||||
# Year-month: future only when the MONTH is future.
|
||||
assert is_future_release('2026-07', today=TODAY) is True
|
||||
assert is_future_release('2026-06', today=TODAY) is False
|
||||
assert is_future_release('2026-05', today=TODAY) is False
|
||||
|
||||
|
||||
def test_garbage_never_blocks():
|
||||
for bad in ('', None, 'unknown', 'soon', '20xx-01-01', '2026-13-45', 123, {}):
|
||||
assert is_future_release(bad, today=TODAY) is False
|
||||
|
||||
|
||||
def test_invalid_day_falls_back_to_month_precision():
|
||||
# 2026-06-99 is unparseable as a date but month precision says "not future".
|
||||
assert is_future_release('2026-06-99', today=TODAY) is False
|
||||
assert is_future_release('2026-07-99', today=TODAY) is True
|
||||
|
||||
|
||||
def test_track_release_date_shapes():
|
||||
assert track_release_date({'album': {'release_date': '2026-10-03'}}) == '2026-10-03'
|
||||
assert track_release_date({'release_date': '2026'}) == '2026'
|
||||
assert track_release_date({'album': 'a-string'}) == ''
|
||||
assert track_release_date({}) == ''
|
||||
assert track_release_date(None) == ''
|
||||
|
||||
|
||||
def test_split_partitions_and_preserves_order():
|
||||
tracks = [
|
||||
{'name': 'out', 'album': {'release_date': '2026-01-01'}},
|
||||
{'name': 'tomorrow', 'album': {'release_date': '2026-06-08'}},
|
||||
{'name': 'no-date', 'album': {}},
|
||||
{'name': 'next-year', 'release_date': '2027'},
|
||||
]
|
||||
released, unreleased = split_released_unreleased(tracks, today=TODAY)
|
||||
assert [t['name'] for t in released] == ['out', 'no-date']
|
||||
assert [t['name'] for t in unreleased] == ['tomorrow', 'next-year']
|
||||
119
tests/test_reorganize_alternate_edition.py
Normal file
119
tests/test_reorganize_alternate_edition.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
"""#767-2: the reorganizer's on-demand alternate-edition path.
|
||||
|
||||
When the walked edition (the first source we have an ID for) clearly misfits the
|
||||
on-disk files — e.g. a 1-track single whose only ID points at the 10-track deluxe
|
||||
— `_resolve_source` must find a better-fitting edition, use it for the plan, and
|
||||
(on apply) persist the canonical pin. A well-fitting album must keep today's exact
|
||||
behavior and never trigger an alternate fetch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import core.library_reorganize as lr
|
||||
import core.metadata.canonical_resolver as cr
|
||||
|
||||
# Provider-shaped raw tracklists (what get_album_tracks_for_source returns).
|
||||
SINGLE_RAW = [{"name": "Scatterbrain", "track_number": 1, "duration_ms": 129_000}]
|
||||
DELUXE_RAW = [{"name": "Intro", "track_number": 1, "duration_ms": 200_000}] + [
|
||||
{"name": "Scatterbrain", "track_number": 2, "duration_ms": 130_000}
|
||||
] + [
|
||||
{"name": f"Bonus {i}", "track_number": i + 2, "duration_ms": 180_000}
|
||||
for i in range(1, 9)
|
||||
]
|
||||
# Resolver-normalised shape (what default_fetch_tracklist returns).
|
||||
SINGLE_NORM = [{"title": "Scatterbrain", "track_number": 1, "duration_ms": 129_000}]
|
||||
DELUXE_NORM = [{"title": t["name"], "duration_ms": t["duration_ms"]} for t in DELUXE_RAW]
|
||||
|
||||
ALBUM_META = {
|
||||
"sp_deluxe": {"name": "Scatterbrain (Deluxe)"},
|
||||
"sp_single": {"name": "Scatterbrain - Single"},
|
||||
}
|
||||
TRACKLISTS = {"sp_deluxe": DELUXE_RAW, "sp_single": SINGLE_RAW}
|
||||
|
||||
|
||||
def _wire(monkeypatch, *, alternates):
|
||||
"""Patch the source-API seams the reorganizer + resolver funnel through."""
|
||||
monkeypatch.setattr(lr, "get_source_priority", lambda primary: ["spotify"])
|
||||
monkeypatch.setattr(lr, "get_album_for_source", lambda s, aid: ALBUM_META.get(aid))
|
||||
monkeypatch.setattr(lr, "get_album_tracks_for_source", lambda s, aid: TRACKLISTS.get(aid))
|
||||
# Resolver-internal fetchers (imported by name inside _resolve_better_edition).
|
||||
norm = {"sp_deluxe": DELUXE_NORM, "sp_single": SINGLE_NORM}
|
||||
monkeypatch.setattr(cr, "default_fetch_tracklist", lambda s, aid: norm.get(aid))
|
||||
monkeypatch.setattr(cr, "default_fetch_alternates", alternates)
|
||||
|
||||
|
||||
def test_misfit_single_resolves_to_the_single_edition(monkeypatch):
|
||||
alt_calls = []
|
||||
|
||||
def alternates(source, aid, **kw):
|
||||
alt_calls.append((source, aid))
|
||||
return [
|
||||
{"album_id": "sp_single", "tracks": SINGLE_NORM},
|
||||
{"album_id": "sp_deluxe", "tracks": DELUXE_NORM},
|
||||
]
|
||||
|
||||
_wire(monkeypatch, alternates=alternates)
|
||||
pins = []
|
||||
album_data = {
|
||||
"spotify_album_id": "sp_deluxe", "title": "Scatterbrain",
|
||||
"artist_id": "a1", "artist_name": "The Band",
|
||||
}
|
||||
file_tracks = [{"duration_ms": 129_000, "title": "Scatterbrain"}] # owns the single
|
||||
|
||||
source, api_album, items = lr._resolve_source(
|
||||
album_data, "spotify",
|
||||
file_tracks=file_tracks,
|
||||
on_better_edition=lambda s, aid, sc: pins.append((s, aid, sc)),
|
||||
)
|
||||
|
||||
assert source == "spotify"
|
||||
assert api_album == ALBUM_META["sp_single"] # used the single, not the deluxe
|
||||
assert len(items) == 1
|
||||
assert alt_calls, "misfit must trigger an alternate-edition fetch"
|
||||
assert pins and pins[0][1] == "sp_single", "apply must persist the better pin"
|
||||
|
||||
|
||||
def test_well_fitting_album_keeps_walk_and_never_expands(monkeypatch):
|
||||
alt_calls = []
|
||||
|
||||
def alternates(source, aid, **kw):
|
||||
alt_calls.append((source, aid))
|
||||
return [{"album_id": "sp_single", "tracks": SINGLE_NORM}]
|
||||
|
||||
_wire(monkeypatch, alternates=alternates)
|
||||
pins = []
|
||||
# The library actually IS the deluxe (10 matching tracks) -> walk fits -> no expand.
|
||||
album_data = {
|
||||
"spotify_album_id": "sp_deluxe", "title": "Scatterbrain (Deluxe)",
|
||||
"artist_id": "a1", "artist_name": "The Band",
|
||||
}
|
||||
file_tracks = [{"duration_ms": t["duration_ms"], "title": t["name"]} for t in DELUXE_RAW]
|
||||
|
||||
source, api_album, items = lr._resolve_source(
|
||||
album_data, "spotify",
|
||||
file_tracks=file_tracks,
|
||||
on_better_edition=lambda s, aid, sc: pins.append((s, aid, sc)),
|
||||
)
|
||||
|
||||
assert source == "spotify" and api_album == ALBUM_META["sp_deluxe"]
|
||||
assert alt_calls == [], "a well-fitting edition must not trigger any alternate fetch"
|
||||
assert pins == [], "no pin written when the walk already fits"
|
||||
|
||||
|
||||
def test_strict_source_never_expands(monkeypatch):
|
||||
# User explicitly picked the source in the modal -> their choice wins, even on
|
||||
# a misfit. No alternate search.
|
||||
alt_calls = []
|
||||
|
||||
def alternates(source, aid, **kw):
|
||||
alt_calls.append((source, aid))
|
||||
return [{"album_id": "sp_single", "tracks": SINGLE_NORM}]
|
||||
|
||||
_wire(monkeypatch, alternates=alternates)
|
||||
album_data = {"spotify_album_id": "sp_deluxe", "title": "Scatterbrain"}
|
||||
file_tracks = [{"duration_ms": 129_000, "title": "Scatterbrain"}]
|
||||
|
||||
source, api_album, items = lr._resolve_source(
|
||||
album_data, "spotify", strict_source=True, file_tracks=file_tracks,
|
||||
)
|
||||
assert source == "spotify" and api_album == ALBUM_META["sp_deluxe"]
|
||||
assert alt_calls == [], "strict_source must not trigger alternate expansion"
|
||||
|
|
@ -52,7 +52,7 @@ SPLIT_MODULES = [
|
|||
|
||||
# Other JS files that exist in static/ but are NOT part of the split
|
||||
NON_SPLIT_JS = {"setup-wizard.js", "docs.js", "helper.js", "particles.js", "worker-orbs.js",
|
||||
"enrichment-manager.js"}
|
||||
"enrichment-manager.js", "origin-history.js", "blocklist.js"}
|
||||
|
||||
# Pre-existing duplicate helper functions that lived in the original monolith.
|
||||
# In a plain <script> context the last-loaded declaration wins. These are NOT
|
||||
|
|
@ -216,8 +216,10 @@ class TestOnclickCoverage:
|
|||
text = _read(_STATIC / module)
|
||||
self.all_fns.update(_all_function_decls(text))
|
||||
|
||||
# Also include non-split JS files that are loaded
|
||||
for extra in ("setup-wizard.js", "docs.js", "helper.js", "enrichment-manager.js"):
|
||||
# Also include non-split JS files that are loaded — driven by the
|
||||
# NON_SPLIT_JS registry so a newly added standalone module can't be
|
||||
# silently missing from onclick coverage (origin-history.js was).
|
||||
for extra in sorted(NON_SPLIT_JS):
|
||||
path = _STATIC / extra
|
||||
if path.exists():
|
||||
self.all_fns.update(_all_function_decls(_read(path)))
|
||||
|
|
|
|||
|
|
@ -60,6 +60,18 @@ def test_gate_open_when_no_auth_and_rate_limited():
|
|||
assert should_use_free_fallback(authenticated=False, rate_limited=True) is True
|
||||
|
||||
|
||||
def test_gate_open_when_budget_exhausted_even_if_authed_and_healthy():
|
||||
# #758-follow-up: the real-API daily budget is spent, but the user has
|
||||
# Spotify Free — switch to the uncapped free source instead of pausing.
|
||||
assert should_use_free_fallback(authenticated=True, rate_limited=False,
|
||||
budget_exhausted=True) is True
|
||||
|
||||
|
||||
def test_gate_closed_when_authed_healthy_and_under_budget():
|
||||
assert should_use_free_fallback(authenticated=True, rate_limited=False,
|
||||
budget_exhausted=False) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# should_offer_spotify_metadata — the availability gate the callers use
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue