From 0bbf44809f382c70bb99d8545ada5f58ebc3e665 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Fri, 24 Apr 2026 16:22:33 +0300 Subject: [PATCH 01/19] Move the import flows and related post-processing pipelines into separate modules - Extract the import pipeline, album import, staging, path, file ops, guards, runtime state, side effects, and metadata enrichment out of . - Canonicalize the refactored import path around and remove legacy , , , and request shapes from the import endpoints. - Make album and track metadata lookups follow the configured provider priority instead of hard-coding Spotify, while still falling back when needed. - Update the import routes and frontend payloads to use the new core helpers. - Add coverage for the extracted helpers and the refactored import flows. PS. apologies to anyone who might check this commit out - the intention was to start small, but things kinda snowballed out of control at some point since the logic just kept going on and on, and everything kinda had to be changed all at once for it all to make any sense --- api/downloads.py | 3 +- api/system.py | 4 +- core/import_album.py | 515 ++ core/import_context.py | 326 ++ core/import_file_ops.py | 475 ++ core/import_filename.py | 73 + core/import_guards.py | 121 + core/import_paths.py | 757 +++ core/import_pipeline.py | 925 +++ core/import_runtime_state.py | 121 + core/import_side_effects.py | 567 ++ core/import_staging.py | 454 ++ core/metadata_enrichment.py | 1486 +++++ core/metadata_service.py | 390 +- core/soulseek_client.py | 68 +- core/watchlist_scanner.py | 3 - tests/test_downloads_pagination.py | 20 +- tests/test_import_album.py | 186 + tests/test_import_context.py | 202 + tests/test_import_file_ops.py | 92 + tests/test_import_filename.py | 33 + tests/test_import_guards.py | 47 + tests/test_import_paths.py | 161 + tests/test_import_pipeline.py | 128 + tests/test_import_side_effects.py | 144 + tests/test_import_staging.py | 265 + tests/test_metadata_enrichment.py | 333 ++ ..._metadata_service_single_import_context.py | 193 + tests/test_watchlist_scanner_scan.py | 6 +- web_server.py | 5053 +---------------- 30 files changed, 8238 insertions(+), 4913 deletions(-) create mode 100644 core/import_album.py create mode 100644 core/import_context.py create mode 100644 core/import_file_ops.py create mode 100644 core/import_filename.py create mode 100644 core/import_guards.py create mode 100644 core/import_paths.py create mode 100644 core/import_pipeline.py create mode 100644 core/import_runtime_state.py create mode 100644 core/import_side_effects.py create mode 100644 core/import_staging.py create mode 100644 core/metadata_enrichment.py create mode 100644 tests/test_import_album.py create mode 100644 tests/test_import_context.py create mode 100644 tests/test_import_file_ops.py create mode 100644 tests/test_import_filename.py create mode 100644 tests/test_import_guards.py create mode 100644 tests/test_import_paths.py create mode 100644 tests/test_import_pipeline.py create mode 100644 tests/test_import_side_effects.py create mode 100644 tests/test_import_staging.py create mode 100644 tests/test_metadata_enrichment.py create mode 100644 tests/test_metadata_service_single_import_context.py diff --git a/api/downloads.py b/api/downloads.py index 448b80cd..fd36e91e 100644 --- a/api/downloads.py +++ b/api/downloads.py @@ -5,6 +5,7 @@ Download management endpoints — list, cancel active downloads. from flask import request, current_app from .auth import require_api_key from .helpers import api_success, api_error +from core.import_runtime_state import download_tasks, tasks_lock def _serialize_download(task_id, task): @@ -53,8 +54,6 @@ def register_routes(bp): descending so newest/in-flight tasks appear first. """ try: - from web_server import download_tasks, tasks_lock - # Parse pagination params try: limit = int(request.args.get("limit", 100)) diff --git a/api/system.py b/api/system.py index 4a0ecb03..b6a72141 100644 --- a/api/system.py +++ b/api/system.py @@ -55,7 +55,7 @@ def register_routes(bp): def system_activity(): """Recent activity feed.""" try: - from web_server import activity_feed + from core.import_runtime_state import activity_feed items = list(activity_feed) if activity_feed else [] return api_success({"activities": items}) except Exception as e: @@ -74,7 +74,7 @@ def register_routes(bp): # Active download count download_count = 0 try: - from web_server import download_tasks, tasks_lock + from core.import_runtime_state import download_tasks, tasks_lock with tasks_lock: download_count = sum( 1 for t in download_tasks.values() diff --git a/core/import_album.py b/core/import_album.py new file mode 100644 index 00000000..953e091e --- /dev/null +++ b/core/import_album.py @@ -0,0 +1,515 @@ +"""Album import helpers for staging matching and post-processing context.""" + +from __future__ import annotations + +import os +from typing import Any, Dict, Iterable, List, Optional, Set + +from core.import_context import normalize_import_context +from core.import_file_ops import read_staging_file_metadata +from core.import_staging import AUDIO_EXTENSIONS, get_staging_path +from utils.logging_config import get_logger + + +logger = get_logger("import_album") + +def get_client_for_source(source: str): + from core.metadata_service import get_client_for_source as _get_client_for_source + + return _get_client_for_source(source) + + +def get_artist_album_tracks( + album_id: str, + artist_name: str = "", + album_name: str = "", + source: Optional[str] = None, +): + from core.metadata_service import get_artist_album_tracks as _get_artist_album_tracks + + return _get_artist_album_tracks( + album_id, + artist_name=artist_name, + album_name=album_name, + source_override=source, + ) + + +try: + from core.matching_engine import MusicMatchingEngine + _MATCHING_ENGINE_IMPORT_ERROR = None +except Exception as exc: # pragma: no cover - only hits in stripped-down environments + MusicMatchingEngine = None # type: ignore[assignment] + _MATCHING_ENGINE_IMPORT_ERROR = exc + + +_MATCHING_ENGINE = None + + +def _get_matching_engine() -> Any: + global _MATCHING_ENGINE + if _MATCHING_ENGINE is None: + if MusicMatchingEngine is None: + raise RuntimeError("Music matching engine is unavailable") from _MATCHING_ENGINE_IMPORT_ERROR + _MATCHING_ENGINE = MusicMatchingEngine() + return _MATCHING_ENGINE + + +def _normalize_artist_entries(artists: Any) -> List[Dict[str, Any]]: + if not artists: + return [] + + if isinstance(artists, (str, bytes)): + artists = [artists] + elif isinstance(artists, dict): + artists = [artists] + else: + try: + artists = list(artists) + except TypeError: + artists = [artists] + + normalized: List[Dict[str, Any]] = [] + for artist in artists: + if isinstance(artist, dict): + entry: Dict[str, Any] = {} + name = artist.get("name") or artist.get("artist_name") or artist.get("title") or "" + artist_id = artist.get("id") or artist.get("artist_id") or "" + if name: + entry["name"] = str(name) + if artist_id: + entry["id"] = str(artist_id) + genres = artist.get("genres") + if genres is not None: + entry["genres"] = genres + if entry: + normalized.append(entry) + continue + + name = str(artist).strip() + if name: + normalized.append({"name": name}) + + return normalized + + +def _normalize_album_source(album: Dict[str, Any], source: str = "") -> str: + album_source = source or album.get("source") or "" + return str(album_source).strip().lower() + + +def _strip_legacy_source_fields(payload: Any) -> Any: + if not isinstance(payload, dict): + return payload + + cleaned = dict(payload) + cleaned.pop("_source", None) + cleaned.pop("provider", None) + return cleaned + + +def _extract_track_artist_name(track: Dict[str, Any]) -> str: + artists = track.get("artists") or [] + if isinstance(artists, (str, bytes)): + artists = [artists] + elif isinstance(artists, dict): + artists = [artists] + else: + try: + artists = list(artists) + except TypeError: + artists = [artists] + + if not artists: + return "" + + first = artists[0] + if isinstance(first, dict): + return str(first.get("name") or first.get("artist_name") or first.get("title") or "").strip() + return str(first or "").strip() + + +def _coerce_track_int(value: Any, default: int = 1) -> int: + if value in (None, ""): + return default + try: + return int(str(value).split("/")[0].strip() or default) + except (TypeError, ValueError): + return default + + +def _collect_staging_files(file_paths: Optional[Iterable[str]] = None) -> List[Dict[str, Any]]: + staging_path = get_staging_path() + file_filter: Optional[Set[str]] = set(file_paths) if file_paths else None + staging_files: List[Dict[str, Any]] = [] + + if not os.path.isdir(staging_path): + return staging_files + + for root, _dirs, filenames in os.walk(staging_path): + for filename in filenames: + ext = os.path.splitext(filename)[1].lower() + if ext not in AUDIO_EXTENSIONS: + continue + + full_path = os.path.join(root, filename) + if file_filter is not None and full_path not in file_filter: + continue + + meta = read_staging_file_metadata(full_path, filename) + staging_files.append( + { + "filename": filename, + "full_path": full_path, + "title": meta.get("title", ""), + "artist": meta.get("albumartist") or meta.get("artist") or "", + "album": meta.get("album", ""), + "albumartist": meta.get("albumartist") or meta.get("artist") or "", + "track_number": meta.get("track_number", 1), + "disc_number": meta.get("disc_number", 1), + } + ) + + return staging_files + + +def _normalize_match_track(track: Dict[str, Any], source: str, album: Dict[str, Any]) -> Dict[str, Any]: + track_album = track.get("album") if isinstance(track.get("album"), dict) else album + if isinstance(track_album, dict): + track_album = _strip_legacy_source_fields(track_album) + track_source = _normalize_album_source(track, source) + track_artists = _normalize_artist_entries(track.get("artists") or []) + + if not track_artists and album.get("artists"): + track_artists = _normalize_artist_entries(album.get("artists")) + + return { + "id": track.get("id", ""), + "name": track.get("name", "Unknown Track"), + "track_number": _coerce_track_int(track.get("track_number", 1), default=1), + "disc_number": _coerce_track_int(track.get("disc_number", 1), default=1), + "duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0), + "artists": track_artists, + "uri": track.get("uri", ""), + "album": track_album, + "source": track_source, + } + + +def _score_album_track_match(track: Dict[str, Any], staging_file: Dict[str, Any], album_name: str) -> float: + engine = _get_matching_engine() + + track_name = track.get("name", "") + staging_title = staging_file.get("title", "") + score = 0.0 + + title_sim = engine.similarity_score( + engine.normalize_string(track_name), + engine.normalize_string(staging_title or ""), + ) + score += title_sim * 0.45 + + track_artist_name = _extract_track_artist_name(track) + staging_artist = staging_file.get("artist") or "" + if track_artist_name and staging_artist: + artist_sim = engine.similarity_score( + engine.normalize_string(track_artist_name), + engine.normalize_string(staging_artist), + ) + score += artist_sim * 0.15 + else: + score += 0.075 + + track_number = _coerce_track_int(track.get("track_number", 1), default=1) + staging_track_number = _coerce_track_int(staging_file.get("track_number", 1), default=1) + if staging_track_number and track_number: + if staging_track_number == track_number: + score += 0.30 + elif abs(staging_track_number - track_number) <= 1: + score += 0.12 + + staging_album = staging_file.get("album") or "" + if staging_album and album_name: + album_sim = engine.similarity_score( + engine.normalize_string(staging_album), + engine.normalize_string(album_name), + ) + score += album_sim * 0.10 + + return score + + +def _fetch_artist_data_for_source(client: Any, artist_id: str, source: str) -> Any: + if source == "spotify": + try: + return client.get_artist(artist_id, allow_fallback=False) + except TypeError: + return client.get_artist(artist_id) + return client.get_artist(artist_id) + + +def resolve_album_artist_context(album: Dict[str, Any], source: str = "") -> Dict[str, Any]: + """Build a neutral artist context for album import processing.""" + album = dict(album or {}) + source = _normalize_album_source(album, source) + + artists = _normalize_artist_entries(album.get("artists") or []) + if not artists: + artist_name = album.get("artist") or album.get("artist_name") or "" + artist_id = album.get("artist_id") or "" + if artist_name or artist_id: + artist_entry: Dict[str, Any] = {} + if artist_name: + artist_entry["name"] = str(artist_name) + if artist_id: + artist_entry["id"] = str(artist_id) + artists = [artist_entry] + + primary_artist = artists[0] if artists else {} + artist_name = str( + primary_artist.get("name") + or album.get("artist") + or album.get("artist_name") + or "Unknown Artist" + ).strip() + artist_id = str(primary_artist.get("id") or album.get("artist_id") or "").strip() + + genres: List[Any] = [] + if artist_id and source: + client = get_client_for_source(source) + if client and hasattr(client, "get_artist"): + try: + artist_data = _fetch_artist_data_for_source(client, artist_id, source) + raw_genres = artist_data.get("genres") if isinstance(artist_data, dict) else getattr(artist_data, "genres", []) + if isinstance(raw_genres, str): + genres = [raw_genres] + elif raw_genres: + try: + genres = list(raw_genres) + except TypeError: + genres = [raw_genres] + except Exception as exc: + logger.debug("Could not resolve artist genres for %s on %s: %s", artist_id, source, exc) + + return { + "id": artist_id, + "name": artist_name, + "genres": genres, + "source": source, + } + + +def build_album_import_context( + album: Dict[str, Any], + track: Dict[str, Any], + *, + artist_context: Optional[Dict[str, Any]] = None, + total_discs: int = 1, + source: str = "", +) -> Dict[str, Any]: + """Build a neutral post-processing context for one album track.""" + album = dict(album or {}) + track = dict(track or {}) + source = _normalize_album_source(album, source) + + album_artists = _normalize_artist_entries(album.get("artists") or []) + if not album_artists and artist_context: + album_artists = _normalize_artist_entries([artist_context]) + + if artist_context: + artist_ctx = dict(artist_context) + else: + artist_ctx = resolve_album_artist_context(album, source) + + artist_ctx = _strip_legacy_source_fields(artist_ctx) + artist_ctx.setdefault("genres", []) + artist_ctx.setdefault("source", source) + artist_ctx["genres"] = artist_ctx.get("genres") or [] + + track_artists = _normalize_artist_entries(track.get("artists") or []) + if not track_artists: + track_artists = album_artists or [artist_ctx] + + track_album_value = track.get("album") + if isinstance(track_album_value, dict): + track_album_name = ( + track_album_value.get("name") + or track_album_value.get("title") + or album.get("name") + or album.get("album_name") + or "" + ) + track_album_id = str(track_album_value.get("id") or track_album_value.get("album_id") or "").strip() + track_album_type = track_album_value.get("album_type") or album.get("album_type") or "album" + track_album_release = track_album_value.get("release_date") or album.get("release_date") or "" + track_album_image = track_album_value.get("image_url") or album.get("image_url") or "" + else: + track_album_name = str(track_album_value or album.get("name") or album.get("album_name") or "").strip() + track_album_id = str(album.get("id") or album.get("album_id") or "").strip() + track_album_type = album.get("album_type") or "album" + track_album_release = album.get("release_date") or "" + track_album_image = album.get("image_url") or "" + + album_name = str(album.get("name") or album.get("album_name") or track_album_name or "Unknown Album").strip() + artist_name = str( + artist_ctx.get("name") + or album.get("artist") + or album.get("artist_name") + or "Unknown Artist" + ).strip() + + track_number = _coerce_track_int(track.get("track_number", 1), default=1) + disc_number = _coerce_track_int(track.get("disc_number", 1), default=1) + + normalized_track = { + "id": str(track.get("id") or track.get("track_id") or "").strip(), + "name": str(track.get("name") or "Unknown Track").strip(), + "track_number": track_number, + "disc_number": disc_number, + "duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0), + "artists": track_artists, + "uri": str(track.get("uri") or "").strip(), + "album": track_album_name, + "album_id": track_album_id, + "album_type": track_album_type, + "release_date": track_album_release, + "source": source, + } + + normalized_album = { + "id": str(album.get("id") or album.get("album_id") or track_album_id or "").strip(), + "name": album_name, + "artist": artist_name, + "artist_name": artist_name, + "artist_id": str(artist_ctx.get("id") or album.get("artist_id") or "").strip(), + "artists": album_artists, + "release_date": str(album.get("release_date") or track_album_release or "").strip(), + "total_tracks": int(album.get("total_tracks") or track.get("total_tracks") or 0) or 1, + "total_discs": int(total_discs or 1) if str(total_discs or 1).isdigit() else total_discs or 1, + "album_type": str(album.get("album_type") or track_album_type or "album").strip() or "album", + "image_url": str(album.get("image_url") or track_album_image or "").strip(), + "images": album.get("images") or ([] if not track_album_image else [{"url": track_album_image}]), + "source": source, + } + + original_search = { + "title": normalized_track["name"], + "artist": artist_name, + "album": album_name, + "track_number": track_number, + "disc_number": disc_number, + "clean_title": normalized_track["name"], + "clean_album": album_name, + "clean_artist": artist_name, + "artists": track_artists, + "duration_ms": normalized_track["duration_ms"], + "id": normalized_track["id"], + "source": source, + } + + context = { + "artist": artist_ctx, + "album": normalized_album, + "track_info": normalized_track, + "original_search_result": original_search, + "is_album_download": True, + "has_clean_metadata": bool(normalized_track["id"]), + "has_full_metadata": bool(normalized_track["id"]), + "source": source, + } + + normalized_context = normalize_import_context(context) + normalized_context["artist"] = _strip_legacy_source_fields(normalized_context.get("artist")) + normalized_context["album"] = _strip_legacy_source_fields(normalized_context.get("album")) + normalized_context["track_info"] = _strip_legacy_source_fields(normalized_context.get("track_info")) + normalized_context["original_search_result"] = _strip_legacy_source_fields(normalized_context.get("original_search_result")) + return normalized_context + + +def build_album_import_match_payload( + album_id: str, + *, + album_name: str = "", + album_artist: str = "", + file_paths: Optional[Iterable[str]] = None, + source: Optional[str] = None, +) -> Dict[str, Any]: + """Build the album import match payload using provider-priority metadata lookup.""" + album_response = get_artist_album_tracks( + album_id, + artist_name=album_artist, + album_name=album_name, + source=source, + ) + + album = _strip_legacy_source_fields(dict(album_response.get("album") or {})) + source = _normalize_album_source(album, album_response.get("source") or source or "") + tracks = list(album_response.get("tracks") or []) + if not album_response.get("success") or not tracks: + return { + "success": False, + "error": album_response.get("error", "Album not found"), + "status_code": album_response.get("status_code", 404), + "album": { + "id": album_id, + "name": album_name or album_id, + "artist": album_artist or "Unknown Artist", + "artist_name": album_artist or "Unknown Artist", + "artist_id": "", + "artists": [], + "release_date": "", + "total_tracks": 0, + "total_discs": 1, + "album_type": "album", + "image_url": "", + "images": [], + "source": source, + }, + "matches": [], + "unmatched_files": [], + "source": source, + "source_priority": album_response.get("source_priority", []), + "resolved_album_id": album_response.get("resolved_album_id") or album_id, + } + + staging_files = _collect_staging_files(file_paths) + album_name_for_match = album.get("name") or album_name or "" + matches: List[Dict[str, Any]] = [] + used_files: Set[int] = set() + + for track in tracks: + normalized_track = _normalize_match_track(track, source, album) + best_match = None + best_score = 0.0 + + for index, staging_file in enumerate(staging_files): + if index in used_files: + continue + + score = _score_album_track_match(normalized_track, staging_file, album_name_for_match) + if score > best_score and score >= 0.4: + best_score = score + best_match = index + + matches.append( + { + "track": normalized_track, + "staging_file": staging_files[best_match] if best_match is not None else None, + "confidence": round(best_score, 2) if best_match is not None else 0, + } + ) + + if best_match is not None: + used_files.add(best_match) + + unmatched_files = [sf for index, sf in enumerate(staging_files) if index not in used_files] + + return { + "success": True, + "album": album, + "matches": matches, + "unmatched_files": unmatched_files, + "source": source, + "source_priority": album_response.get("source_priority", []), + "resolved_album_id": album_response.get("resolved_album_id") or album_id, + } diff --git a/core/import_context.py b/core/import_context.py new file mode 100644 index 00000000..cf9504ff --- /dev/null +++ b/core/import_context.py @@ -0,0 +1,326 @@ +"""Helpers for normalizing and reading import contexts. + +These functions keep the single-import pipeline source-agnostic while still +accepting legacy `spotify_*` payloads from older callers. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + + +def _as_dict(value: Any) -> Dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _first_value(mapping: Dict[str, Any], *keys: str, default: Any = "") -> Any: + for key in keys: + if key in mapping: + value = mapping.get(key) + if value not in (None, ""): + return value + return default + + +def _first_id_value(*values: Any) -> str: + for value in values: + if value in (None, ""): + continue + text = str(value).strip() + if text: + return text + return "" + + +def extract_artist_name(artist: Any) -> str: + if isinstance(artist, dict): + return str(artist.get("name", "") or "") + if hasattr(artist, "name"): + return str(getattr(artist, "name") or "") + return str(artist) if artist else "" + + +def normalize_import_context(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """Normalize an import context to neutral fields in place and drop legacy aliases.""" + if not isinstance(context, dict): + return {} + + artist = _as_dict(context.get("artist") or context.get("spotify_artist")) + album = _as_dict(context.get("album") or context.get("spotify_album")) + track_info = _as_dict(context.get("track_info")) + original_search = _as_dict(context.get("original_search_result")) + + context["artist"] = artist + context["album"] = album + context["track_info"] = track_info + context["original_search_result"] = original_search + context.pop("spotify_artist", None) + context.pop("spotify_album", None) + + for clean_key, legacy_key in ( + ("clean_title", "spotify_clean_title"), + ("clean_album", "spotify_clean_album"), + ("clean_artist", "spotify_clean_artist"), + ): + if clean_key not in original_search or original_search.get(clean_key) in (None, ""): + legacy_value = original_search.get(legacy_key) + if legacy_value not in (None, ""): + original_search[clean_key] = legacy_value + original_search.pop(legacy_key, None) + + has_clean = bool(context.get("has_clean_metadata", context.get("has_clean_spotify_data", False))) + has_full = bool(context.get("has_full_metadata", context.get("has_full_spotify_metadata", False))) + context["has_clean_metadata"] = has_clean + context["has_full_metadata"] = has_full + context.pop("has_clean_spotify_data", None) + context.pop("has_full_spotify_metadata", None) + + return context + + +def get_import_context_artist(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("artist") or context.get("spotify_artist")) + + +def get_import_context_album(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("album") or context.get("spotify_album")) + + +def get_import_track_info(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("track_info")) + + +def get_import_original_search(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("original_search_result")) + + +def get_import_source(context: Optional[Dict[str, Any]]) -> str: + if not isinstance(context, dict): + return "" + + source = context.get("source") + if source: + return str(source) + + track_info = get_import_track_info(context) + source = _first_value(track_info, "source", default="") + if source: + return str(source) + + original_search = get_import_original_search(context) + source = _first_value(original_search, "source", default="") + if source: + return str(source) + + album = get_import_context_album(context) + source = _first_value(album, "source", default="") + if source: + return str(source) + + artist = get_import_context_artist(context) + source = _first_value(artist, "source", default="") + return str(source) if source else "" + + +def get_import_clean_title( + context: Optional[Dict[str, Any]], + album_info: Optional[Dict[str, Any]] = None, + default: str = "Unknown Track", +) -> str: + original_search = get_import_original_search(context) + title = _first_value( + original_search, + "clean_title", + "title", + default="", + ) + if not title and album_info: + title = _first_value(album_info, "clean_track_name", "track_name", default="") + if not title: + track_info = get_import_track_info(context) + title = _first_value(track_info, "name", "title", default="") + return str(title or default) + + +def get_import_clean_album( + context: Optional[Dict[str, Any]], + album_info: Optional[Dict[str, Any]] = None, + default: str = "Unknown Album", +) -> str: + original_search = get_import_original_search(context) + album = _first_value( + original_search, + "clean_album", + "album", + default="", + ) + if not album and album_info: + album = _first_value(album_info, "album_name", "clean_album_name", default="") + if not album: + album_ctx = get_import_context_album(context) + album = _first_value(album_ctx, "name", default="") + return str(album or default) + + +def get_import_clean_artist(context: Optional[Dict[str, Any]], default: str = "Unknown Artist") -> str: + original_search = get_import_original_search(context) + artist = _first_value( + original_search, + "clean_artist", + "artist", + default="", + ) + if not artist: + artist_ctx = get_import_context_artist(context) + artist = _first_value(artist_ctx, "name", default="") + return str(artist or default) + + +def get_import_has_clean_metadata(context: Optional[Dict[str, Any]]) -> bool: + if not isinstance(context, dict): + return False + return bool(context.get("has_clean_metadata", False)) + + +def get_import_has_full_metadata(context: Optional[Dict[str, Any]]) -> bool: + if not isinstance(context, dict): + return False + return bool(context.get("has_full_metadata", False)) + + +def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]: + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + artist = get_import_context_artist(context) + album = get_import_context_album(context) + + return { + "track_id": _first_id_value( + _first_value(track_info, "id", "track_id", "trackId", "source_track_id", default=""), + _first_value(track_info, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""), + _first_value(original_search, "id", "track_id", "source_track_id", default=""), + _first_value(original_search, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""), + ), + "artist_id": _first_id_value( + _first_value(artist, "id", "artist_id", "source_artist_id", default=""), + _first_value(artist, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""), + _first_value(original_search, "artist_id", "source_artist_id", default=""), + _first_value(original_search, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""), + ), + "album_id": _first_id_value( + _first_value(album, "id", "album_id", "collectionId", "source_album_id", default=""), + _first_value(album, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), + _first_value(original_search, "album_id", "source_album_id", default=""), + _first_value(original_search, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), + _first_value(track_info, "album_id", "source_album_id", default=""), + _first_value(track_info, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), + ), + } + + +def get_source_tag_names(source: str) -> Dict[str, Optional[str]]: + source_name = (source or "").strip().lower() + if source_name == "spotify": + return {"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"} + if source_name == "itunes": + return {"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"} + if source_name == "deezer": + return {"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None} + if source_name == "hydrabase": + return {"track": None, "artist": None, "album": None} + if source_name == "discogs": + return {"track": None, "artist": None, "album": None} + return {"track": None, "artist": None, "album": None} + + +def get_library_source_id_columns(source: str) -> Dict[str, Optional[str]]: + source_name = (source or "").strip().lower() + if source_name == "spotify": + return {"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"} + if source_name == "itunes": + return {"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"} + if source_name == "deezer": + return {"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"} + if source_name == "hydrabase": + return {"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"} + if source_name == "discogs": + return {"artist": "discogs_id", "album": "discogs_id", "track": None} + return {} + + +def build_import_album_info( + context: Optional[Dict[str, Any]], + *, + album_info: Optional[Dict[str, Any]] = None, + force_album: bool = False, +) -> Dict[str, Any]: + """Build the album-info payload used by post-processing.""" + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + artist_ctx = get_import_context_artist(context) + + track_number = ( + (album_info or {}).get("track_number") + or track_info.get("track_number") + or original_search.get("track_number") + or 1 + ) + disc_number = ( + (album_info or {}).get("disc_number") + or track_info.get("disc_number") + or original_search.get("disc_number") + or 1 + ) + + clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track")) + album_name = get_import_clean_album(context, album_info=album_info, default=original_search.get("album", "Unknown Album")) + album_image_url = ( + (album_info or {}).get("album_image_url") + or album_ctx.get("image_url") + or "" + ) + total_tracks = ( + album_ctx.get("total_tracks") + or track_info.get("total_tracks") + or (album_info or {}).get("total_tracks") + or 0 + ) + album_type = (album_ctx.get("album_type") or track_info.get("album_type") or "album") + source = get_import_source(context) + + artist_name = artist_ctx.get("name") or original_search.get("artist") or get_import_clean_artist(context) + normalized_album = str(album_name or "").strip().lower() + normalized_title = str(clean_track_name or "").strip().lower() + normalized_artist = str(artist_name or "").strip().lower() + is_album = bool( + force_album + or ( + normalized_album + and total_tracks + and int(total_tracks) > 1 + and normalized_album != normalized_title + and normalized_album != normalized_artist + ) + ) + + return { + "is_album": is_album, + "album_name": album_name, + "track_number": int(track_number) if str(track_number).isdigit() else track_number, + "disc_number": int(disc_number) if str(disc_number).isdigit() else disc_number, + "clean_track_name": clean_track_name, + "album_image_url": album_image_url, + "confidence": (album_info or {}).get("confidence", 1.0 if is_album or force_album else 0.0), + "source": source, + "album_type": album_type, + "total_tracks": int(total_tracks) if str(total_tracks).isdigit() else total_tracks, + } diff --git a/core/import_file_ops.py b/core/import_file_ops.py new file mode 100644 index 00000000..261687a8 --- /dev/null +++ b/core/import_file_ops.py @@ -0,0 +1,475 @@ +"""Shared file and path helpers for import processing.""" + +from __future__ import annotations + +import logging +import os +import re +import shutil +import subprocess +import time +from pathlib import Path +from typing import Any, Dict, Optional + +logger = logging.getLogger("import_file_ops") + + +def _get_config_manager(): + from config.settings import config_manager + return config_manager + + +def safe_move_file(src, dst): + """Move a file safely across filesystems.""" + src = Path(src) + dst = Path(dst) + + dst.parent.mkdir(parents=True, exist_ok=True) + + if not src.exists(): + if dst.exists(): + logger.info(f"Source gone but destination exists, file already transferred: {dst.name}") + return + raise FileNotFoundError(f"Source file not found and destination does not exist: {src}") + + if dst.exists(): + for _attempt in range(3): + try: + dst.unlink() + break + except PermissionError: + if _attempt < 2: + time.sleep(1) + else: + logger.warning(f"Could not remove locked destination after 3 attempts: {dst.name}") + except Exception: + break + + try: + shutil.move(str(src), str(dst)) + return + except FileNotFoundError: + if dst.exists(): + logger.info(f"Source moved by another thread, destination exists: {dst.name}") + return + raise + except (OSError, PermissionError) as e: + error_msg = str(e).lower() + + if dst.exists() and dst.stat().st_size > 0: + logger.warning(f"Move raised {type(e).__name__} but destination exists, treating as success: {e}") + try: + src.unlink() + except Exception: + logger.info(f"Could not delete source file (may be owned by another process): {src}") + return + + if "cross-device" in error_msg or "operation not permitted" in error_msg or "permission denied" in error_msg: + logger.warning(f"Cross-device move detected, using fallback copy method: {e}") + try: + with open(src, "rb") as f_src: + with open(dst, "wb") as f_dst: + shutil.copyfileobj(f_src, f_dst) + f_dst.flush() + os.fsync(f_dst.fileno()) + + try: + src.unlink() + except PermissionError: + logger.info(f"Could not delete source file (may be owned by another process): {src}") + logger.info(f"Successfully moved file using fallback method: {src} -> {dst}") + return + except Exception as fallback_error: + logger.error(f"Fallback copy also failed: {fallback_error}") + raise + raise + + +def extract_track_number_from_filename(filename: str, title: str = None) -> int: + """Extract track number from a filename. Returns 1 if not found.""" + basename = os.path.splitext(os.path.basename(filename))[0].strip() + + match = re.match(r"^\d[\-\.](\d{1,2})\s*[\-\.]\s*", basename) + if match: + num = int(match.group(1)) + if 1 <= num <= 99: + return num + + match = re.match(r"^\(?(\d{1,3})\)?\s*[\-\.)\]]\s*", basename) + if match: + num = int(match.group(1)) + if 1 <= num <= 999: + return num + + return 1 + + +def _coerce_tag_number(value: Any, default: int = 1) -> int: + if value in (None, ""): + return default + + if isinstance(value, (list, tuple)): + value = value[0] if value else None + + if value in (None, ""): + return default + + text = str(value).strip() + if not text: + return default + + match = re.match(r"^(\d+)", text) + if match: + try: + return int(match.group(1)) + except ValueError: + return default + + try: + return int(text) + except (TypeError, ValueError): + return default + + +def read_staging_file_metadata(file_path: str, filename: Optional[str] = None) -> Dict[str, Any]: + """Read common audio tag metadata from a staging file.""" + try: + from mutagen import File as MutagenFile + + tags = MutagenFile(file_path, easy=True) + except Exception: + tags = None + + filename = filename or os.path.basename(file_path) + stem = os.path.splitext(os.path.basename(filename))[0] + + def _first_tag(*keys: str) -> str: + if not tags: + return "" + for key in keys: + try: + value = tags.get(key) # type: ignore[attr-defined] + except Exception: + value = None + if value: + if isinstance(value, (list, tuple)): + value = value[0] if value else "" + text = str(value).strip() + if text: + return text + return "" + + title = _first_tag("title") + artist = _first_tag("artist") + albumartist = _first_tag("albumartist") + album = _first_tag("album") + + if not title: + title = stem + if not albumartist: + albumartist = artist + + track_number = _coerce_tag_number(_first_tag("tracknumber", "track_number"), default=0) + if not track_number: + track_number = extract_track_number_from_filename(filename or file_path) + + disc_number = _coerce_tag_number(_first_tag("discnumber", "disc_number"), default=1) + + return { + "title": title, + "artist": artist, + "albumartist": albumartist, + "album": album, + "track_number": track_number, + "disc_number": disc_number, + } + + +def cleanup_empty_directories(download_path, moved_file_path): + """Remove empty directories after a move, ignoring hidden files.""" + try: + current_dir = os.path.dirname(moved_file_path) + while current_dir != download_path and current_dir.startswith(download_path): + is_empty = not any(not f.startswith(".") for f in os.listdir(current_dir)) + if is_empty: + logger.warning(f"Removing empty directory: {current_dir}") + os.rmdir(current_dir) + current_dir = os.path.dirname(current_dir) + else: + break + except Exception as e: + logger.error(f"An error occurred during directory cleanup: {e}") + + +def get_audio_quality_string(file_path): + """Return a compact audio quality string for the given file.""" + try: + ext = os.path.splitext(file_path)[1].lower() + + if ext == ".flac": + from mutagen.flac import FLAC + audio = FLAC(file_path) + return f"FLAC {audio.info.bits_per_sample}bit" + + if ext == ".mp3": + from mutagen.mp3 import MP3, BitrateMode + + audio = MP3(file_path) + bitrate_kbps = audio.info.bitrate // 1000 + if audio.info.bitrate_mode == BitrateMode.VBR: + return "MP3-VBR" + return f"MP3-{bitrate_kbps}" + + if ext in (".m4a", ".aac", ".mp4"): + from mutagen.mp4 import MP4 + audio = MP4(file_path) + return f"M4A-{audio.info.bitrate // 1000}" + + if ext == ".ogg": + from mutagen.oggvorbis import OggVorbis + audio = OggVorbis(file_path) + return f"OGG-{audio.info.bitrate // 1000}" + + if ext == ".opus": + from mutagen.oggopus import OggOpus + + audio = OggOpus(file_path) + return f"OPUS-{audio.info.bitrate // 1000}" + + return "" + except Exception as e: + logger.debug(f"Could not determine audio quality for {file_path}: {e}") + return "" + + +def get_quality_tier_from_extension(file_path): + """Classify a file extension into a quality tier.""" + if not file_path: + return ("unknown", 999) + + ext = os.path.splitext(file_path)[1].lower() + quality_tiers = { + "lossless": { + "extensions": [".flac", ".ape", ".wav", ".alac", ".dsf", ".dff", ".aiff", ".aif"], + "tier": 1, + }, + "high_lossy": { + "extensions": [".opus", ".ogg"], + "tier": 2, + }, + "standard_lossy": { + "extensions": [".m4a", ".aac"], + "tier": 3, + }, + "low_lossy": { + "extensions": [".mp3", ".wma"], + "tier": 4, + }, + } + + for tier_name, tier_data in quality_tiers.items(): + if ext in tier_data["extensions"]: + return (tier_name, tier_data["tier"]) + + return ("unknown", 999) + + +def downsample_hires_flac(final_path, context): + """Downsample a hi-res FLAC to 16-bit/44.1kHz if enabled.""" + from mutagen.flac import FLAC + + config_manager = _get_config_manager() + if not config_manager.get("lossy_copy.downsample_hires", False): + return None + + if os.path.splitext(final_path)[1].lower() != ".flac": + return None + + try: + audio = FLAC(final_path) + original_bits = audio.info.bits_per_sample + original_rate = audio.info.sample_rate + except Exception as e: + logger.error(f"[Downsample] Could not read FLAC info: {e}") + return None + + if original_bits <= 16 and original_rate <= 44100: + return None + + logger.info(f"[Downsample] Converting {original_bits}-bit/{original_rate}Hz -> 16-bit/44100Hz: {os.path.basename(final_path)}") + + ffmpeg_bin = shutil.which("ffmpeg") + if not ffmpeg_bin: + local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg") + if os.path.isfile(local): + ffmpeg_bin = local + else: + logger.warning("[Downsample] ffmpeg not found - skipping hi-res conversion") + return None + + temp_path = final_path + ".tmp.flac" + try: + result = subprocess.run( + [ + ffmpeg_bin, "-i", final_path, + "-sample_fmt", "s16", + "-ar", "44100", + "-map_metadata", "0", + "-compression_level", "8", + "-y", temp_path, + ], + capture_output=True, + text=True, + timeout=300, + ) + + if result.returncode != 0: + logger.error(f"[Downsample] ffmpeg failed: {result.stderr[:200]}") + if os.path.exists(temp_path): + os.remove(temp_path) + return None + + if not os.path.isfile(temp_path) or os.path.getsize(temp_path) == 0: + logger.warning("[Downsample] Output file missing or empty") + if os.path.exists(temp_path): + os.remove(temp_path) + return None + + verify_audio = FLAC(temp_path) + if verify_audio.info.bits_per_sample != 16: + logger.info(f"[Downsample] Output not 16-bit ({verify_audio.info.bits_per_sample}-bit), aborting") + os.remove(temp_path) + return None + + os.replace(temp_path, final_path) + logger.info(f"[Downsample] Converted to 16-bit/44.1kHz: {os.path.basename(final_path)}") + + new_quality = "FLAC 16bit" + try: + updated_audio = FLAC(final_path) + updated_audio["QUALITY"] = new_quality + updated_audio.save() + except Exception as tag_err: + logger.error(f"[Downsample] Could not update QUALITY tag: {tag_err}") + + old_quality = context.get("_audio_quality", "") + context["_audio_quality"] = new_quality + + if old_quality and old_quality != new_quality and old_quality in os.path.basename(final_path): + new_basename = os.path.basename(final_path).replace(old_quality, new_quality) + new_path = os.path.join(os.path.dirname(final_path), new_basename) + try: + os.rename(final_path, new_path) + logger.info(f"[Downsample] Renamed: {os.path.basename(final_path)} -> {new_basename}") + for lyrics_ext in (".lrc", ".txt"): + old_lyrics = os.path.splitext(final_path)[0] + lyrics_ext + if os.path.isfile(old_lyrics): + new_lyrics = os.path.splitext(new_path)[0] + lyrics_ext + os.rename(old_lyrics, new_lyrics) + return new_path + except Exception as rename_err: + logger.error(f"[Downsample] Could not rename file: {rename_err}") + + return final_path + except subprocess.TimeoutExpired: + logger.info(f"[Downsample] Conversion timed out for: {os.path.basename(final_path)}") + if os.path.exists(temp_path): + os.remove(temp_path) + except Exception as e: + logger.error(f"[Downsample] Conversion error: {e}") + if os.path.exists(temp_path): + try: + os.remove(temp_path) + except Exception: + pass + return None + + +def create_lossy_copy(final_path): + """Convert a FLAC file to a lossy copy using the configured codec.""" + from mutagen.flac import FLAC + + config_manager = _get_config_manager() + if not config_manager.get("lossy_copy.enabled", False): + return None + + if os.path.splitext(final_path)[1].lower() != ".flac": + return None + + codec = config_manager.get("lossy_copy.codec", "mp3").lower() + bitrate = config_manager.get("lossy_copy.bitrate", "320") + + if codec == "opus" and int(bitrate) > 256: + bitrate = "256" + + codec_map = { + "mp3": ("libmp3lame", ".mp3", f"MP3-{bitrate}", ["-vn", "-id3v2_version", "3"]), + "opus": ("libopus", ".opus", f"OPUS-{bitrate}", ["-vn", "-map", "0:a", "-vbr", "on"]), + "aac": ("aac", ".m4a", f"AAC-{bitrate}", ["-vn", "-movflags", "+faststart"]), + } + + if codec not in codec_map: + logger.info(f"[Lossy Copy] Unknown codec '{codec}' - skipping conversion") + return None + + ffmpeg_codec, out_ext, quality_label, extra_args = codec_map[codec] + out_path = os.path.splitext(final_path)[0] + out_ext + + original_quality = get_audio_quality_string(final_path) + if original_quality: + out_basename = os.path.basename(out_path) + if original_quality in out_basename: + out_basename = out_basename.replace(original_quality, quality_label) + out_path = os.path.join(os.path.dirname(out_path), out_basename) + + ffmpeg_bin = shutil.which("ffmpeg") + if not ffmpeg_bin: + local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg") + if os.path.isfile(local): + ffmpeg_bin = local + else: + logger.warning(f"[Lossy Copy] ffmpeg not found - skipping {codec.upper()} conversion") + return None + + try: + logger.info(f"[Lossy Copy] Converting to {quality_label}: {os.path.basename(final_path)}") + cmd = [ + ffmpeg_bin, "-i", final_path, + "-codec:a", ffmpeg_codec, + "-b:a", f"{bitrate}k", + "-map_metadata", "0", + ] + extra_args + ["-y", out_path] + + result = subprocess.run(cmd, capture_output=True, text=True, timeout=120) + + if result.returncode == 0: + logger.info(f"[Lossy Copy] Created {quality_label} copy: {os.path.basename(out_path)}") + try: + from mutagen import File as MutagenFile + audio = MutagenFile(out_path) + if audio is not None: + if codec == "mp3": + from mutagen.id3 import TXXX + audio.tags.add(TXXX(encoding=3, desc="QUALITY", text=[quality_label])) + elif codec == "opus": + audio["QUALITY"] = [quality_label] + elif codec == "aac": + from mutagen.mp4 import MP4FreeForm + audio["----:com.apple.iTunes:QUALITY"] = [MP4FreeForm(quality_label.encode("utf-8"))] + audio.save() + except Exception as tag_err: + logger.error(f"[Lossy Copy] Could not update QUALITY tag: {tag_err}") + return out_path + + logger.error(f"[Lossy Copy] ffmpeg failed: {result.stderr[:200]}") + if os.path.exists(out_path): + try: + os.remove(out_path) + except Exception: + pass + return None + except subprocess.TimeoutExpired: + logger.warning(f"[Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}") + except Exception as e: + logger.error(f"[Lossy Copy] Conversion error: {e}") + return None diff --git a/core/import_filename.py b/core/import_filename.py new file mode 100644 index 00000000..9298750d --- /dev/null +++ b/core/import_filename.py @@ -0,0 +1,73 @@ +"""Filename parsing helpers used by import flows.""" + +from __future__ import annotations + +import os +import re +from typing import Any, Dict + + +_TRACK_PATTERNS = ( + r"^(\d+)\s*[-\.]\s*(.+?)\s*[-–]\s*(.+)$", + r"^(.+?)\s*[-–]\s*(.+)$", + r"^(\d+)\s*[-\.]\s*(.+)$", +) + + +def parse_filename_metadata(filename: str) -> Dict[str, Any]: + """Extract artist/title/album hints from a loose filename.""" + raw_path = str(filename or "") + normalized_path = raw_path.replace("\\", "/") + base_name = os.path.splitext(os.path.basename(normalized_path))[0] + + result: Dict[str, Any] = { + "artist": "", + "title": "", + "album": "", + "track_number": None, + } + + if not base_name: + return result + + for pattern in _TRACK_PATTERNS: + match = re.match(pattern, base_name) + if not match: + continue + + groups = match.groups() + if len(groups) == 3: + try: + result["track_number"] = int(groups[0]) + result["artist"] = result["artist"] or groups[1].strip() + result["title"] = result["title"] or groups[2].strip() + except ValueError: + result["artist"] = result["artist"] or groups[0].strip() + result["title"] = result["title"] or f"{groups[1]} - {groups[2]}".strip() + elif len(groups) == 2: + if groups[0].isdigit(): + try: + result["track_number"] = int(groups[0]) + result["title"] = result["title"] or groups[1].strip() + except ValueError: + pass + else: + result["artist"] = result["artist"] or groups[0].strip() + result["title"] = result["title"] or groups[1].strip() + break + + if not result["title"]: + result["title"] = base_name + + if not result["album"] and "/" in normalized_path: + path_parts = normalized_path.split("/") + for part in reversed(path_parts[:-1]): + if not part or part.startswith("@"): + continue + + cleaned = re.sub(r"^\d+\s*[-\.]\s*", "", part).strip() + if len(cleaned) > 3: + result["album"] = cleaned + break + + return result diff --git a/core/import_guards.py b/core/import_guards.py new file mode 100644 index 00000000..de2a3185 --- /dev/null +++ b/core/import_guards.py @@ -0,0 +1,121 @@ +"""Import post-processing guards and quarantine helpers.""" + +from __future__ import annotations + +import json +import os +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Optional + +from core.import_context import ( + get_import_clean_artist, + get_import_clean_title, + get_import_context_artist, + get_import_original_search, + get_import_track_info, + normalize_import_context, +) +from core.import_file_ops import safe_move_file +from database.music_database import MusicDatabase +from utils.logging_config import get_logger + + +logger = get_logger("import_guards") + + +def _get_config_manager(): + from config.settings import config_manager + + return config_manager + + +def move_to_quarantine(file_path: str, context: dict, reason: str, automation_engine=None) -> str: + """Move a file to the quarantine folder and write a metadata sidecar.""" + config_manager = _get_config_manager() + download_dir = config_manager.get("soulseek.download_path", "./downloads") + quarantine_dir = Path(download_dir) / "ss_quarantine" + quarantine_dir.mkdir(parents=True, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + original_name = Path(file_path).stem + file_ext = Path(file_path).suffix + + quarantine_filename = f"{timestamp}_{original_name}{file_ext}.quarantined" + quarantine_path = quarantine_dir / quarantine_filename + + safe_move_file(file_path, str(quarantine_path)) + + metadata_path = quarantine_dir / f"{timestamp}_{original_name}.json" + context = normalize_import_context(context) + original_search = get_import_original_search(context) + artist_context = get_import_context_artist(context) + + metadata = { + "original_filename": Path(file_path).name, + "quarantine_reason": reason, + "timestamp": datetime.now().isoformat(), + "expected_track": get_import_clean_title(context, default=original_search.get("title", "Unknown")), + "expected_artist": get_import_clean_artist(context, default=(artist_context.get("name", "") if isinstance(artist_context, dict) else "Unknown")), + "context_key": context.get("context_key", "unknown"), + } + + try: + with open(metadata_path, "w", encoding="utf-8") as f: + json.dump(metadata, f, indent=2, ensure_ascii=False) + except Exception as exc: + logger.warning("Failed to write quarantine metadata: %s", exc) + + logger.warning("File quarantined: %s - Reason: %s", quarantine_path, reason) + + if automation_engine: + try: + ti = context.get("track_info", {}) + artists = ti.get("artists", []) + artist_name = "" + if artists: + first = artists[0] + artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first) + automation_engine.emit( + "download_quarantined", + { + "artist": artist_name, + "title": ti.get("name", ""), + "reason": reason or "Unknown", + }, + ) + except Exception: + pass + + return str(quarantine_path) + + +def check_flac_bit_depth(file_path: str, context: dict) -> Optional[str]: + """Return a rejection message if a FLAC file violates the configured bit depth.""" + if not context.get("_audio_quality", "").startswith("FLAC"): + return None + + config_manager = _get_config_manager() + quality_profile = MusicDatabase().get_quality_profile() + flac_config = quality_profile.get("qualities", {}).get("flac", {}) + flac_pref = flac_config.get("bit_depth", "any") + if flac_pref == "any": + return None + + actual_bits = context["_audio_quality"].replace("FLAC ", "").replace("bit", "") + if actual_bits == flac_pref: + return None + + flac_fallback = flac_config.get("bit_depth_fallback", True) + downsample_enabled = config_manager.get("lossy_copy.downsample_hires", False) + track_info = context.get("track_info", {}) + track_name = track_info.get("name", os.path.basename(file_path)) + + if flac_fallback or downsample_enabled: + if downsample_enabled: + logger.info("[FLAC Downsample] Accepted %s-bit FLAC (will be downsampled to %s-bit): %s", actual_bits, flac_pref, track_name) + else: + logger.warning("[FLAC Fallback] Accepted %s-bit FLAC (preferred %s-bit): %s", actual_bits, flac_pref, track_name) + return None + + return f"FLAC bit depth mismatch: file is {actual_bits}-bit, preference is {flac_pref}-bit" diff --git a/core/import_paths.py b/core/import_paths.py new file mode 100644 index 00000000..7848425c --- /dev/null +++ b/core/import_paths.py @@ -0,0 +1,757 @@ +"""Shared path and naming helpers for import processing.""" + +from __future__ import annotations + +import json +import logging +import os +import re +import threading +from pathlib import Path +from typing import Any + +from core.import_context import ( + get_import_clean_title, + get_import_context_album, + get_import_original_search, + get_import_source, + get_import_track_info, + normalize_import_context, +) + +logger = logging.getLogger("import_paths") + +_album_cache_lock = threading.Lock() +_album_editions: dict[str, str] = {} +_album_name_cache: dict[str, str] = {} + + +def _get_config_manager(): + try: + from config.settings import config_manager + return config_manager + except Exception: + class _FallbackConfig: + @staticmethod + def get(key, default=None): + return default + + return _FallbackConfig() + + +def _get_itunes_client(): + try: + from core.metadata_service import get_itunes_client + return get_itunes_client() + except Exception: + return None + + +def _get_album_tracks_for_source(source: str, album_id: str): + try: + from core.metadata_service import get_album_tracks_for_source + return get_album_tracks_for_source(source, album_id) + except Exception: + return None + + +def _extract_artist_name(artist_context: Any) -> str: + if not artist_context: + return "" + if isinstance(artist_context, dict): + return str(artist_context.get("name", "") or "").strip() + return str(artist_context).strip() + + +def docker_resolve_path(path_str: str) -> str: + """Resolve Docker-hosted Windows paths into container paths.""" + if os.path.exists("/.dockerenv") and len(path_str) >= 3 and path_str[1] == ":" and path_str[0].isalpha(): + drive_letter = path_str[0].lower() + rest_of_path = path_str[2:].replace("\\", "/") + return f"/host/mnt/{drive_letter}{rest_of_path}" + return path_str + + +def build_simple_download_destination(context, file_path: str): + """Build the destination path for a simple download into Transfer.""" + context = normalize_import_context(context) + search_result = context.get("search_result", {}) or {} + if not isinstance(search_result, dict): + search_result = {} + + transfer_dir = Path(docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer"))) + album_name = None + original_filename = search_result.get("filename", "") + if "/" in original_filename or "\\" in original_filename: + path_parts = original_filename.replace("\\", "/").split("/") + if len(path_parts) >= 2: + album_name = path_parts[-2] + if not album_name: + album_value = search_result.get("album") + if isinstance(album_value, dict): + album_name = album_value.get("name", "") + else: + album_name = album_value + + filename = Path(file_path).name + if album_name and str(album_name).lower() not in {"unknown", "unknown album", ""}: + album_name = sanitize_filename(str(album_name)) + destination_dir = transfer_dir / album_name + else: + album_name = "" + destination_dir = transfer_dir + + destination_dir.mkdir(parents=True, exist_ok=True) + return destination_dir / filename, album_name, filename + + +def sanitize_filename(filename: str) -> str: + """Sanitize filename for file system compatibility.""" + sanitized = re.sub(r'[<>:"/\\|?*]', "_", filename) + sanitized = re.sub(r"\s+", " ", sanitized).strip() + sanitized = sanitized.rstrip(". ") or "_" + if re.match(r"^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)", sanitized, re.IGNORECASE): + sanitized = "_" + sanitized + return sanitized[:200] + + +def sanitize_context_values(context: dict) -> dict: + """Sanitize all string values in a template context for path safety.""" + sanitized = {} + for key, value in context.items(): + if isinstance(value, str) and value: + sanitized[key] = sanitize_filename(value) + else: + sanitized[key] = value + return sanitized + + +def clean_track_title(track_title: str, artist_name: str) -> str: + """Clean up track title by removing artist prefix and other noise.""" + original = (track_title or "").strip() + cleaned = original + cleaned = re.sub(r"^\d{1,2}[\.\s\-]+", "", cleaned) + artist_pattern = re.escape(artist_name or "") + r"\s*-\s*" + cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"^[A-Za-z0-9\.]+\s*-\s*\d{1,2}\s*-\s*", "", cleaned) + quality_patterns = [ + r"\s*[\[\(][0-9]+\s*kbps[\]\)]\s*", + r"\s*[\[\(]flac[\]\)]\s*", + r"\s*[\[\(]mp3[\]\)]\s*", + ] + for pattern in quality_patterns: + cleaned = re.sub(pattern, "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"^[-\s\.]+", "", cleaned) + cleaned = re.sub(r"[-\s\.]+$", "", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned).strip() + return cleaned if cleaned else original + + +def get_base_album_name(album_name: str) -> str: + """Extract the base album name without edition indicators.""" + base_name = album_name or "" + base_name = re.sub( + r"\s*[\[\(][^)\]]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^)\]]*[\]\)]\s*$", + "", + base_name, + flags=re.IGNORECASE, + ) + base_name = re.sub(r"\s*[\[\(][^)\]]*\bedition\b[^)\]]*[\]\)]\s*$", "", base_name, flags=re.IGNORECASE) + base_name = re.sub( + r"\s+(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", + "", + base_name, + flags=re.IGNORECASE, + ) + return base_name.strip() + + +def detect_deluxe_edition(album_name: str) -> bool: + """Detect if an album name indicates a deluxe/special edition.""" + if not album_name: + return False + + album_lower = album_name.lower() + deluxe_indicators = [ + "deluxe", + "deluxe edition", + "special edition", + "expanded edition", + "extended edition", + "bonus", + "remastered", + "anniversary", + "collectors edition", + "limited edition", + "silver edition", + "gold edition", + "platinum edition", + ] + for indicator in deluxe_indicators: + if indicator in album_lower: + logger.info("Detected deluxe edition: %r contains %r", album_name, indicator) + return True + return False + + +def normalize_base_album_name(base_album: str, artist_name: str) -> str: + """Normalize the base album name to handle case variations and known corrections.""" + normalized_lower = (base_album or "").lower().strip() + known_corrections = { + # Add specific album name corrections here as needed. + } + + for variant, correction in known_corrections.items(): + if normalized_lower == variant.lower(): + logger.info("Album correction applied: %r -> %r", base_album, correction) + return correction + + normalized = base_album or "" + normalized = re.sub(r"\s*&\s*", " & ", normalized) + normalized = re.sub(r"\s+", " ", normalized) + normalized = normalized.strip() + logger.info("Album variant normalization: %r -> %r", base_album, normalized) + return normalized + + +def clean_album_title(album_title: str, artist_name: str) -> str: + """Clean up album title by removing common prefixes, suffixes, and artist redundancy.""" + original = (album_title or "").strip() + cleaned = original + logger.info("Album Title Cleaning: %r (artist: %r)", original, artist_name) + + cleaned = re.sub(r"^Album\s*-\s*", "", cleaned, flags=re.IGNORECASE) + artist_pattern = re.escape(artist_name or "") + r"\s*-\s*" + cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*[\[\(]\d{4}[\]\)]\s*", " ", cleaned) + + quality_patterns = [ + r"\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*", + r"\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*", + r"\s*[\[\(].*?flac.*?[\]\)]\s*", + r"\s*[\[\(].*?mp3.*?[\]\)]\s*", + r"\s*[\[\(].*?itunes.*?[\]\)]\s*", + r"\s*[\[\(].*?web.*?[\]\)]\s*", + r"\s*[\[\(].*?cd.*?[\]\)]\s*", + ] + for pattern in quality_patterns: + cleaned = re.sub(pattern, " ", cleaned, flags=re.IGNORECASE) + + cleaned = re.sub(r"\s*[\[\(][^\]\)]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*[\[\(][^\]\)]*\bedition\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"^[-\s\.]+", "", cleaned) + cleaned = re.sub(r"[-\s\.]+$", "", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned).strip() + return cleaned if cleaned else original + + +def resolve_album_group(artist_context: dict, album_info: dict, original_album: str = None) -> str: + """Smart album grouping: upgrade to deluxe if any track is deluxe.""" + try: + with _album_cache_lock: + artist_name = _extract_artist_name(artist_context) + detected_album = (album_info or {}).get("album_name", "") + + if detected_album: + base_album = get_base_album_name(detected_album) + elif original_album: + cleaned_original = clean_album_title(original_album, artist_name) + base_album = get_base_album_name(cleaned_original) + else: + base_album = get_base_album_name(detected_album) + + base_album = normalize_base_album_name(base_album, artist_name) + album_key = f"{artist_name}::{base_album}" + is_deluxe_track = False + if detected_album: + is_deluxe_track = detect_deluxe_edition(detected_album) + elif original_album: + is_deluxe_track = detect_deluxe_edition(original_album) + + if album_key in _album_name_cache: + cached_name = _album_name_cache[album_key] + current_edition = _album_editions.get(album_key, "standard") + if is_deluxe_track and current_edition == "standard": + final_album_name = f"{base_album} (Deluxe Edition)" + _album_editions[album_key] = "deluxe" + _album_name_cache[album_key] = final_album_name + logger.info("Album cache upgrade: %r -> %r", album_key, final_album_name) + return final_album_name + logger.info("Using cached album name for %r: %r", album_key, cached_name) + return cached_name + + logger.info("Album grouping - Key: %r, Detected: %r", album_key, detected_album) + + current_edition = _album_editions.get(album_key, "standard") + if is_deluxe_track and current_edition == "standard": + logger.info("UPGRADE: Album %r upgraded from standard to deluxe!", base_album) + _album_editions[album_key] = "deluxe" + current_edition = "deluxe" + + if current_edition == "deluxe": + final_album_name = f"{base_album} (Deluxe Edition)" + else: + final_album_name = base_album + + _album_name_cache[album_key] = final_album_name + + logger.info("Album resolution: %r -> %r (edition: %s)", detected_album, final_album_name, current_edition) + return final_album_name + except Exception as e: + logger.error("Error resolving album group: %s", e) + album_name = (album_info or {}).get("album_name", "Unknown Album") + return album_name + + +def get_album_type_display(raw_type, track_count) -> str: + """Return the display form of an album's type for the $albumtype template variable.""" + raw = (raw_type or "").strip().lower() + try: + tc = int(track_count or 0) + except (TypeError, ValueError): + tc = 0 + + if raw in ("compilation", "compile"): + return "Compilation" + if raw == "album": + return "Album" + if raw in ("single", "ep"): + if tc <= 3: + return "Single" + if tc <= 6: + return "EP" + return "Album" + + if tc <= 0: + return "Album" + if tc <= 3: + return "Single" + if tc <= 6: + return "EP" + return "Album" + + +def _replace_template_variables(template: str, context: dict) -> str: + clean_context = sanitize_context_values(context) + result = template + + album_artist_value = clean_context.get("albumartist", clean_context.get("artist", "Unknown Artist")) + collab_mode = _get_config_manager().get("file_organization.collab_artist_mode", "first") + if collab_mode == "first" and album_artist_value: + artists_list = context.get("_artists_list") + if artists_list and len(artists_list) > 1: + first = artists_list[0] + album_artist_value = first.get("name", first) if isinstance(first, dict) else str(first) + elif artists_list and len(artists_list) == 1: + itunes_artist_id = context.get("_itunes_artist_id") + if itunes_artist_id and ("," in album_artist_value or " & " in album_artist_value): + try: + resolved_client = _get_itunes_client() + if resolved_client and hasattr(resolved_client, "resolve_primary_artist"): + resolved = resolved_client.resolve_primary_artist(itunes_artist_id) + if resolved and resolved != album_artist_value: + album_artist_value = resolved + except Exception: + pass + + bracket_map = { + "albumartist": album_artist_value, + "albumtype": clean_context.get("albumtype", "Album"), + "playlist": clean_context.get("playlist_name", ""), + "artistletter": (clean_context.get("artist", "U") or "U")[0].upper(), + "artist": clean_context.get("artist", "Unknown Artist"), + "album": clean_context.get("album", "Unknown Album"), + "title": clean_context.get("title", "Unknown Track"), + "track": f"{_coerce_int(clean_context.get('track_number', 1), 1):02d}", + "disc": str(_coerce_int(clean_context.get("disc_number", 1), 1)), + "discnum": str(_coerce_int(clean_context.get("disc_number", 1), 1)), + "year": str(clean_context.get("year", "")), + "quality": clean_context.get("quality", ""), + } + for var_name, val in bracket_map.items(): + result = result.replace("${" + var_name + "}", val) + + result = result.replace("$albumartist", album_artist_value) + result = result.replace("$albumtype", clean_context.get("albumtype", "Album")) + result = result.replace("$playlist", clean_context.get("playlist_name", "")) + result = result.replace("$artistletter", (clean_context.get("artist", "U") or "U")[0].upper()) + result = result.replace("$artist", clean_context.get("artist", "Unknown Artist")) + result = result.replace("$album", clean_context.get("album", "Unknown Album")) + result = result.replace("$title", clean_context.get("title", "Unknown Track")) + result = result.replace("$track", f"{clean_context.get('track_number', 1):02d}") + result = result.replace("$year", str(clean_context.get("year", ""))) + + result = re.sub(r"\s+", " ", result) + result = re.sub(r"\s*-\s*-\s*", " - ", result) + result = result.strip() + return result + + +def apply_path_template(template: str, context: dict) -> str: + """Apply a template to build a path string.""" + return _replace_template_variables(template, context) + + +def get_file_path_from_template_raw(template: str, context: dict) -> tuple[str, str]: + """Build file path using a user-provided template string directly.""" + full_path = apply_path_template(template, context) + + quality_value = context.get("quality", "") + disc_number = _coerce_int(context.get("disc_number", 1), 1) + disc_value = f"{disc_number:02d}" + disc_value_raw = str(disc_number) + + path_parts = full_path.split("/") + if len(path_parts) > 1: + folder_parts = path_parts[:-1] + filename_base = path_parts[-1] + + cleaned_folders = [] + for part in folder_parts: + part = part.replace("$quality", "") + part = part.replace("$discnum", "") + part = part.replace("$disc", "") + part = re.sub(r"\s*\[\s*\]", "", part) + part = re.sub(r"\s*\(\s*\)", "", part) + part = re.sub(r"\s*\{\s*\}", "", part) + part = re.sub(r"\s*-\s*$", "", part) + part = re.sub(r"^\s*-\s*", "", part) + part = re.sub(r"\s+", " ", part).strip() + if part: + cleaned_folders.append(part) + + filename_base = filename_base.replace("$quality", quality_value) + filename_base = filename_base.replace("$discnum", disc_value_raw) + filename_base = filename_base.replace("$disc", disc_value) + filename_base = re.sub(r"\s*\[\s*\]", "", filename_base) + filename_base = re.sub(r"\s*\(\s*\)", "", filename_base) + filename_base = re.sub(r"\s*\{\s*\}", "", filename_base) + filename_base = re.sub(r"\s*-\s*$", "", filename_base) + filename_base = re.sub(r"\s+", " ", filename_base).strip() + + sanitized_folders = [sanitize_filename(part) for part in cleaned_folders] + folder_path = os.path.join(*sanitized_folders) if sanitized_folders else "" + return folder_path, sanitize_filename(filename_base) + + full_path = full_path.replace("$quality", quality_value) + full_path = full_path.replace("$discnum", disc_value_raw) + full_path = full_path.replace("$disc", disc_value) + full_path = re.sub(r"\s*\[\s*\]", "", full_path) + full_path = re.sub(r"\s*\(\s*\)", "", full_path) + full_path = re.sub(r"\s*\{\s*\}", "", full_path) + full_path = re.sub(r"\s*-\s*$", "", full_path) + full_path = re.sub(r"\s+", " ", full_path).strip() + return "", sanitize_filename(full_path) + + +def get_file_path_from_template(context: dict, template_type: str = "album_path") -> tuple[str, str]: + """Build complete file path using configured templates.""" + if not _get_config_manager().get("file_organization.enabled", True): + return None, None + + templates = _get_config_manager().get("file_organization.templates", {}) + template = templates.get(template_type) + if not template: + default_templates = { + "album_path": "$albumartist/$albumartist - $album/$track - $title", + "single_path": "$artist/$artist - $title/$title", + "compilation_path": "Compilations/$album/$track - $artist - $title", + "playlist_path": "$playlist/$artist - $title", + } + template = default_templates.get(template_type, "$artist/$album/$track - $title") + + full_path = apply_path_template(template, context) + + path_parts = full_path.split("/") + quality_value = context.get("quality", "") + disc_number = _coerce_int(context.get("disc_number", 1), 1) + disc_value = f"{disc_number:02d}" + disc_value_raw = str(disc_number) + + if len(path_parts) > 1: + folder_parts = path_parts[:-1] + filename_base = path_parts[-1] + + cleaned_folders = [] + for part in folder_parts: + part = part.replace("$quality", "") + part = part.replace("$discnum", "") + part = part.replace("$disc", "") + part = re.sub(r"\s*\[\s*\]", "", part) + part = re.sub(r"\s*\(\s*\)", "", part) + part = re.sub(r"\s*\{\s*\}", "", part) + part = re.sub(r"\s*-\s*$", "", part) + part = re.sub(r"^\s*-\s*", "", part) + part = re.sub(r"\s+", " ", part).strip() + if part: + cleaned_folders.append(part) + + filename_base = filename_base.replace("$quality", quality_value) + filename_base = filename_base.replace("$discnum", disc_value_raw) + filename_base = filename_base.replace("$disc", disc_value) + filename_base = re.sub(r"\s*\[\s*\]", "", filename_base) + filename_base = re.sub(r"\s*\(\s*\)", "", filename_base) + filename_base = re.sub(r"\s*\{\s*\}", "", filename_base) + filename_base = re.sub(r"\s*-\s*$", "", filename_base) + filename_base = re.sub(r"\s+", " ", filename_base).strip() + + sanitized_folders = [sanitize_filename(part) for part in cleaned_folders] + folder_path = os.path.join(*sanitized_folders) if sanitized_folders else "" + filename = sanitize_filename(filename_base) + return folder_path, filename + + full_path = full_path.replace("$quality", quality_value) + full_path = full_path.replace("$discnum", disc_value_raw) + full_path = full_path.replace("$disc", disc_value) + full_path = re.sub(r"\s*\[\s*\]", "", full_path) + full_path = re.sub(r"\s*\(\s*\)", "", full_path) + full_path = re.sub(r"\s*\{\s*\}", "", full_path) + full_path = re.sub(r"\s*-\s*$", "", full_path) + full_path = re.sub(r"\s+", " ", full_path).strip() + return "", sanitize_filename(full_path) + + +def _max_disc_number(album_tracks: Any) -> int: + items = [] + if isinstance(album_tracks, dict): + items = album_tracks.get("items") or album_tracks.get("tracks") or [] + elif isinstance(album_tracks, list): + items = album_tracks + + max_disc = 1 + for track in items: + if not isinstance(track, dict): + continue + try: + disc_number = int(track.get("disc_number", 1) or 1) + except (TypeError, ValueError): + disc_number = 1 + if disc_number > max_disc: + max_disc = disc_number + return max_disc + + +def _coerce_int(value: Any, default: int = 1) -> int: + try: + coerced = int(value) + except (TypeError, ValueError): + return default + return coerced if coerced > 0 else default + + +def build_final_path_for_track(context, artist_context, album_info, file_ext): + """Shared path builder used by both post-processing and verification.""" + transfer_dir = docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer")) + context = normalize_import_context(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + album_context = get_import_context_album(context) + source = get_import_source(context) + playlist_folder_mode = track_info.get("_playlist_folder_mode", False) + artist_name = _extract_artist_name(artist_context) + + source_info = track_info.get("source_info") or {} + if isinstance(source_info, str): + try: + source_info = json.loads(source_info) + except (json.JSONDecodeError, TypeError): + source_info = {} + if source_info.get("enhance") and source_info.get("original_file_path"): + original_path = source_info["original_file_path"] + original_dir = os.path.dirname(original_path) + original_stem = os.path.splitext(os.path.basename(original_path))[0] + final_path = os.path.join(original_dir, original_stem + file_ext) + os.makedirs(original_dir, exist_ok=True) + logger.info("[Enhance] Using original file location: %s", final_path) + return final_path, True + + year = "" + if album_context and album_context.get("release_date"): + release_date = album_context["release_date"] + if release_date and len(release_date) >= 4: + year = release_date[:4] + + raw_album_type = "" + if album_context: + raw_album_type = album_context.get("album_type", "") or "" + total_tracks = (album_context.get("total_tracks", 0) or 0) if album_context else 0 + album_type_display = get_album_type_display(raw_album_type, total_tracks) + + if playlist_folder_mode: + playlist_name = track_info.get("_playlist_name", "Unknown Playlist") + track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track")) + _artists = original_search.get("artists") or track_info.get("artists") or [] + + template_context = { + "artist": artist_name, + "albumartist": artist_name, + "album": track_name, + "title": track_name, + "playlist_name": playlist_name, + "track_number": 1, + "disc_number": 1, + "year": year, + "quality": context.get("_audio_quality", ""), + "albumtype": album_type_display, + "_artists_list": _artists, + "_itunes_artist_id": str(artist_context.get("id", "")) if isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source == "itunes" else None, + } + + folder_path, filename_base = get_file_path_from_template(template_context, "playlist_path") + if folder_path and filename_base: + final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + return final_path, True + + playlist_name_sanitized = sanitize_filename(playlist_name) + playlist_dir = os.path.join(transfer_dir, playlist_name_sanitized) + os.makedirs(playlist_dir, exist_ok=True) + artist_name_sanitized = sanitize_filename(template_context["artist"]) + track_name_sanitized = sanitize_filename(track_name) + new_filename = f"{artist_name_sanitized} - {track_name_sanitized}{file_ext}" + return os.path.join(playlist_dir, new_filename), True + + if album_info and album_info.get("is_album"): + clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track")) + track_number = _coerce_int(album_info.get("track_number", 1), 1) + disc_number = _coerce_int(album_info.get("disc_number", 1), 1) + _artists = original_search.get("artists") or track_info.get("artists") or [] + _album_ctx = album_context + _itunes_aid = None + _is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer") + if _is_itunes and isinstance(artist_context, dict): + _aid = artist_context.get("id", "") + if str(_aid).isdigit(): + _itunes_aid = str(_aid) + if not _itunes_aid and _album_ctx: + _ext = _album_ctx.get("external_urls", {}) + if isinstance(_ext, dict) and _ext.get("itunes_artist_id"): + _itunes_aid = _ext["itunes_artist_id"] + + _artist_name = artist_name + _album_artist_name = _artist_name + _album_artists_for_collab = None + _explicit_artist_ctx = track_info.get("_explicit_artist_context") if isinstance(track_info, dict) else None + if isinstance(_explicit_artist_ctx, dict) and _explicit_artist_ctx.get("name"): + _album_artist_name = _explicit_artist_ctx["name"] + _album_artists_for_collab = [_explicit_artist_ctx] + elif isinstance(_explicit_artist_ctx, str) and _explicit_artist_ctx: + _album_artist_name = _explicit_artist_ctx + _album_artists_for_collab = [{"name": _explicit_artist_ctx}] + else: + _sa_artists = _album_ctx.get("artists", []) if _album_ctx else [] + if _sa_artists: + _first_sa = _sa_artists[0] + if isinstance(_first_sa, dict) and _first_sa.get("name"): + _album_artist_name = _first_sa["name"] + elif isinstance(_first_sa, str) and _first_sa: + _album_artist_name = _first_sa + _album_artists_for_collab = _sa_artists + + template_context = { + "artist": _artist_name, + "albumartist": _album_artist_name, + "album": album_info["album_name"], + "title": clean_track_name, + "track_number": track_number, + "disc_number": disc_number, + "year": year, + "quality": context.get("_audio_quality", ""), + "albumtype": album_type_display, + "_artists_list": _album_artists_for_collab if _album_artists_for_collab else _artists, + "_itunes_artist_id": _itunes_aid, + } + total_discs = _coerce_int(album_context.get("total_discs", 1) if album_context else 1, 1) + + if total_discs <= 1 and album_context and album_context.get("id"): + if disc_number > 1: + total_discs = disc_number + else: + try: + _album_tracks = _get_album_tracks_for_source(source, str(album_context["id"])) + if _album_tracks: + total_discs = _max_disc_number(_album_tracks) + if total_discs > 1: + album_context["total_discs"] = total_discs + logger.info( + "[Multi-Disc] Resolved %s discs for single-track download of %r", + total_discs, + album_context.get("name"), + ) + except Exception as _disc_err: + logger.warning("[Multi-Disc] Could not resolve total_discs: %s", _disc_err) + + album_template = _get_config_manager().get("file_organization.templates.album_path", "") + user_controls_disc = "$disc" in album_template + disc_label = _get_config_manager().get("file_organization.disc_label", "Disc") + + folder_path, filename_base = get_file_path_from_template(template_context, "album_path") + if folder_path and filename_base: + if total_discs > 1 and not user_controls_disc: + disc_folder = f"{disc_label} {disc_number}" + final_path = os.path.join(transfer_dir, folder_path, disc_folder, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path, disc_folder), exist_ok=True) + else: + final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + return final_path, True + + artist_name_sanitized = sanitize_filename(template_context["albumartist"]) + album_name_sanitized = sanitize_filename(album_info["album_name"]) + artist_dir = os.path.join(transfer_dir, artist_name_sanitized) + album_folder_name = f"{artist_name_sanitized} - {album_name_sanitized}" + album_dir = os.path.join(artist_dir, album_folder_name) + if total_discs > 1: + album_dir = os.path.join(album_dir, f"{disc_label} {disc_number}") + os.makedirs(album_dir, exist_ok=True) + final_track_name_sanitized = sanitize_filename(clean_track_name) + new_filename = f"{track_number:02d} - {final_track_name_sanitized}{file_ext}" + return os.path.join(album_dir, new_filename), True + + clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track")) + _artists = original_search.get("artists") or track_info.get("artists") or [] + _album_ctx = album_context + _itunes_aid = None + _is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer") + if _is_itunes and isinstance(artist_context, dict): + _aid = artist_context.get("id", "") + if str(_aid).isdigit(): + _itunes_aid = str(_aid) + if not _itunes_aid and _album_ctx: + _ext = _album_ctx.get("external_urls", {}) + if isinstance(_ext, dict) and _ext.get("itunes_artist_id"): + _itunes_aid = _ext["itunes_artist_id"] + + template_context = { + "artist": artist_name, + "albumartist": artist_name, + "album": album_info.get("album_name", clean_track_name) if album_info else clean_track_name, + "title": clean_track_name, + "track_number": 1, + "disc_number": 1, + "year": year, + "quality": context.get("_audio_quality", ""), + "albumtype": album_type_display, + "_artists_list": _artists, + "_itunes_artist_id": _itunes_aid, + } + + folder_path, filename_base = get_file_path_from_template(template_context, "single_path") + if filename_base: + if folder_path: + final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + else: + final_path = os.path.join(transfer_dir, filename_base + file_ext) + os.makedirs(transfer_dir, exist_ok=True) + return final_path, True + + artist_name_sanitized = sanitize_filename(template_context["artist"]) + final_track_name_sanitized = sanitize_filename(clean_track_name) + artist_dir = os.path.join(transfer_dir, artist_name_sanitized) + single_folder_name = f"{artist_name_sanitized} - {final_track_name_sanitized}" + single_dir = os.path.join(artist_dir, single_folder_name) + os.makedirs(single_dir, exist_ok=True) + new_filename = f"{final_track_name_sanitized}{file_ext}" + return os.path.join(single_dir, new_filename), True diff --git a/core/import_pipeline.py b/core/import_pipeline.py new file mode 100644 index 00000000..ede3ae18 --- /dev/null +++ b/core/import_pipeline.py @@ -0,0 +1,925 @@ +"""Import/post-processing pipeline for downloads and imported files.""" + +from __future__ import annotations + +import json +import os +import threading +import time + +from config.settings import config_manager +from core.import_file_ops import ( + cleanup_empty_directories, + create_lossy_copy, + downsample_hires_flac, + extract_track_number_from_filename, + get_audio_quality_string, + get_quality_tier_from_extension, + safe_move_file, +) +from core.import_context import ( + build_import_album_info, + extract_artist_name, + get_import_clean_artist, + get_import_clean_title, + get_import_context_artist, + get_import_has_clean_metadata, + get_import_original_search, + get_import_source, + get_import_track_info, + normalize_import_context, +) +from core.import_guards import check_flac_bit_depth, move_to_quarantine +from core.import_side_effects import ( + check_and_remove_from_wishlist, + emit_track_downloaded, + record_download_provenance, + record_library_history_download, + record_retag_download, + record_soulsync_library_entry, +) +from core.import_runtime_state import ( + add_activity_item, + detect_album_info_web, + download_batches, + download_tasks, + matched_context_lock, + matched_downloads_context, + mark_task_completed as _mark_task_completed, + _post_process_locks, + _post_process_locks_lock, + _processed_download_ids, + tasks_lock, +) +from core.metadata_enrichment import ( + download_cover_art, + enhance_file_metadata, + generate_lrc_file, + wipe_source_tags, +) +from core.import_paths import ( + build_final_path_for_track, + build_simple_download_destination, + docker_resolve_path, + resolve_album_group, +) +from database.music_database import get_database +from utils.logging_config import get_logger + + +logger = get_logger("import_pipeline") +pp_logger = get_logger("post_processing") + + +def post_process_matched_download(context_key, context, file_path, runtime): + on_download_completed = getattr(runtime, "on_download_completed", None) + automation_engine = getattr(runtime, "automation_engine", None) + web_scan_manager = getattr(runtime, "web_scan_manager", None) + repair_worker = getattr(runtime, "repair_worker", None) + + def _notify_download_completed(batch_id, task_id, success=True): + if on_download_completed: + on_download_completed(batch_id, task_id, success=success) + + with _post_process_locks_lock: + if context_key not in _post_process_locks: + _post_process_locks[context_key] = threading.Lock() + file_lock = _post_process_locks[context_key] + + file_lock.acquire() + try: + if not os.path.exists(file_path): + existing_final = context.get('_final_processed_path') + if existing_final and os.path.exists(existing_final): + logger.info( + f"[Race Guard] Source gone but destination exists — already processed by another thread: " + f"{os.path.basename(existing_final)}" + ) + return + logger.error( + f"[Race Guard] Source file gone and no known destination — marking as failed: " + f"{os.path.basename(file_path)}" + ) + context['_race_guard_failed'] = True + return + + _basename = os.path.basename(file_path) + _prev_size = -1 + for _stability_check in range(5): + try: + _cur_size = os.path.getsize(file_path) + except OSError: + _cur_size = -1 + if _cur_size == _prev_size and _cur_size > 0: + break + _prev_size = _cur_size + if _stability_check == 0: + logger.info(f"Waiting for file to stabilise: {_basename} ({_cur_size} bytes)") + time.sleep(1.5) + else: + logger.info(f"File may still be writing after stability checks: {_basename} ({_prev_size} bytes)") + + _skip_acoustid = False + try: + from core.acoustid_verification import AcoustIDVerification, VerificationResult + + verifier = AcoustIDVerification() + available, available_reason = verifier.quick_check_available() + if available and not _skip_acoustid: + context = normalize_import_context(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + artist_context = get_import_context_artist(context) + + expected_track = get_import_clean_title(context, default=original_search.get('title', '')) + expected_artist = '' + track_artists = track_info.get('artists', []) + if track_artists: + first = track_artists[0] + if isinstance(first, dict): + expected_artist = first.get('name', '') + elif isinstance(first, str): + expected_artist = first + if not expected_artist: + expected_artist = extract_artist_name(artist_context) or get_import_clean_artist(context, default='') + + if expected_track and expected_artist: + logger.info(f"Running AcoustID verification for: '{expected_track}' by '{expected_artist}'") + verification_result, verification_msg = verifier.verify_audio_file( + file_path, + expected_track, + expected_artist, + context, + ) + logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}") + context['_acoustid_result'] = verification_result.value + + if verification_result == VerificationResult.FAIL: + try: + quarantine_path = move_to_quarantine( + file_path, + context, + verification_msg, + automation_engine, + ) + logger.error(f"File quarantined due to verification failure: {quarantine_path}") + except Exception as quarantine_error: + logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}") + logger.error(f"Quarantine failed, deleting wrong file: {file_path}") + try: + os.remove(file_path) + except Exception as del_error: + logger.error(f"Could not delete wrong file either: {del_error}") + + context['_acoustid_quarantined'] = True + context['_acoustid_failure_msg'] = verification_msg + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"AcoustID verification failed: {verification_msg}" + ) + + if task_id and batch_id: + _notify_download_completed(batch_id, task_id, success=False) + return + else: + logger.warning("AcoustID verification skipped: missing track/artist info") + context['_acoustid_result'] = 'skip' + else: + logger.info(f"ℹ️ AcoustID verification not available: {available_reason}") + context['_acoustid_result'] = 'disabled' + except Exception as verify_error: + logger.error(f"AcoustID verification error (continuing normally): {verify_error}") + context['_acoustid_result'] = 'error' + + search_result = context.get('search_result', {}) or {} + if not isinstance(search_result, dict): + search_result = {} + is_simple_download = search_result.get('is_simple_download', False) + if is_simple_download: + logger.info(f"Processing simple download (no metadata enhancement): {file_path}") + + destination, album_name, filename = build_simple_download_destination(context, file_path) + if album_name: + logger.info(f"Moving to album folder: {album_name}") + else: + logger.info("Moving to Transfer root (single track)") + + safe_move_file(file_path, destination) + logger.info(f"Moved simple download to: {destination}") + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + if web_scan_manager: + threading.Thread( + target=lambda: web_scan_manager.request_scan("Simple download completed"), + daemon=True, + ).start() + + activity_target = f"{album_name}/{filename}" if album_name else filename + add_activity_item("", "Download Complete", activity_target, "Now") + logger.info(f"Simple download post-processing complete: {activity_target}") + context['_simple_download_completed'] = True + context['_final_path'] = str(destination) + emit_track_downloaded(context, automation_engine) + record_library_history_download(context) + record_download_provenance(context) + return + + logger.info(f"Starting robust post-processing for: {context_key}") + + context = normalize_import_context(context) + artist_context = get_import_context_artist(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + has_clean_metadata = get_import_has_clean_metadata(context) + + if not artist_context: + logger.error("Post-processing failed: Missing artist context.") + return + + _junk_artist_names = {'', 'unknown', 'unknown artist', 'various artists', 'none', 'null'} + _artist_name = (artist_context.get('name', '') if isinstance(artist_context, dict) else '').strip() + if _artist_name.lower() in _junk_artist_names: + logger.info(f"[Unknown Artist Guard] Artist name is '{_artist_name}' — attempting to resolve") + _resolved = False + track_info_guard = track_info or {} + original_search_guard = original_search or {} + + _ti_artists = track_info_guard.get('artists', []) + if isinstance(_ti_artists, list) and _ti_artists: + _first = _ti_artists[0] + _name = _first.get('name', '') if isinstance(_first, dict) else str(_first) + if _name and _name.strip().lower() not in _junk_artist_names: + artist_context['name'] = _name.strip() + logger.info(f"[Unknown Artist Guard] Resolved from track_info.artists: '{_name}'") + _resolved = True + + if not _resolved: + _os_artist = original_search_guard.get('artist') or original_search_guard.get('artist_name') or '' + if isinstance(_os_artist, str) and _os_artist.strip().lower() not in _junk_artist_names: + artist_context['name'] = _os_artist.strip() + logger.info(f"[Unknown Artist Guard] Resolved from original_search_result: '{_os_artist}'") + _resolved = True + + if not _resolved: + _track_id = track_info_guard.get('id') or track_info_guard.get('track_id') or '' + if _track_id: + try: + from core.metadata_service import get_client_for_source, get_primary_source + + _guard_source = get_import_source(context) or get_primary_source() + _fb_client = get_client_for_source(_guard_source) or get_client_for_source(get_primary_source()) + if hasattr(_fb_client, 'get_track_details'): + _details = _fb_client.get_track_details(str(_track_id)) + if _details and isinstance(_details, dict): + _d_artists = _details.get('artists', []) + if isinstance(_d_artists, list) and _d_artists: + _d_first = _d_artists[0] + _d_name = _d_first.get('name', '') if isinstance(_d_first, dict) else str(_d_first) + if _d_name and _d_name.strip().lower() not in _junk_artist_names: + artist_context['name'] = _d_name.strip() + logger.info(f"[Unknown Artist Guard] Resolved from metadata API: '{_d_name}'") + _resolved = True + except Exception as _guard_err: + logger.error(f"[Unknown Artist Guard] Metadata re-fetch failed: {_guard_err}") + + if not _resolved: + logger.error(f"[Unknown Artist Guard] Could not resolve artist — proceeding with '{_artist_name}'") + + context['artist'] = artist_context + + playlist_folder_mode = track_info.get("_playlist_folder_mode", False) + logger.debug(f"[Debug] Post-processing - track_info type: {type(track_info)}, is None: {track_info is None}, is empty: {not track_info}") + logger.debug(f"[Debug] Post-processing - playlist_folder_mode: {playlist_folder_mode}") + if track_info: + logger.debug(f"[Debug] Post-processing - track_info keys: {list(track_info.keys())}") + + if playlist_folder_mode: + playlist_name = track_info.get("_playlist_name", "Unknown Playlist") + logger.info(f"[Playlist Folder Mode] Organizing in playlist folder: {playlist_name}") + + file_ext = os.path.splitext(file_path)[1] + final_path, _ = build_final_path_for_track(context, artist_context, None, file_ext) + logger.info(f"Playlist mode final path: '{final_path}'") + + if not os.path.exists(file_path): + if os.path.exists(final_path): + logger.info( + f"[Playlist Folder Mode] Source gone but destination exists — already processed by another thread: " + f"{os.path.basename(final_path)}" + ) + context['_final_processed_path'] = final_path + return + pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: Source file not found and destination does not exist: {file_path}") + raise FileNotFoundError(f"Source file not found and destination does not exist: {file_path}") + + context['_audio_quality'] = get_audio_quality_string(file_path) + if context['_audio_quality']: + logger.info(f"Audio quality detected: {context['_audio_quality']}") + + rejection_reason = check_flac_bit_depth(file_path, context) + if rejection_reason: + try: + quarantine_path = move_to_quarantine( + file_path, + context, + rejection_reason, + automation_engine, + ) + logger.info(f"File quarantined due to bit depth filter: {quarantine_path}") + except Exception as quarantine_error: + logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") + try: + os.remove(file_path) + except Exception: + pass + + context['_bitdepth_rejected'] = True + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}" + if task_id and batch_id: + _notify_download_completed(batch_id, task_id, success=False) + return + + try: + logger.warning( + f"[Metadata Input] Playlist mode - artist: '{artist_context.get('name', 'MISSING')}' " + f"(id: {artist_context.get('id', 'MISSING')})" + ) + enhance_file_metadata(file_path, context, artist_context, None) + except Exception as meta_err: + import traceback + pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") + wipe_source_tags(file_path) + + logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") + safe_move_file(file_path, final_path) + context['_final_processed_path'] = final_path + + if config_manager.get('post_processing.replaygain_enabled', False): + try: + from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF + if _rg_ffmpeg_ok(): + lufs, peak_dbfs = _rg_analyze(final_path) + gain_db = _RG_REF - lufs + _rg_write(final_path, gain_db, peak_dbfs) + pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB — {os.path.basename(final_path)}") + except Exception as rg_err: + pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") + + downsampled_path = downsample_hires_flac(final_path, context) + if downsampled_path: + final_path = downsampled_path + context['_final_processed_path'] = final_path + + blasphemy_path = create_lossy_copy(final_path) + if blasphemy_path: + context['_final_processed_path'] = blasphemy_path + + downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + cleanup_empty_directories(downloads_path, file_path) + + logger.info(f"[Playlist Folder Mode] Post-processing complete: {final_path}") + + try: + check_and_remove_from_wishlist(context) + except Exception as wishlist_error: + logger.error(f"[Playlist Folder] Error checking wishlist removal: {wishlist_error}") + + emit_track_downloaded(context, automation_engine) + record_library_history_download(context) + record_download_provenance(context) + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id and batch_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['stream_processed'] = True + download_tasks[task_id]['status'] = 'completed' + logger.info(f"[Playlist Folder Mode] Marked task {task_id} as completed") + _notify_download_completed(batch_id, task_id, success=True) + return + + is_album_download = bool(context.get("is_album_download", False)) + album_info = build_import_album_info(context, force_album=is_album_download) + + if is_album_download: + if has_clean_metadata: + logger.info("Album context with clean metadata found - using normalized album info") + else: + logger.warning("Album context found without clean metadata - using normalized album info") + elif not album_info.get('is_album'): + logger.info("Single track download - attempting album detection") + detected_album_info = detect_album_info_web(context, artist_context) + if detected_album_info: + album_info = detected_album_info + + if album_info and album_info['is_album'] and not is_album_download: + logger.info( + "SMART ALBUM GROUPING for track=%r original_album=%r", + album_info.get('clean_track_name', 'Unknown'), + album_info.get('album_name', 'None'), + ) + original_album = original_search.get("album") if original_search.get("album") else None + consistent_album_name = resolve_album_group(artist_context, album_info, original_album) + album_info['album_name'] = consistent_album_name + logger.info("Album grouping complete: final_album=%r", consistent_album_name) + elif album_info and album_info['is_album'] and is_album_download: + logger.info( + "EXPLICIT ALBUM DOWNLOAD - preserving album name=%r; skipping smart grouping", + album_info.get('album_name', 'None'), + ) + + context['_audio_quality'] = get_audio_quality_string(file_path) + if context['_audio_quality']: + logger.info(f"Audio quality detected: {context['_audio_quality']}") + + rejection_reason = check_flac_bit_depth(file_path, context) + if rejection_reason: + try: + quarantine_path = move_to_quarantine( + file_path, + context, + rejection_reason, + automation_engine, + ) + logger.info(f"File quarantined due to bit depth filter: {quarantine_path}") + except Exception as quarantine_error: + logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") + try: + os.remove(file_path) + except Exception: + pass + + context['_bitdepth_rejected'] = True + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}" + if task_id and batch_id: + _notify_download_completed(batch_id, task_id, success=False) + return + + file_ext = os.path.splitext(file_path)[1] + clean_track_name = get_import_clean_title( + context, + album_info=album_info, + default=original_search.get('title', 'Unknown Track'), + ) + track_number = album_info.get('track_number', 1) + logger.debug( + "Final track_number processing: source=%s album_info_track_number=%s track_number=%s", + album_info.get('source', 'unknown'), + album_info.get('track_number', 'NOT_FOUND'), + track_number, + ) + if track_number is None: + track_number = extract_track_number_from_filename(file_path) + logger.info( + "Track number was None; extracted from filename=%r -> %s", + os.path.basename(file_path), + track_number, + ) + if not isinstance(track_number, int) or track_number < 1: + logger.error(f"Invalid track number ({track_number}), defaulting to 1") + track_number = 1 + + logger.debug(f"FINAL track_number used for filename: {track_number}") + album_info['track_number'] = track_number + album_info['clean_track_name'] = clean_track_name + logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata") + + final_path, _ = build_final_path_for_track(context, artist_context, album_info, file_ext) + logger.info(f"Resolved path: '{final_path}'") + context['_final_processed_path'] = final_path + + try: + logger.warning(f"[Metadata Input] artist: '{artist_context.get('name', 'MISSING')}' (id: {artist_context.get('id', 'MISSING')})") + if album_info: + logger.warning( + f"[Metadata Input] album: '{album_info.get('album_name', 'MISSING')}', " + f"track#: {album_info.get('track_number', 'MISSING')}, disc#: {album_info.get('disc_number', 'MISSING')}, " + f"source: {album_info.get('source', 'unknown')}" + ) + else: + logger.info("[Metadata Input] album_info: None (single track)") + enhance_file_metadata(file_path, context, artist_context, album_info) + except Exception as meta_err: + import traceback + pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") + wipe_source_tags(file_path) + + _enhance_source_info = context.get('track_info', {}).get('source_info') or {} + if isinstance(_enhance_source_info, str): + try: + _enhance_source_info = json.loads(_enhance_source_info) + except (json.JSONDecodeError, TypeError): + _enhance_source_info = {} + is_enhance_download = _enhance_source_info.get('enhance', False) + + logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") + if os.path.exists(final_path): + if not os.path.exists(file_path): + logger.info(f"[Protection] Destination exists and source already gone - file already transferred: {os.path.basename(final_path)}") + return + try: + from mutagen import File as MutagenFile + existing_file = MutagenFile(final_path) + has_metadata = existing_file is not None and len(existing_file.tags or {}) > 2 + if has_metadata and not is_enhance_download: + _replace_lower = config_manager.get('import.replace_lower_quality', False) + if _replace_lower: + _existing_tier = get_quality_tier_from_extension(final_path) + _incoming_tier = get_quality_tier_from_extension(file_path) + if _incoming_tier[1] < _existing_tier[1]: + logger.info(f"[Quality Replace] Replacing {_existing_tier[0]} with {_incoming_tier[0]}: {os.path.basename(final_path)}") + try: + os.remove(final_path) + except Exception as e: + logger.error(f"[Quality Replace] Could not remove existing file: {e}") + else: + logger.info( + f"[Protection] Existing file is same or better quality ({_existing_tier[0]} vs {_incoming_tier[0]}) - skipping: " + f"{os.path.basename(final_path)}" + ) + try: + os.remove(file_path) + except FileNotFoundError: + pass + except Exception as e: + logger.error(f"[Protection] Error removing redundant file: {e}") + return + else: + logger.info(f"[Protection] Existing file already has metadata enhancement - skipping overwrite: {os.path.basename(final_path)}") + logger.info(f"[Protection] Removing redundant download file: {os.path.basename(file_path)}") + try: + os.remove(file_path) + except FileNotFoundError: + logger.error(f"[Protection] Could not remove redundant file (already gone): {file_path}") + except Exception as e: + logger.error(f"[Protection] Error removing redundant file: {e}") + return + elif is_enhance_download: + logger.info(f"[Enhance] Quality enhance mode — replacing existing file: {os.path.basename(final_path)}") + try: + os.remove(final_path) + except Exception as e: + logger.error(f"[Enhance] Could not remove existing file for replacement: {e}") + else: + logger.info(f"[Protection] Existing file lacks metadata - safe to overwrite: {os.path.basename(final_path)}") + try: + os.remove(final_path) + except FileNotFoundError: + pass + except Exception as check_error: + logger.error(f"[Protection] Error checking existing file metadata, proceeding with overwrite: {check_error}") + try: + if os.path.exists(final_path): + os.remove(final_path) + except Exception as e: + logger.error(f"[Protection] Failed to remove existing file for overwrite: {e}") + + if not os.path.exists(file_path): + if os.path.exists(final_path): + logger.info(f"[Pre-Move] Source already gone and destination exists - another thread completed transfer: {os.path.basename(final_path)}") + download_cover_art(album_info, os.path.dirname(final_path), context) + generate_lrc_file(final_path, context, artist_context, album_info) + return + expected_dir = os.path.dirname(final_path) + expected_stem = os.path.splitext(os.path.basename(final_path))[0] + expected_ext = os.path.splitext(final_path)[1] + found_variant = None + check_exts = {expected_ext} + if expected_ext == '.flac' and config_manager.get('lossy_copy.enabled', False) and config_manager.get('lossy_copy.delete_original', False): + _lossy_ext_map = {'mp3': '.mp3', 'opus': '.opus', 'aac': '.m4a'} + _lossy_codec = config_manager.get('lossy_copy.codec', 'mp3') + check_exts.add(_lossy_ext_map.get(_lossy_codec, '.mp3')) + if os.path.exists(expected_dir): + for f in os.listdir(expected_dir): + f_ext = os.path.splitext(f)[1].lower() + if f_ext in check_exts and os.path.splitext(f)[0].startswith(expected_stem): + found_variant = os.path.join(expected_dir, f) + break + if found_variant: + logger.debug(f"[Pre-Move] Source gone but found variant in destination (stream processor handled it): {os.path.basename(found_variant)}") + context['_final_processed_path'] = found_variant + download_cover_art(album_info, expected_dir, context) + generate_lrc_file(found_variant, context, artist_context, album_info) + return + logger.warning(f"[Pre-Move] Source file gone and no matching file in destination: {os.path.basename(file_path)}") + raise FileNotFoundError(f"Source file vanished before move and destination does not exist: {file_path}") + + safe_move_file(file_path, final_path) + + if is_enhance_download and _enhance_source_info.get('original_file_path'): + original_enhance_path = _enhance_source_info['original_file_path'] + if os.path.normpath(original_enhance_path) != os.path.normpath(final_path) and os.path.exists(original_enhance_path): + try: + os.remove(original_enhance_path) + old_fmt = os.path.splitext(original_enhance_path)[1] + new_fmt = os.path.splitext(final_path)[1] + logger.info(f"[Enhance] Upgraded {old_fmt} → {new_fmt}: {os.path.basename(final_path)}") + except Exception as e: + logger.error(f"[Enhance] Could not remove old-format file: {e}") + elif is_enhance_download: + old_fmt = _enhance_source_info.get('original_format', 'unknown') + new_fmt = os.path.splitext(final_path)[1] + logger.info(f"[Enhance] Replaced in-place ({old_fmt} → {new_fmt}): {os.path.basename(final_path)}") + + download_cover_art(album_info, os.path.dirname(final_path), context) + generate_lrc_file(final_path, context, artist_context, album_info) + + if config_manager.get('post_processing.replaygain_enabled', False): + try: + from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF + if _rg_ffmpeg_ok(): + lufs, peak_dbfs = _rg_analyze(final_path) + gain_db = _RG_REF - lufs + _rg_write(final_path, gain_db, peak_dbfs) + pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB, peak {peak_dbfs:.2f} dBFS — {os.path.basename(final_path)}") + except Exception as rg_err: + pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") + + downsampled_path = downsample_hires_flac(final_path, context) + if downsampled_path: + final_path = downsampled_path + context['_final_processed_path'] = final_path + + blasphemy_path = create_lossy_copy(final_path) + if blasphemy_path: + context['_final_processed_path'] = blasphemy_path + + downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + cleanup_empty_directories(downloads_path, file_path) + + logger.info(f"Post-processing complete for: {context.get('_final_processed_path', final_path)}") + + emit_track_downloaded(context, automation_engine) + record_library_history_download(context) + record_download_provenance(context) + record_soulsync_library_entry(context, artist_context, album_info) + + try: + if not playlist_folder_mode: + completed_path = context.get('_final_processed_path', final_path) + record_retag_download(context, artist_context, album_info, completed_path) + except Exception as retag_err: + logger.error(f"[Post-Process] Retag data capture failed (non-fatal): {retag_err}") + + try: + completed_path = context.get('_final_processed_path', final_path) + batch_id_for_repair = context.get('batch_id') + if completed_path and batch_id_for_repair and repair_worker: + album_folder = os.path.dirname(str(completed_path)) + if album_folder: + repair_worker.register_folder(batch_id_for_repair, album_folder) + except Exception as repair_err: + logger.error(f"[Post-Process] Repair folder registration failed: {repair_err}") + + try: + completed_path = context.get('_final_processed_path', final_path) + batch_id_for_consistency = context.get('batch_id') + if completed_path and batch_id_for_consistency and album_info and album_info.get('is_album'): + _file_info = { + 'path': str(completed_path), + 'track_number': album_info.get('track_number', 1), + 'disc_number': album_info.get('disc_number', 1), + 'title': get_import_clean_title( + context, + album_info=album_info, + default=album_info.get('clean_track_name', ''), + ), + } + with tasks_lock: + if batch_id_for_consistency in download_batches: + download_batches[batch_id_for_consistency].setdefault('_consistency_files', []).append(_file_info) + except Exception as cons_err: + logger.error(f"[Post-Process] Album consistency registration failed: {cons_err}") + + try: + check_and_remove_from_wishlist(context) + except Exception as wishlist_error: + logger.error(f"[Post-Process] Error checking wishlist removal: {wishlist_error}") + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id and batch_id: + logger.info(f"[Post-Process] Calling completion callback for task {task_id} in batch {batch_id}") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['stream_processed'] = True + download_tasks[task_id]['status'] = 'completed' + logger.info(f"[Post-Process] Marked task {task_id} as completed") + _notify_download_completed(batch_id, task_id, success=True) + + except Exception as e: + import traceback + pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: {e}") + pp_logger.info(traceback.format_exc()) + logger.error(f"\nCRITICAL ERROR in post-processing for {context_key}: {e}") + traceback.print_exc() + + source_exists = os.path.exists(file_path) if file_path else False + if source_exists: + if context_key in _processed_download_ids: + _processed_download_ids.remove(context_key) + logger.warning(f"Removed {context_key} from processed set - will retry on next check") + with matched_context_lock: + if context_key not in matched_downloads_context: + matched_downloads_context[context_key] = context + logger.warning(f"Re-added {context_key} to context for retry") + else: + logger.warning(f"Source file gone, not retrying: {context_key}") + finally: + file_lock.release() + with _post_process_locks_lock: + _post_process_locks.pop(context_key, None) + + +def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime): + on_download_completed = getattr(runtime, "on_download_completed", None) + + def _notify_download_completed(batch_id, task_id, success=True): + if on_download_completed: + on_download_completed(batch_id, task_id, success=success) + + logger = pp_logger + try: + original_task_id = context.pop('task_id', None) + original_batch_id = context.pop('batch_id', None) + post_process_matched_download(context_key, context, file_path, runtime) + if original_task_id: + context['task_id'] = original_task_id + if original_batch_id: + context['batch_id'] = original_batch_id + + if context.get('_race_guard_failed'): + logger.info(f"Race guard: source file gone for task {task_id} — marking as failed") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = 'Source file was already processed or removed by another task' + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) + return + + if context.get('_acoustid_quarantined'): + failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed') + logger.info(f"File was quarantined by AcoustID verification (task={task_id}): {failure_msg}") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {failure_msg}" + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) + return + + if context.get('_simple_download_completed'): + expected_final_path = context.get('_final_path') + if expected_final_path and os.path.exists(expected_final_path): + with tasks_lock: + if task_id in download_tasks: + _mark_task_completed(task_id, context.get('track_info')) + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=True) + return + logger.info( + f"FAILED simple download file not found at: {expected_final_path} " + f"(task={task_id}, context={context_key})" + ) + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"Downloaded file not found at expected location: {os.path.basename(expected_final_path)}" + ) + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) + return + + expected_final_path = context.get('_final_processed_path') + if not expected_final_path: + logger.info(f"No _final_processed_path in context for task {task_id} — cannot verify, assuming success") + with tasks_lock: + if task_id in download_tasks: + _mark_task_completed(task_id, context.get('track_info')) + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=True) + return + + if os.path.exists(expected_final_path): + redownload_ctx = None + with tasks_lock: + if task_id in download_tasks: + _mark_task_completed(task_id, context.get('track_info')) + download_tasks[task_id]['metadata_enhanced'] = True + redownload_ctx = download_tasks[task_id].get('_redownload_context') + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + if redownload_ctx: + try: + old_path = redownload_ctx.get('old_file_path') + lib_track_id = redownload_ctx.get('library_track_id') + if redownload_ctx.get('delete_old_file') and old_path and os.path.exists(old_path): + if os.path.normpath(old_path) != os.path.normpath(expected_final_path): + os.remove(old_path) + logger.info(f"[Redownload] Deleted old file: {old_path}") + if lib_track_id and expected_final_path: + _rd_db = get_database() + _rd_conn = _rd_db._get_connection() + _rd_cursor = _rd_conn.cursor() + _rd_cursor.execute( + """ + UPDATE tracks SET file_path = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (expected_final_path, lib_track_id), + ) + _rd_conn.commit() + _rd_conn.close() + logger.info(f"[Redownload] Updated DB path for track {lib_track_id}") + except Exception as e: + logger.error(f"[Redownload] Post-processing hook error: {e}") + + _notify_download_completed(batch_id, task_id, success=True) + else: + track_name = get_import_clean_title(context, default=context_key) + logger.info(f"FAILED verification for '{track_name}' (task={task_id})") + logger.info(f" expected_final_path: {expected_final_path}") + logger.info(f" file_path (source): {file_path}, exists={os.path.exists(file_path)}") + logger.info( + f" is_album={context.get('is_album_download', False)}, " + f"has_clean_data={get_import_has_clean_metadata(context)}" + ) + expected_dir = os.path.dirname(expected_final_path) + if os.path.exists(expected_dir): + dir_contents = os.listdir(expected_dir) + logger.info(f" directory contains {len(dir_contents)} files: {dir_contents[:20]}") + else: + logger.info(f" directory does not exist: {expected_dir}") + + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f'File verification failed: expected file at {os.path.basename(expected_final_path)} but it was not found after processing' + ) + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + _notify_download_completed(batch_id, task_id, success=False) + except Exception as e: + import traceback + logger.info(f"EXCEPTION in post-processing for '{context_key}' (task={task_id}): {e}") + logger.info(traceback.format_exc()) + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"Post-processing verification failed: {str(e)}" + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) diff --git a/core/import_runtime_state.py b/core/import_runtime_state.py new file mode 100644 index 00000000..9abf2baa --- /dev/null +++ b/core/import_runtime_state.py @@ -0,0 +1,121 @@ +"""Shared runtime state and tiny helpers for import/post-processing code.""" + +from __future__ import annotations + +import threading +import time +from typing import Any, Dict, Optional + +from core.import_context import ( + build_import_album_info, + extract_artist_name, + get_import_clean_artist, + get_import_context_album, + get_import_original_search, + get_import_track_info, + normalize_import_context, +) + +matched_context_lock = threading.Lock() +matched_downloads_context: Dict[str, Dict[str, Any]] = {} +tasks_lock = threading.Lock() +download_tasks: Dict[str, Dict[str, Any]] = {} +download_batches: Dict[str, Dict[str, Any]] = {} +_processed_download_ids = set() +_post_process_locks: Dict[str, threading.Lock] = {} +_post_process_locks_lock = threading.Lock() + +activity_feed = [] +activity_feed_lock = threading.Lock() +_activity_toast_emitter = None + + +def set_activity_toast_emitter(emitter) -> None: + """Set the WebSocket-style emitter used by add_activity_item.""" + global _activity_toast_emitter + _activity_toast_emitter = emitter + + +def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True): + """Append an activity item and emit a toast if an emitter is configured.""" + activity_item = { + "icon": icon, + "title": title, + "subtitle": subtitle, + "time": time_ago, + "timestamp": time.time(), + "show_toast": show_toast, + } + with activity_feed_lock: + activity_feed.append(activity_item) + if len(activity_feed) > 20: + activity_feed.pop(0) + + if show_toast and _activity_toast_emitter is not None: + try: + _activity_toast_emitter("dashboard:toast", activity_item) + except Exception: + pass + + return activity_item + + +def mark_task_completed(task_id: str, track_info: Optional[Dict[str, Any]] = None) -> bool: + """Mark a download task as completed in the shared task registry.""" + with tasks_lock: + task = download_tasks.get(task_id) + if not task: + return False + + task["status"] = "completed" + task["stream_processed"] = True + task["status_change_time"] = time.time() + if track_info is not None: + task["track_info"] = track_info + return True + + +def detect_album_info_web(context, artist_context=None): + """Best-effort album detection for single-track downloads.""" + context = normalize_import_context(context) + if artist_context is None: + artist_context = context.get("artist") or {} + + album_info = build_import_album_info(context) + if album_info.get("is_album"): + return album_info + + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + + album_name = ( + album_ctx.get("name") + or track_info.get("album") + or original_search.get("album") + or "" + ) + track_name = ( + track_info.get("name") + or original_search.get("title") + or "" + ) + artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="") + + if album_name and track_name and album_name.strip().lower() not in { + track_name.strip().lower(), + artist_name.strip().lower(), + }: + return build_import_album_info( + context, + album_info={ + "album_name": album_name, + "track_number": track_info.get("track_number", 1), + "disc_number": track_info.get("disc_number", 1), + "album_image_url": album_ctx.get("image_url", ""), + "confidence": 0.5, + }, + force_album=True, + ) + + return None diff --git a/core/import_side_effects.py b/core/import_side_effects.py new file mode 100644 index 00000000..6c456e3d --- /dev/null +++ b/core/import_side_effects.py @@ -0,0 +1,567 @@ +"""Import post-processing side effects that do not need web runtime state.""" + +from __future__ import annotations + +import hashlib +import json +import os +from typing import Any, Dict, List, Optional + +from core.import_context import ( + extract_artist_name, + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_context_album, + get_import_context_artist, + get_import_original_search, + get_import_source, + get_import_source_ids, + get_import_track_info, + normalize_import_context, + get_library_source_id_columns, +) +from core.wishlist_service import get_wishlist_service +from database.music_database import get_database +from utils.logging_config import get_logger + + +logger = get_logger("import_side_effects") + + +def _get_config_manager(): + from config.settings import config_manager + + return config_manager + + +def _primary_track_artist_name(track_info: Dict[str, Any]) -> str: + artists = (track_info or {}).get("artists", []) + if isinstance(artists, list) and artists: + first = artists[0] + if isinstance(first, dict): + return str(first.get("name", "") or "") + return str(first or "") + if isinstance(artists, str): + return artists + return str((track_info or {}).get("artist", "") or "") + + +def _all_profile_wishlist_tracks(wishlist_service) -> List[Dict[str, Any]]: + database = get_database() + all_profiles = database.get_all_profiles() + wishlist_tracks: List[Dict[str, Any]] = [] + for profile in all_profiles: + wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=profile["id"])) + return wishlist_tracks + + +def _stable_soulsync_id(text: str) -> str: + return str(abs(int(hashlib.md5(text.encode("utf-8", errors="replace")).hexdigest(), 16)) % (10 ** 9)) + + +def emit_track_downloaded(context: Dict[str, Any], automation_engine=None) -> None: + """Emit the track_downloaded automation event.""" + try: + if not automation_engine: + return + + ti = context.get("track_info") or context.get("search_result") or {} + artist_name = "" + artists = ti.get("artists", []) + if artists: + first = artists[0] + artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first) + + automation_engine.emit( + "track_downloaded", + { + "artist": artist_name, + "title": ti.get("name", ti.get("title", "")), + "album": ti.get("album", ""), + "quality": context.get("_audio_quality", "Unknown"), + }, + ) + except Exception: + pass + + +def record_library_history_download(context: Dict[str, Any]) -> None: + """Record a completed download to the library_history table.""" + try: + search_result = context.get("original_search_result") or context.get("search_result") or {} + username = search_result.get("username", context.get("_download_username", "")) + source_map = { + "youtube": "YouTube", + "tidal": "Tidal", + "qobuz": "Qobuz", + "hifi": "HiFi", + "deezer_dl": "Deezer", + "lidarr": "Lidarr", + } + download_source = source_map.get(username, "Soulseek") + + ti = context.get("track_info") or context.get("search_result") or {} + artist_name = _primary_track_artist_name(ti) + if not artist_name: + artist_name = ti.get("artist", "") + + album_raw = ti.get("album", "") + album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "") + title = ti.get("name", ti.get("title", "")) + quality = context.get("_audio_quality", "") + file_path = context.get("_final_processed_path", context.get("_final_path", "")) + + thumb_url = "" + album_context = get_import_context_album(context) + if album_context: + thumb_url = album_context.get("image_url", "") + if not thumb_url: + images = album_context.get("images", []) + if images: + thumb_url = images[0].get("url", "") + if not thumb_url: + album_info = context.get("album_info", {}) + if isinstance(album_info, dict): + thumb_url = album_info.get("album_image_url", "") + + source_filename = search_result.get("filename", "") + source_track_id = search_result.get("track_id", "") or search_result.get("id", "") or ti.get("id", "") + source_track_title = search_result.get("title", "") or search_result.get("name", "") + source_artist = search_result.get("artist", "") + if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr"): + stream_id = source_filename.split("||")[0] + if stream_id and not source_track_id: + source_track_id = stream_id + + acoustid_result = context.get("_acoustid_result", "") + + db = get_database() + db.add_library_history_entry( + event_type="download", + title=title, + artist_name=artist_name, + album_name=album_name, + quality=quality, + file_path=file_path, + thumb_url=thumb_url, + download_source=download_source, + source_track_id=source_track_id, + source_track_title=source_track_title, + source_filename=source_filename, + acoustid_result=acoustid_result, + source_artist=source_artist, + ) + except Exception: + pass + + +def record_download_provenance(context: Dict[str, Any]) -> None: + """Record source provenance for a completed download.""" + try: + search_result = context.get("original_search_result") or context.get("search_result") or {} + username = search_result.get("username", context.get("_download_username", "")) + filename = search_result.get("filename", "") + source_service = { + "youtube": "youtube", + "tidal": "tidal", + "qobuz": "qobuz", + "hifi": "hifi", + "deezer_dl": "deezer", + "lidarr": "lidarr", + }.get(username, "soulseek") + + ti = context.get("track_info") or context.get("search_result") or {} + artist_name = _primary_track_artist_name(ti) + if not artist_name: + artist_name = ti.get("artist", "") + + album_raw = ti.get("album", "") + album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "") + title = ti.get("name", ti.get("title", "")) + + file_path = context.get("_final_processed_path", context.get("_final_path", "")) + quality = context.get("_audio_quality", "") + size = search_result.get("size", 0) + + bit_depth = None + sample_rate = None + bitrate = None + try: + if file_path and os.path.isfile(file_path): + from mutagen import File as MutagenFile + + audio = MutagenFile(file_path) + if audio and audio.info: + sample_rate = getattr(audio.info, "sample_rate", None) + bitrate = getattr(audio.info, "bitrate", None) + bit_depth = getattr(audio.info, "bits_per_sample", None) + except Exception: + pass + + db = get_database() + db.record_track_download( + file_path=file_path, + source_service=source_service, + source_username=username, + source_filename=filename, + source_size=size or 0, + audio_quality=quality, + track_title=title, + track_artist=artist_name, + track_album=album_name, + bit_depth=bit_depth, + sample_rate=sample_rate, + bitrate=bitrate, + ) + except Exception: + pass + + +def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any]) -> None: + """Write imported media to the SoulSync library tables when the active server is SoulSync.""" + try: + config_manager = _get_config_manager() + if config_manager.get_active_media_server() != "soulsync": + return + + context = normalize_import_context(context) + final_path = context.get("_final_processed_path") + if not final_path: + return + + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + source = get_import_source(context) + source_ids = get_import_source_ids(context) + source_columns = get_library_source_id_columns(source) + + artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="") + if not artist_name or artist_name in ("Unknown", "Unknown Artist"): + return + + album_name = "" + if album_info and isinstance(album_info, dict): + album_name = album_info.get("album_name", "") + if not album_name: + album_name = album_ctx.get("name", "") or original_search.get("album", "") + if not album_name: + album_name = track_info.get("name", "Unknown") + + track_name = get_import_clean_title( + context, + album_info=album_info, + default=track_info.get("name", "") or original_search.get("title", ""), + ) + track_number = (track_info.get("track_number") or (album_info.get("track_number") if isinstance(album_info, dict) else None)) or 1 + duration_ms = track_info.get("duration_ms", 0) or 0 + + year = None + release_date = album_ctx.get("release_date", "") + if release_date and len(release_date) >= 4: + try: + year = int(release_date[:4]) + except ValueError: + pass + + image_url = album_ctx.get("image_url", "") + if not image_url: + images = album_ctx.get("images", []) + if images and isinstance(images, list) and len(images) > 0: + img = images[0] + image_url = img.get("url", "") if isinstance(img, dict) else str(img) + + artist_source_id = source_ids.get("artist_id", "") + album_source_id = source_ids.get("album_id", "") + track_source_id = source_ids.get("track_id", "") + for key in ("auto_import", "from_sync_modal", "explicit_artist", "explicit_album", ""): + if artist_source_id == key: + artist_source_id = "" + if album_source_id == key: + album_source_id = "" + if track_source_id == key: + track_source_id = "" + + genres = (artist_context or {}).get("genres", []) if isinstance(artist_context, dict) else [] + if genres: + from core.genre_filter import filter_genres as _filter_genres + + genres = _filter_genres(genres, config_manager) + genres_json = json.dumps(genres) if genres else "" + + bitrate = 0 + try: + from mutagen import File as MutagenFile + + audio = MutagenFile(final_path) + if audio and hasattr(audio, "info") and audio.info and hasattr(audio.info, "bitrate"): + bitrate = int(audio.info.bitrate / 1000) if audio.info.bitrate else 0 + except Exception: + pass + + artist_id = _stable_soulsync_id(artist_name.lower().strip()) + album_id = _stable_soulsync_id(f"{artist_name}::{album_name}".lower().strip()) + track_id = _stable_soulsync_id(final_path) + total_tracks = album_ctx.get("total_tracks", 0) or 0 + + db = get_database() + with db._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute("SELECT id FROM artists WHERE id = ? AND server_source = 'soulsync'", (artist_id,)) + if not cursor.fetchone(): + cursor.execute( + "SELECT id FROM artists WHERE name COLLATE NOCASE = ? AND server_source = 'soulsync' LIMIT 1", + (artist_name,), + ) + existing_by_name = cursor.fetchone() + if existing_by_name: + artist_id = existing_by_name[0] + else: + cursor.execute("SELECT id FROM artists WHERE id = ?", (artist_id,)) + if cursor.fetchone(): + artist_id = _stable_soulsync_id(artist_name.lower().strip() + "::soulsync") + cursor.execute( + """ + INSERT INTO artists (id, name, genres, thumb_url, server_source, created_at, updated_at) + VALUES (?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + (artist_id, artist_name, genres_json, image_url), + ) + artist_source_col = source_columns.get("artist") + if artist_source_col and artist_source_id: + try: + cursor.execute( + f"UPDATE artists SET {artist_source_col} = ? WHERE id = ?", + (artist_source_id, artist_id), + ) + except Exception: + pass + + cursor.execute("SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'", (album_id,)) + if not cursor.fetchone(): + cursor.execute( + "SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? AND server_source = 'soulsync' LIMIT 1", + (album_name, artist_id), + ) + existing_album_by_name = cursor.fetchone() + if existing_album_by_name: + album_id = existing_album_by_name[0] + else: + cursor.execute("SELECT id FROM albums WHERE id = ?", (album_id,)) + if cursor.fetchone(): + album_id = _stable_soulsync_id(f"{artist_name}::{album_name}::soulsync".lower().strip()) + cursor.execute( + """ + INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, track_count, + duration, server_source, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + (album_id, artist_id, album_name, year, image_url, genres_json, total_tracks, duration_ms), + ) + album_source_col = source_columns.get("album") + if album_source_col and album_source_id: + try: + cursor.execute( + f"UPDATE albums SET {album_source_col} = ? WHERE id = ?", + (album_source_id, album_id), + ) + except Exception: + pass + + track_artist = None + track_artists_list = track_info.get("artists", []) or original_search.get("artists", []) + if track_artists_list: + first_track_artist = track_artists_list[0] + if isinstance(first_track_artist, dict): + ta_name = first_track_artist.get("name", "") + else: + ta_name = str(first_track_artist) + if ta_name and ta_name.lower() != artist_name.lower(): + track_artist = ta_name + + cursor.execute("SELECT id FROM tracks WHERE file_path = ?", (final_path,)) + if not cursor.fetchone(): + cursor.execute( + """ + INSERT INTO tracks (id, album_id, artist_id, title, track_number, + duration, file_path, bitrate, track_artist, server_source, + created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + ( + track_id, + album_id, + artist_id, + track_name, + track_number, + duration_ms, + final_path, + bitrate, + track_artist, + ), + ) + track_source_col = source_columns.get("track") + if track_source_col and track_source_id: + try: + cursor.execute( + f"UPDATE tracks SET {track_source_col} = ? WHERE id = ?", + (track_source_id, track_id), + ) + track_album_col = source_columns.get("track_album") + if track_album_col and album_source_id: + cursor.execute( + f"UPDATE tracks SET {track_album_col} = ? WHERE id = ?", + (album_source_id, track_id), + ) + except Exception: + pass + + conn.commit() + logger.info("[SoulSync Library] Added: %s / %s / %s", artist_name, album_name, track_name) + except Exception as exc: + logger.error("[SoulSync Library] Could not record library entry: %s", exc) + + +def record_retag_download(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any], final_path: str) -> None: + """Record a completed download for later re-tagging.""" + try: + db = get_database() + + context = normalize_import_context(context) + artist_context = get_import_context_artist(context) or (artist_context if isinstance(artist_context, dict) else {}) + album_context = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + source = get_import_source(context) + source_ids = get_import_source_ids(context) + + artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="Unknown Artist") + is_album = album_info and album_info.get("is_album", False) + group_type = "album" if is_album else "single" + album_name = album_info.get("album_name", "") if album_info else get_import_clean_album(context, default=original_search.get("album", "Unknown")) + + image_url = album_info.get("album_image_url") if album_info else None + if not image_url: + image_url = album_context.get("image_url", "") + if not image_url and album_context.get("images"): + images = album_context.get("images", []) + if images and isinstance(images[0], dict): + image_url = images[0].get("url", "") + + total_tracks = album_context.get("total_tracks", 1) if album_context else 1 + release_date = album_context.get("release_date", "") if album_context else "" + + spotify_album_id = None + itunes_album_id = None + if source == "spotify": + spotify_album_id = source_ids.get("album_id", "") or None + elif source == "itunes": + itunes_album_id = source_ids.get("album_id", "") or None + + group_id = db.find_retag_group(artist_name, album_name) + if group_id is None: + group_id = db.add_retag_group( + group_type=group_type, + artist_name=artist_name, + album_name=album_name, + image_url=image_url, + spotify_album_id=spotify_album_id, + itunes_album_id=itunes_album_id, + total_tracks=total_tracks, + release_date=release_date, + ) + if group_id is None: + return + + track_number = album_info.get("track_number", 1) if album_info else (track_info.get("track_number", 1) or 1) + disc_number = original_search.get("disc_number") or (album_info.get("disc_number", 1) if album_info else track_info.get("disc_number", 1) or 1) + title = get_import_clean_title( + context, + album_info=album_info, + default=album_info.get("clean_track_name", "Unknown Track") if album_info else "Unknown Track", + ) + file_format = os.path.splitext(str(final_path))[1].lstrip(".").lower() + + spotify_track_id = None + itunes_track_id = None + if source == "spotify": + spotify_track_id = source_ids.get("track_id", "") or None + elif source == "itunes": + itunes_track_id = source_ids.get("track_id", "") or None + + if not db.retag_track_exists(group_id, str(final_path)): + db.add_retag_track( + group_id=group_id, + track_number=track_number, + disc_number=disc_number, + title=title, + file_path=str(final_path), + file_format=file_format, + spotify_track_id=spotify_track_id, + itunes_track_id=itunes_track_id, + ) + logger.info("[Retag] Recorded track for retag: '%s' in '%s'", title, album_name) + + db.trim_retag_groups(100) + except Exception as exc: + logger.error("[Retag] Could not record track for retag: %s", exc) + + +def check_and_remove_from_wishlist(context: Dict[str, Any]) -> None: + """Check whether a successful download should be removed from the wishlist.""" + try: + wishlist_service = get_wishlist_service() + spotify_track_id = None + + track_info = context.get("track_info", {}) + if track_info.get("id"): + spotify_track_id = track_info["id"] + logger.info("[Wishlist] Found Spotify ID from track_info: %s", spotify_track_id) + elif context.get("original_search_result", {}).get("id"): + spotify_track_id = context["original_search_result"]["id"] + logger.info("[Wishlist] Found Spotify ID from original_search_result: %s", spotify_track_id) + elif "wishlist_id" in track_info: + wishlist_id = track_info["wishlist_id"] + logger.info("[Wishlist] Found wishlist_id in context: %s", wishlist_id) + wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service) + for wishlist_track in wishlist_tracks: + if wishlist_track.get("wishlist_id") == wishlist_id: + spotify_track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id") + logger.info("[Wishlist] Found Spotify ID from wishlist entry: %s", spotify_track_id) + break + + if not spotify_track_id: + track_name = track_info.get("name") or context.get("original_search_result", {}).get("title", "") + artist_name = _primary_track_artist_name(track_info) or _primary_track_artist_name(context.get("original_search_result", {})) + + if track_name and artist_name: + logger.warning("[Wishlist] No Spotify ID found, checking for fuzzy match: '%s' by '%s'", track_name, artist_name) + + wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service) + for wishlist_track in wishlist_tracks: + wl_name = wishlist_track.get("name", "").lower() + wl_artists = wishlist_track.get("artists", []) + wl_artist_name = "" + if wl_artists: + if isinstance(wl_artists[0], dict): + wl_artist_name = wl_artists[0].get("name", "").lower() + else: + wl_artist_name = str(wl_artists[0]).lower() + if wl_name == track_name.lower() and wl_artist_name == artist_name.lower(): + spotify_track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id") + logger.info("[Wishlist] Found fuzzy match - Spotify ID: %s", spotify_track_id) + break + + if spotify_track_id: + logger.info("[Wishlist] Attempting to remove track from wishlist: %s", spotify_track_id) + removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) + if removed: + logger.info("[Wishlist] Successfully removed track from wishlist: %s", spotify_track_id) + else: + logger.warning("ℹ️ [Wishlist] Track not found in wishlist or already removed: %s", spotify_track_id) + else: + logger.warning("ℹ️ [Wishlist] No Spotify track ID found for wishlist removal check") + except Exception as exc: + logger.error("[Wishlist] Error in wishlist removal check: %s", exc) diff --git a/core/import_staging.py b/core/import_staging.py new file mode 100644 index 00000000..da93580a --- /dev/null +++ b/core/import_staging.py @@ -0,0 +1,454 @@ +"""Shared staging folder and import suggestion helpers.""" + +from __future__ import annotations + +import os +import threading +from typing import Any, Dict, List, Optional, Tuple + +from core.import_paths import docker_resolve_path +from utils.logging_config import get_logger + +logger = get_logger("import_staging") + +AUDIO_EXTENSIONS = {".mp3", ".flac", ".ogg", ".opus", ".m4a", ".aac", ".wav", ".wma", ".aiff", ".aif", ".ape"} + +_import_suggestions_cache_lock = threading.Lock() +_import_suggestions_cache: Dict[str, Any] = { + "suggestions": [], + "building": False, + "built": False, +} + + +def _get_config_manager(): + try: + from config.settings import config_manager + + return config_manager + except Exception: + class _FallbackConfig: + @staticmethod + def get(key, default=None): + return default + + return _FallbackConfig() + + +def get_staging_path() -> str: + """Resolve the configured staging folder path.""" + raw = _get_config_manager().get("import.staging_path", "./Staging") + return docker_resolve_path(raw) + + +def get_import_suggestions_cache() -> Dict[str, Any]: + """Expose the shared import suggestions cache.""" + return _import_suggestions_cache + + +def get_primary_source() -> str: + from core.metadata_service import get_primary_source as _get_primary_source + + return _get_primary_source() + + +def get_source_priority(preferred_source: str): + from core.metadata_service import get_source_priority as _get_source_priority + + return _get_source_priority(preferred_source) + + +def get_client_for_source(source: str): + from core.metadata_service import get_client_for_source as _get_client_for_source + + return _get_client_for_source(source) + + +def _search_albums_for_source(source: str, client: Any, query: str, limit: int = 5): + from core.metadata_service import _search_albums_for_source as _metadata_search_albums_for_source + + return _metadata_search_albums_for_source(source, client, query, limit=limit) + + +def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 5): + from core.metadata_service import _search_tracks_for_source as _metadata_search_tracks_for_source + + return _metadata_search_tracks_for_source(source, client, query, limit=limit) + + +def _extract_value(value: Any, *names: str, default: Any = None) -> Any: + if value is None: + return default + + if isinstance(value, (str, bytes)): + return default + + for name in names: + if isinstance(value, dict): + if name in value and value[name] is not None: + return value[name] + else: + candidate = getattr(value, name, None) + if candidate is not None: + return candidate + + return default + + +def _extract_artist_names(artists: Any) -> List[str]: + if not artists: + return [] + + if isinstance(artists, (str, bytes)): + artist = str(artists).strip() + return [artist] if artist else [] + + try: + items = list(artists) + except TypeError: + items = [artists] + + names: List[str] = [] + for artist in items: + if isinstance(artist, dict): + name = str(_extract_value(artist, "name", "artist_name", "title", default="") or "").strip() + else: + candidate = getattr(artist, "name", None) + if candidate is None: + candidate = artist + name = str(candidate or "").strip() + if name: + names.append(name) + + return names + + +def _normalize_album_result(album: Any, source: str) -> Dict[str, Any]: + album_id = str(_extract_value(album, "id", "album_id", "release_id", default="") or "").strip() + album_name = str(_extract_value(album, "name", "title", default="") or "").strip() + artists = _extract_artist_names(_extract_value(album, "artists", default=[])) + artist_name = ", ".join(artists) if artists else str( + _extract_value(album, "artist_name", "artist", default="Unknown Artist") or "Unknown Artist" + ).strip() + release_date = str(_extract_value(album, "release_date", "releaseDate", default="") or "").strip() + album_type = str(_extract_value(album, "album_type", "type", default="album") or "album").strip() or "album" + + total_tracks = _extract_value(album, "total_tracks", "track_count", default=0) + if isinstance(total_tracks, (list, tuple, set)): + total_tracks = len(total_tracks) + try: + total_tracks = int(total_tracks or 0) + except (TypeError, ValueError): + total_tracks = 0 + + image_url = _extract_value(album, "image_url", "thumb_url", "cover_image", "cover_url", default="") + if not image_url: + images = _extract_value(album, "images", default=[]) or [] + if isinstance(images, dict): + images = [images] + elif isinstance(images, (str, bytes)): + images = [images] + try: + images = list(images) + except TypeError: + images = [images] + if images: + first_image = images[0] + if isinstance(first_image, (str, bytes)): + image_url = str(first_image).strip() + else: + image_url = _extract_value(first_image, "url", "image_url", "src", default="") + + return { + "id": album_id or album_name or "unknown-album", + "name": album_name or album_id or "Unknown Album", + "artist": artist_name or "Unknown Artist", + "release_date": release_date, + "total_tracks": total_tracks, + "image_url": str(image_url or ""), + "album_type": album_type, + "source": source, + } + + +def _album_fingerprint(album: Dict[str, Any]) -> Tuple[str, str, str, str]: + return ( + str(album.get("name", "") or "").strip().casefold(), + str(album.get("artist", "") or "").strip().casefold(), + str(album.get("release_date", "") or "").strip()[:10].casefold(), + str(album.get("album_type", "") or "").strip().casefold(), + ) + + +def _normalize_track_result(track: Any, source: str) -> Dict[str, Any]: + track_id = str(_extract_value(track, "id", "track_id", "trackId", default="") or "").strip() + track_name = str(_extract_value(track, "name", "title", "track_name", default="") or "").strip() + artists = _extract_artist_names(_extract_value(track, "artists", default=[])) + artist_name = ", ".join(artists) if artists else str( + _extract_value(track, "artist", "artist_name", default="Unknown Artist") or "Unknown Artist" + ).strip() + + album_value = _extract_value(track, "album", default=None) + album_name = "" + album_id = str(_extract_value(track, "album_id", "collectionId", "albumId", default="") or "").strip() + if isinstance(album_value, dict): + album_name = str(_extract_value(album_value, "name", "title", default="") or "").strip() + album_id = album_id or str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip() + if not album_name: + album_name = album_id + elif isinstance(album_value, (str, bytes)): + album_name = str(album_value).strip() + elif album_value is not None: + album_name = str(_extract_value(album_value, "name", "title", default=album_value) or "").strip() + if not album_id: + album_id = str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip() + + image_url = _extract_value(track, "image_url", "thumb_url", "cover_image", default="") + if not image_url: + images = _extract_value(track, "images", default=[]) or [] + if isinstance(images, dict): + images = [images] + elif isinstance(images, (str, bytes)): + images = [images] + try: + images = list(images) + except TypeError: + images = [images] + if images: + first_image = images[0] + if isinstance(first_image, (str, bytes)): + image_url = str(first_image).strip() + else: + image_url = _extract_value(first_image, "url", "image_url", "src", default="") + if not image_url and album_value is not None: + album_images = _extract_value(album_value, "images", default=[]) or [] + if isinstance(album_images, dict): + album_images = [album_images] + elif isinstance(album_images, (str, bytes)): + album_images = [album_images] + try: + album_images = list(album_images) + except TypeError: + album_images = [album_images] + if album_images: + first_album_image = album_images[0] + if isinstance(first_album_image, (str, bytes)): + image_url = str(first_album_image).strip() + else: + image_url = _extract_value(first_album_image, "url", "image_url", "src", default="") + + duration_ms = _extract_value(track, "duration_ms", "duration", "trackTimeMillis", default=0) + try: + duration_ms = int(duration_ms or 0) + except (TypeError, ValueError): + duration_ms = 0 + + track_number = _extract_value(track, "track_number", "trackNumber", default=1) + try: + track_number = int(track_number or 1) + except (TypeError, ValueError): + track_number = 1 + + return { + "id": track_id or track_name or "unknown-track", + "name": track_name or track_id or "Unknown Track", + "artist": artist_name or "Unknown Artist", + "album": album_name or "", + "album_id": album_id or "", + "duration_ms": duration_ms, + "image_url": str(image_url or ""), + "track_number": track_number, + "source": source, + } + + +def _read_staging_audio_tags(file_path: str) -> Tuple[Optional[str], Optional[str]]: + try: + from mutagen import File as MutagenFile + + tags = MutagenFile(file_path, easy=True) + if not tags: + return None, None + + album = (tags.get("album") or [None])[0] + artist = (tags.get("artist") or (tags.get("albumartist") or [None]))[0] + album_text = str(album).strip() if album else "" + artist_text = str(artist).strip() if artist else "" + return (album_text or None, artist_text or None) + except Exception: + return None, None + + +def _collect_import_suggestion_queries(staging_path: str) -> List[str]: + tag_albums: Dict[Tuple[str, str], int] = {} + folder_hints: Dict[str, int] = {} + + for root, _dirs, filenames in os.walk(staging_path): + audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS] + if not audio_files: + continue + + rel_dir = os.path.relpath(root, staging_path) + if rel_dir != ".": + top_folder = rel_dir.split(os.sep)[0] + folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files) + + for fname in audio_files: + full_path = os.path.join(root, fname) + album, artist = _read_staging_audio_tags(full_path) + if album: + key = (album.strip(), (artist or "").strip()) + tag_albums[key] = tag_albums.get(key, 0) + 1 + + queries: List[str] = [] + seen_lower = set() + + for (album, artist), _count in sorted(tag_albums.items(), key=lambda item: -item[1]): + q = f"{album} {artist}".strip() if artist else album + if q and q.lower() not in seen_lower: + seen_lower.add(q.lower()) + queries.append(q) + + for folder, _count in sorted(folder_hints.items(), key=lambda item: -item[1]): + q = folder.replace("_", " ") + if q and q.lower() not in seen_lower: + seen_lower.add(q.lower()) + queries.append(q) + + return queries[:5] + + +def search_import_albums(query: str, limit: int = 12) -> List[Dict[str, Any]]: + """Search albums using the configured metadata provider first.""" + query = (query or "").strip() + if not query: + return [] + + results: List[Dict[str, Any]] = [] + seen = set() + source_chain = get_source_priority(get_primary_source()) + + for source in source_chain: + client = get_client_for_source(source) + if not client: + continue + + source_results = _search_albums_for_source(source, client, query, limit=limit) + if not source_results: + continue + + added_for_source = False + for album in source_results: + suggestion = _normalize_album_result(album, source) + fingerprint = _album_fingerprint(suggestion) + if fingerprint in seen: + continue + seen.add(fingerprint) + results.append(suggestion) + added_for_source = True + if len(results) >= limit: + return results[:limit] + + if added_for_source: + break + + return results[:limit] + + +def search_import_tracks(query: str, limit: int = 30) -> List[Dict[str, Any]]: + """Search tracks using the configured metadata provider priority order.""" + query = (query or "").strip() + if not query: + return [] + + results: List[Dict[str, Any]] = [] + source_chain = get_source_priority(get_primary_source()) + + for source in source_chain: + client = get_client_for_source(source) + if not client: + continue + + source_results = _search_tracks_for_source(source, client, query, limit=limit) + if not source_results: + continue + + for track in source_results: + results.append(_normalize_track_result(track, source)) + if len(results) >= limit: + return results[:limit] + break + + return results[:limit] + + +def _build_import_suggestions_background(): + cache = _import_suggestions_cache + + with _import_suggestions_cache_lock: + if cache["building"]: + return + cache["building"] = True + + try: + staging_path = get_staging_path() + if not os.path.isdir(staging_path): + with _import_suggestions_cache_lock: + cache["suggestions"] = [] + cache["built"] = True + return + + queries = _collect_import_suggestion_queries(staging_path) + if not queries: + with _import_suggestions_cache_lock: + cache["suggestions"] = [] + cache["built"] = True + return + + suggestions: List[Dict[str, Any]] = [] + seen = set() + for query in queries: + try: + albums = search_import_albums(query, limit=2) + for album in albums: + fingerprint = _album_fingerprint(album) + if fingerprint in seen: + continue + seen.add(fingerprint) + suggestions.append(album) + except Exception as exc: + logger.warning("Import suggestion search failed for %r: %s", query, exc) + + with _import_suggestions_cache_lock: + cache["suggestions"] = suggestions[:8] + cache["built"] = True + + logger.info( + "Import suggestions cache built: %s suggestions from %s hints", + len(cache["suggestions"]), + len(queries), + ) + except Exception as exc: + logger.error("Error building import suggestions cache: %s", exc) + with _import_suggestions_cache_lock: + cache["suggestions"] = [] + cache["built"] = True + finally: + with _import_suggestions_cache_lock: + cache["building"] = False + + +def start_import_suggestions_cache(): + """Start building the import suggestions cache in a background thread.""" + threading.Thread( + target=_build_import_suggestions_background, + daemon=True, + name="import-suggestions-cache", + ).start() + + +def refresh_import_suggestions_cache(): + """Invalidate and rebuild the suggestions cache.""" + with _import_suggestions_cache_lock: + _import_suggestions_cache["built"] = False + start_import_suggestions_cache() diff --git a/core/metadata_enrichment.py b/core/metadata_enrichment.py new file mode 100644 index 00000000..293af162 --- /dev/null +++ b/core/metadata_enrichment.py @@ -0,0 +1,1486 @@ +"""Source-aware metadata enrichment helpers for imported audio files.""" + +from __future__ import annotations + +import json +import os +import re +import threading +import time +import urllib.request +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Dict, Optional + +from core.import_context import ( + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_context_album, + get_import_context_artist, + get_import_original_search, + get_import_source, + get_import_source_ids, + get_import_track_info, + get_source_tag_names, + normalize_import_context, +) +from config.settings import config_manager +from core.metadata_service import get_itunes_client +from database.music_database import get_database +from utils.logging_config import get_logger + + +logger = get_logger("metadata_enrichment") + +_FILE_LOCKS: Dict[str, threading.Lock] = {} +_FILE_LOCKS_LOCK = threading.Lock() + +_MB_RELEASE_CACHE: Dict[tuple, str] = {} +_MB_RELEASE_CACHE_LOCK = threading.RLock() +_MB_RELEASE_DETAIL_CACHE: Dict[str, Dict[str, Any]] = {} +_MB_RELEASE_DETAIL_CACHE_LOCK = threading.RLock() + +_EDITION_PAREN_RE = re.compile( + r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' + r'limited|bonus|platinum|gold|super\s*deluxe|standard)' + r'(?:\s+(?:edition|version))?[^)\]]*[\)\]]', + re.IGNORECASE, +) +_EDITION_BARE_RE = re.compile( + r'\s+(?:-\s+)?(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' + r'limited|bonus|platinum|gold|super\s*deluxe|standard)' + r'(?:\s+(?:edition|version))?\s*$', + re.IGNORECASE, +) + + +class _NullConfigManager: + def get(self, _key: str, default: Any = None) -> Any: + return default + + +def _get_logger(runtime=None): + return logger + + +def _get_config_manager(runtime=None): + return config_manager + + +def _get_database(runtime=None): + try: + return get_database() + except Exception: + return None + + +def _get_itunes_client(runtime=None): + try: + return get_itunes_client() + except Exception: + worker = getattr(runtime, "itunes_enrichment_worker", None) + if worker and getattr(worker, "client", None): + return worker.client + return getattr(runtime, "itunes_client", None) + + +def _extract_artist_name(artist: Any) -> str: + if isinstance(artist, dict): + return str(artist.get("name", "") or "") + if hasattr(artist, "name"): + return str(getattr(artist, "name") or "") + return str(artist) if artist else "" + + +def _get_mutagen_symbols(runtime=None): + """Lazy mutagen import so tests can monkeypatch this without the package installed.""" + try: + from mutagen import File as MutagenFile + from mutagen.apev2 import APEv2, APENoHeaderError + from mutagen.flac import FLAC, Picture + from mutagen.id3 import ( + APIC, + ID3, + TBPM, + TCOP, + TDOR, + TDRC, + TCON, + TIT2, + TALB, + TPE1, + TPE2, + TPOS, + TPUB, + TRCK, + TSRC, + TXXX, + UFID, + TMED, + ) + from mutagen.mp4 import MP4, MP4Cover, MP4FreeForm + from mutagen.oggvorbis import OggVorbis + try: + from mutagen.oggopus import OggOpus + except Exception: + OggOpus = None + except Exception as exc: + _get_logger(runtime).debug("Mutagen unavailable for metadata enrichment: %s", exc) + return None + + return SimpleNamespace( + File=MutagenFile, + APEv2=APEv2, + APENoHeaderError=APENoHeaderError, + FLAC=FLAC, + Picture=Picture, + ID3=ID3, + APIC=APIC, + TBPM=TBPM, + TCOP=TCOP, + TDOR=TDOR, + TDRC=TDRC, + TCON=TCON, + TIT2=TIT2, + TALB=TALB, + TPE1=TPE1, + TPE2=TPE2, + TPOS=TPOS, + TPUB=TPUB, + TRCK=TRCK, + TSRC=TSRC, + TXXX=TXXX, + UFID=UFID, + TMED=TMED, + MP4=MP4, + MP4Cover=MP4Cover, + MP4FreeForm=MP4FreeForm, + OggVorbis=OggVorbis, + OggOpus=OggOpus, + ) + + +def _get_file_lock(file_path: str) -> threading.Lock: + with _FILE_LOCKS_LOCK: + lock = _FILE_LOCKS.get(file_path) + if lock is None: + lock = threading.Lock() + _FILE_LOCKS[file_path] = lock + return lock + + +def _is_ogg_opus(audio_file: Any) -> bool: + return type(audio_file).__name__ == "OggOpus" + + +def _is_vorbis_like(audio_file: Any, symbols: Any) -> bool: + vorbis_classes = tuple( + cls for cls in ( + getattr(symbols, "FLAC", None), + getattr(symbols, "OggVorbis", None), + ) if cls is not None + ) + return bool(vorbis_classes) and isinstance(audio_file, vorbis_classes) or _is_ogg_opus(audio_file) + + +def _save_audio_file(audio_file: Any, symbols: Any) -> None: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.save(v1=0, v2_version=4) + elif isinstance(audio_file, symbols.FLAC): + audio_file.save(deleteid3=True) + else: + audio_file.save() + + +def _strip_all_non_audio_tags(file_path: str, runtime=None) -> dict: + summary = {"apev2_stripped": False, "apev2_tag_count": 0} + if os.path.splitext(file_path)[1].lower() != ".mp3": + return summary + + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return summary + + try: + apev2_tags = symbols.APEv2(file_path) + tag_count = len(apev2_tags) + tag_keys = list(apev2_tags.keys()) + apev2_tags.delete(file_path) + summary["apev2_stripped"] = True + summary["apev2_tag_count"] = tag_count + _get_logger(runtime).info("Stripped %s APEv2 tags: %s", tag_count, ", ".join(tag_keys[:10])) + except symbols.APENoHeaderError: + pass + except Exception as exc: + _get_logger(runtime).error("Could not strip APEv2 tags (non-fatal): %s", exc) + return summary + + +def _verify_metadata_written(file_path: str, runtime=None) -> bool: + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return False + + try: + check = symbols.File(file_path) + if check is None or check.tags is None: + _get_logger(runtime).info("[VERIFY] Tags are None after save: %s", file_path) + return False + + title_found = False + artist_found = False + if isinstance(check.tags, symbols.ID3): + title_found = bool(check.tags.getall("TIT2")) + artist_found = bool(check.tags.getall("TPE1")) + try: + symbols.APEv2(file_path) + _get_logger(runtime).info("[VERIFY] APEv2 tags still present after processing!") + return False + except symbols.APENoHeaderError: + pass + elif _is_vorbis_like(check, symbols): + title_found = bool(check.get("title")) + artist_found = bool(check.get("artist")) + elif isinstance(check, symbols.MP4): + title_found = bool(check.get("\xa9nam")) + artist_found = bool(check.get("\xa9ART")) + + if not title_found or not artist_found: + _get_logger(runtime).warning("[VERIFY] Missing metadata - title:%s artist:%s", title_found, artist_found) + return False + + _get_logger(runtime).info("[VERIFY] Metadata verified OK") + return True + except Exception as exc: + _get_logger(runtime).error("[VERIFY] Verification error (non-fatal): %s", exc) + return False + + +def _get_image_dimensions(data: bytes): + try: + if data[:8] == b"\x89PNG\r\n\x1a\n": + import struct + + w, h = struct.unpack(">II", data[16:24]) + return w, h + if data[:2] == b"\xff\xd8": + import struct + + i = 2 + while i < len(data) - 9: + if data[i] != 0xFF: + break + marker = data[i + 1] + if marker in (0xC0, 0xC2): + h, w = struct.unpack(">HH", data[i + 5 : i + 9]) + return w, h + length = struct.unpack(">H", data[i + 2 : i + 4])[0] + i += 2 + length + except Exception: + pass + return None, None + + +def extract_source_metadata(context: dict, artist: dict, album_info: dict, runtime=None) -> dict: + if album_info is None: + album_info = {} + + cfg = _get_config_manager(runtime) + context = normalize_import_context(context) + original_search = get_import_original_search(context) + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + source = get_import_source(context) + source_ids = get_import_source_ids(context) + + artist_dict = artist if isinstance(artist, dict) else { + "name": _extract_artist_name(artist), + "id": getattr(artist, "id", ""), + "genres": list(getattr(artist, "genres", []) or []), + } + + metadata: Dict[str, Any] = { + "source": source, + "source_track_id": source_ids["track_id"], + "source_artist_id": source_ids["artist_id"], + "source_album_id": source_ids["album_id"], + } + + metadata["title"] = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "")) + if original_search.get("clean_title"): + _get_logger(runtime).info("Metadata: Using clean title: '%s'", metadata["title"]) + elif album_info.get("clean_track_name"): + _get_logger(runtime).info("Metadata: Using album info clean name: '%s'", metadata["title"]) + else: + _get_logger(runtime).warning("Metadata: Using original title as fallback: '%s'", metadata["title"]) + + artists = original_search.get("artists") + if isinstance(artists, list) and artists: + all_artists = [] + for artist_item in artists: + if isinstance(artist_item, dict) and artist_item.get("name"): + all_artists.append(artist_item["name"]) + elif isinstance(artist_item, str): + all_artists.append(artist_item) + else: + all_artists.append(str(artist_item)) + metadata["artist"] = ", ".join(all_artists) + _get_logger(runtime).info("Metadata: Using all artists: '%s'", metadata["artist"]) + else: + metadata["artist"] = artist_dict.get("name", "") or get_import_clean_artist(context) + _get_logger(runtime).info("Metadata: Using primary artist: '%s'", metadata["artist"]) + + raw_album_artist = artist_dict.get("name", "") or metadata["artist"] + track_info_ctx = track_info or {} + explicit_artist = track_info_ctx.get("_explicit_artist_context") if isinstance(track_info_ctx, dict) else None + album_artists_for_collab = None + + if isinstance(explicit_artist, dict) and explicit_artist.get("name"): + raw_album_artist = explicit_artist["name"] + album_artists_for_collab = [explicit_artist] + elif isinstance(explicit_artist, str) and explicit_artist: + raw_album_artist = explicit_artist + album_artists_for_collab = [{"name": explicit_artist}] + elif album_ctx and isinstance(album_ctx, dict): + album_artists = album_ctx.get("artists", []) + if album_artists: + first_album_artist = album_artists[0] + if isinstance(first_album_artist, dict) and first_album_artist.get("name"): + raw_album_artist = first_album_artist["name"] + elif isinstance(first_album_artist, str) and first_album_artist: + raw_album_artist = first_album_artist + album_artists_for_collab = album_artists + + collab_mode = cfg.get("file_organization.collab_artist_mode", "first") + if collab_mode == "first" and raw_album_artist: + context_artists = album_artists_for_collab or original_search.get("artists") or track_info_ctx.get("artists") or [] + if len(context_artists) > 1: + first = context_artists[0] + raw_album_artist = first.get("name", first) if isinstance(first, dict) else str(first) + elif len(context_artists) == 1 and ("," in raw_album_artist or " & " in raw_album_artist): + artist_id = str(artist_dict.get("id", "")) + if source == "itunes" and artist_id.isdigit(): + try: + itunes_client = _get_itunes_client(runtime) + if itunes_client and hasattr(itunes_client, "resolve_primary_artist"): + resolved = itunes_client.resolve_primary_artist(artist_id) + if resolved and resolved != raw_album_artist: + raw_album_artist = resolved + except Exception: + pass + metadata["album_artist"] = raw_album_artist + + if album_info.get("is_album"): + metadata["album"] = album_info.get("album_name", "Unknown Album") + metadata["track_number"] = album_info.get("track_number", 1) + metadata["total_tracks"] = album_ctx.get("total_tracks", 1) if album_ctx else 1 + _get_logger(runtime).info("[METADATA] Album track - track_number: %s, album: %s", metadata["track_number"], metadata["album"]) + else: + if album_ctx and album_ctx.get("name"): + _get_logger(runtime).info("[SAFEGUARD] Using album context name instead of track title for album metadata") + metadata["album"] = album_ctx["name"] + metadata["track_number"] = album_info.get("track_number", 1) if album_info else 1 + metadata["total_tracks"] = album_ctx.get("total_tracks", 1) + else: + metadata["album"] = metadata["title"] + metadata["track_number"] = 1 + metadata["total_tracks"] = 1 + + disc_num = original_search.get("disc_number") + if disc_num is None and album_info: + disc_num = album_info.get("disc_number") + metadata["disc_number"] = disc_num if disc_num is not None else 1 + + if album_ctx and album_ctx.get("release_date"): + metadata["date"] = album_ctx["release_date"][:4] + + genres = artist_dict.get("genres") or [] + if genres: + from core.genre_filter import filter_genres + + filtered = filter_genres(list(genres[:2]), cfg) + if filtered: + metadata["genre"] = ", ".join(filtered) + + metadata["album_art_url"] = album_info.get("album_image_url") if album_info else None + if not metadata["album_art_url"] and album_ctx: + album_image = album_ctx.get("image_url") + if not album_image and album_ctx.get("images"): + first_image = album_ctx["images"][0] + album_image = first_image.get("url") if isinstance(first_image, dict) else None + metadata["album_art_url"] = album_image + + _get_logger(runtime).info( + "[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | track=%s/%s | disc=%s", + metadata.get("title"), + metadata.get("artist"), + metadata.get("album_artist"), + metadata.get("album"), + metadata.get("track_number"), + metadata.get("total_tracks"), + metadata.get("disc_number"), + ) + + return metadata + + +def embed_album_art_metadata(audio_file, metadata: dict, runtime=None): + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return + + try: + image_data = None + mime_type = None + + release_mbid = metadata.get("musicbrainz_release_id") + if release_mbid and cfg.get("metadata_enhancement.prefer_caa_art", False): + try: + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) + with urllib.request.urlopen(req, timeout=10) as response: + image_data = response.read() + mime_type = response.info().get_content_type() or "image/jpeg" + if not image_data or len(image_data) <= 1000: + image_data = None + except Exception: + image_data = None + + if not image_data: + art_url = metadata.get("album_art_url") + if not art_url: + logger_.warning("No album art URL available for embedding.") + return + with urllib.request.urlopen(art_url, timeout=10) as response: + image_data = response.read() + mime_type = response.info().get_content_type() or "image/jpeg" + + if not image_data: + logger_.error("Failed to download album art data.") + return + + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.APIC(encoding=3, mime=mime_type, type=3, desc="Cover", data=image_data)) + elif isinstance(audio_file, symbols.FLAC): + picture = symbols.Picture() + picture.data = image_data + picture.type = 3 + picture.mime = mime_type + width, height = _get_image_dimensions(image_data) + picture.width = width or 640 + picture.height = height or 640 + picture.depth = 24 + audio_file.add_picture(picture) + elif isinstance(audio_file, symbols.MP4): + fmt = symbols.MP4Cover.FORMAT_JPEG if "jpeg" in mime_type else symbols.MP4Cover.FORMAT_PNG + audio_file["covr"] = [symbols.MP4Cover(image_data, imageformat=fmt)] + + logger_.info("Album art successfully embedded.") + except Exception as exc: + logger_.error("Error embedding album art: %s", exc) + + +def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=None): + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return + + try: + tag_config = { + "SPOTIFY_TRACK_ID": "spotify.tags.track_id", + "SPOTIFY_ARTIST_ID": "spotify.tags.artist_id", + "SPOTIFY_ALBUM_ID": "spotify.tags.album_id", + "ITUNES_TRACK_ID": "itunes.tags.track_id", + "ITUNES_ARTIST_ID": "itunes.tags.artist_id", + "ITUNES_ALBUM_ID": "itunes.tags.album_id", + "MUSICBRAINZ_RECORDING_ID": "musicbrainz.tags.recording_id", + "MUSICBRAINZ_ARTIST_ID": "musicbrainz.tags.artist_id", + "MUSICBRAINZ_RELEASE_ID": "musicbrainz.tags.release_id", + "MUSICBRAINZ_RELEASEGROUPID": "musicbrainz.tags.release_group_id", + "MUSICBRAINZ_ALBUMARTISTID": "musicbrainz.tags.album_artist_id", + "MUSICBRAINZ_RELEASETRACKID": "musicbrainz.tags.release_track_id", + "RELEASETYPE": "musicbrainz.tags.release_type", + "ORIGINALDATE": "musicbrainz.tags.original_date", + "RELEASESTATUS": "musicbrainz.tags.release_status", + "RELEASECOUNTRY": "musicbrainz.tags.release_country", + "BARCODE": "musicbrainz.tags.barcode", + "MEDIA": "musicbrainz.tags.media", + "TOTALDISCS": "musicbrainz.tags.total_discs", + "CATALOGNUMBER": "musicbrainz.tags.catalog_number", + "SCRIPT": "musicbrainz.tags.script", + "ASIN": "musicbrainz.tags.asin", + "DEEZER_TRACK_ID": "deezer.tags.track_id", + "DEEZER_ARTIST_ID": "deezer.tags.artist_id", + "AUDIODB_TRACK_ID": "audiodb.tags.track_id", + "TIDAL_TRACK_ID": "tidal.tags.track_id", + "TIDAL_ARTIST_ID": "tidal.tags.artist_id", + "QOBUZ_TRACK_ID": "qobuz.tags.track_id", + "QOBUZ_ARTIST_ID": "qobuz.tags.artist_id", + "GENIUS_TRACK_ID": "genius.tags.track_id", + } + + def _tag_enabled(path: str) -> bool: + return cfg.get(path, True) is not False + + def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: + if not a or not b: + return False + from difflib import SequenceMatcher + + norm = lambda s: re.sub(r"[^a-z0-9 ]", "", re.sub(r"\(.*?\)", "", s).lower()).strip() + return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold + + context = normalize_import_context(context) + source = (metadata.get("source") or "").strip().lower() + source_ids = {} + if source: + source_tag_names = get_source_tag_names(source) + source_track_id = metadata.get("source_track_id") + source_artist_id = metadata.get("source_artist_id") + source_album_id = metadata.get("source_album_id") + if cfg.get(f"{source}.embed_tags", True) is not False: + if source_tag_names.get("track") and source_track_id: + source_ids[source_tag_names["track"]] = source_track_id + if source_tag_names.get("artist") and source_artist_id: + source_ids[source_tag_names["artist"]] = source_artist_id + if source_tag_names.get("album") and source_album_id: + source_ids[source_tag_names["album"]] = source_album_id + + if not source_ids: + if cfg.get("spotify.embed_tags", True) is not False: + if metadata.get("spotify_track_id"): + source_ids["SPOTIFY_TRACK_ID"] = metadata["spotify_track_id"] + if metadata.get("spotify_artist_id"): + source_ids["SPOTIFY_ARTIST_ID"] = metadata["spotify_artist_id"] + if metadata.get("spotify_album_id"): + source_ids["SPOTIFY_ALBUM_ID"] = metadata["spotify_album_id"] + if cfg.get("itunes.embed_tags", True) is not False: + if metadata.get("itunes_track_id"): + source_ids["ITUNES_TRACK_ID"] = metadata["itunes_track_id"] + if metadata.get("itunes_artist_id"): + source_ids["ITUNES_ARTIST_ID"] = metadata["itunes_artist_id"] + if metadata.get("itunes_album_id"): + source_ids["ITUNES_ALBUM_ID"] = metadata["itunes_album_id"] + + track_title = metadata.get("title", "") + artist_name = metadata.get("album_artist", "") or metadata.get("artist", "") + track_info = get_import_track_info(context) + explicit_artist = (track_info or {}).get("_explicit_artist_context") if isinstance(track_info, dict) else None + batch_artist_name = None + if isinstance(explicit_artist, dict) and explicit_artist.get("name"): + batch_artist_name = explicit_artist["name"] + elif isinstance(explicit_artist, str) and explicit_artist: + batch_artist_name = explicit_artist + + pp = { + "id_tags": source_ids, + "track_title": track_title, + "artist_name": artist_name, + "batch_artist_name": batch_artist_name, + "metadata": metadata, + "recording_mbid": None, + "artist_mbid": None, + "release_mbid": "", + "mb_genres": [], + "isrc": None, + "deezer_bpm": None, + "deezer_isrc": None, + "audiodb_mood": None, + "audiodb_style": None, + "audiodb_genre": None, + "tidal_isrc": None, + "tidal_copyright": None, + "qobuz_isrc": None, + "qobuz_copyright": None, + "qobuz_label": None, + "lastfm_tags": [], + "lastfm_url": None, + "genius_url": None, + "release_year": None, + } + + source_order = cfg.get("metadata_enhancement.post_process_order", None) + if not isinstance(source_order, list) or not source_order: + source_order = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"] + + db = _get_database(runtime) + + for source_name in source_order: + if source_name == "musicbrainz": + if cfg.get("musicbrainz.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + mb_worker = getattr(runtime, "mb_worker", None) + mb_service = mb_worker.mb_service if mb_worker else None + if not mb_service: + continue + try: + result = mb_service.match_recording(track_title, artist_name) + if result and result.get("mbid"): + pp["recording_mbid"] = result["mbid"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"] + details = mb_service.mb_client.get_recording(pp["recording_mbid"], includes=["isrcs", "genres"]) + if details: + isrcs = details.get("isrcs", []) + if isrcs: + pp["isrc"] = isrcs[0] + pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)] + + track_artist_name = metadata.get("artist", "") or artist_name + if ", " in track_artist_name: + track_artist_name = track_artist_name.split(", ")[0] + artist_result = mb_service.match_artist(track_artist_name) + if artist_result and artist_result.get("mbid"): + pp["artist_mbid"] = artist_result["mbid"] + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"] + + album_name_for_mb = metadata.get("album", "") + if album_name_for_mb: + artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() + rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), artist_key) + rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) + with _MB_RELEASE_CACHE_LOCK: + cached = _MB_RELEASE_CACHE.get(rc_key_norm) + if cached is None: + cached = _MB_RELEASE_CACHE.get(rc_key_exact) + if cached is not None: + pp["release_mbid"] = cached + else: + try: + rc_result = mb_service.match_release(album_name_for_mb, artist_name) + pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" + except Exception: + pp["release_mbid"] = "" + _MB_RELEASE_CACHE[rc_key_norm] = pp["release_mbid"] + _MB_RELEASE_CACHE[rc_key_exact] = pp["release_mbid"] + if pp["release_mbid"]: + pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] + + if pp["release_mbid"]: + with _MB_RELEASE_DETAIL_CACHE_LOCK: + release_detail = _MB_RELEASE_DETAIL_CACHE.get(pp["release_mbid"]) + if release_detail is None: + release_detail = mb_service.mb_client.get_release( + pp["release_mbid"], + includes=["release-groups", "labels", "media", "artist-credits", "recordings"], + ) or {} + with _MB_RELEASE_DETAIL_CACHE_LOCK: + _MB_RELEASE_DETAIL_CACHE[pp["release_mbid"]] = release_detail + if release_detail: + rg = release_detail.get("release-group", {}) + if rg.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"] + ac = release_detail.get("artist-credit", []) + if ac and isinstance(ac[0], dict): + aa = ac[0].get("artist", {}) + if aa.get("id"): + pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"] + if rg.get("primary-type"): + pp["id_tags"]["RELEASETYPE"] = rg["primary-type"] + if rg.get("first-release-date"): + pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"] + if not pp["release_year"] and len(rg["first-release-date"]) >= 4: + year = rg["first-release-date"][:4] + if year.isdigit(): + pp["release_year"] = year + if release_detail.get("status"): + pp["id_tags"]["RELEASESTATUS"] = release_detail["status"] + if release_detail.get("country"): + pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"] + if release_detail.get("barcode"): + pp["id_tags"]["BARCODE"] = release_detail["barcode"] + media_list = release_detail.get("media", []) + if media_list: + fmt = media_list[0].get("format", "") + if fmt: + pp["id_tags"]["MEDIA"] = fmt + pp["id_tags"]["TOTALDISCS"] = str(len(media_list)) + label_info = release_detail.get("label-info", []) + if label_info and isinstance(label_info[0], dict): + cat = label_info[0].get("catalog-number", "") + if cat: + pp["id_tags"]["CATALOGNUMBER"] = cat + text_rep = release_detail.get("text-representation", {}) + if isinstance(text_rep, dict) and text_rep.get("script"): + pp["id_tags"]["SCRIPT"] = text_rep["script"] + if release_detail.get("asin"): + pp["id_tags"]["ASIN"] = release_detail["asin"] + track_num = metadata.get("track_number") + disc_num = metadata.get("disc_number") or 1 + if track_num and media_list: + try: + track_num_int = int(track_num) + disc_num_int = int(disc_num) + for medium in media_list: + if medium.get("position", 1) == disc_num_int: + for mtrack in (medium.get("tracks") or medium.get("track-list", [])): + if mtrack.get("position") == track_num_int: + if mtrack.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"] + release_recording = mtrack.get("recording", {}) + if release_recording.get("id"): + pp["recording_mbid"] = release_recording["id"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"] + break + break + except (ValueError, TypeError): + pass + except Exception as exc: + logger_.error("MusicBrainz lookup failed (non-fatal): %s", exc) + continue + + if source_name == "deezer": + if cfg.get("deezer.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + deezer_worker = getattr(runtime, "deezer_worker", None) + dz_client = deezer_worker.client if deezer_worker else None + if not dz_client: + continue + dz_result = dz_client.search_track(artist_name, track_title) + if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name): + dz_track_id = dz_result["id"] + pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id) + dz_artist_id = dz_result.get("artist", {}).get("id") + if dz_artist_id: + pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id) + dz_details = dz_client.get_track_details(dz_track_id) + if dz_details: + bpm_val = dz_details.get("bpm") + if bpm_val and bpm_val > 0: + pp["deezer_bpm"] = bpm_val + dz_isrc = dz_details.get("isrc") + if dz_isrc: + pp["deezer_isrc"] = dz_isrc + if not pp["release_year"]: + dz_album = dz_result.get("album", {}) + dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or "" + if len(dz_release) >= 4 and dz_release[:4].isdigit(): + pp["release_year"] = dz_release[:4] + except Exception as exc: + logger_.error("Deezer lookup failed (non-fatal): %s", exc) + continue + + if source_name == "audiodb": + if cfg.get("audiodb.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + audiodb_worker = getattr(runtime, "audiodb_worker", None) + adb_client = audiodb_worker.client if audiodb_worker else None + if not adb_client: + continue + adb_result = adb_client.search_track(artist_name, track_title) + if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name): + adb_track_id = adb_result.get("idTrack") + if adb_track_id: + pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id) + adb_mb_track = adb_result.get("strMusicBrainzID") + if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track + pp["recording_mbid"] = adb_mb_track + adb_mb_artist = adb_result.get("strMusicBrainzArtistID") + if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist + pp["artist_mbid"] = adb_mb_artist + pp["audiodb_mood"] = adb_result.get("strMood") or None + pp["audiodb_style"] = adb_result.get("strStyle") or None + pp["audiodb_genre"] = adb_result.get("strGenre") or None + except Exception as exc: + logger_.error("AudioDB lookup failed (non-fatal): %s", exc) + continue + + if source_name == "tidal": + if cfg.get("tidal.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + tidal_client = getattr(runtime, "tidal_client", None) + if not (tidal_client and tidal_client.is_authenticated()): + continue + td_result = tidal_client.search_track(artist_name, track_title) + if td_result and _names_match(td_result.get("title", ""), track_title): + td_track_id = td_result.get("id") + if td_track_id: + pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id) + td_artist = td_result.get("artist", {}) + if isinstance(td_artist, dict) and td_artist.get("id"): + pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"]) + if td_track_id: + td_details = tidal_client.get_track(str(td_track_id)) + if td_details: + pp["tidal_isrc"] = td_details.get("isrc") + td_copyright = td_details.get("copyright") + if isinstance(td_copyright, dict): + td_copyright = td_copyright.get("text", td_copyright.get("name", "")) + pp["tidal_copyright"] = td_copyright or None + if not pp["release_year"]: + td_album = td_result.get("album", {}) + td_release = "" + if isinstance(td_album, dict): + td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "") + if len(td_release) >= 4 and td_release[:4].isdigit(): + pp["release_year"] = td_release[:4] + except Exception as exc: + logger_.error("Tidal lookup failed (non-fatal): %s", exc) + continue + + if source_name == "qobuz": + if cfg.get("qobuz.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None) + qz_client = qobuz_worker.client if qobuz_worker else None + if not (qz_client and qz_client.is_authenticated()): + continue + qz_result = qz_client.search_track(artist_name, track_title) + if qz_result: + qz_performer = qz_result.get("performer") or {} + if not isinstance(qz_performer, dict): + qz_performer = {} + qz_artist_name = qz_performer.get("name", "") + if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name): + qz_track_id = qz_result.get("id") + if qz_track_id: + pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id) + if qz_performer.get("id"): + pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"]) + qz_isrc = qz_result.get("isrc") + if isinstance(qz_isrc, dict): + qz_isrc = qz_isrc.get("value", qz_isrc.get("id", "")) + if qz_isrc: + pp["qobuz_isrc"] = qz_isrc + qz_copyright = qz_result.get("copyright") + if isinstance(qz_copyright, dict): + qz_copyright = qz_copyright.get("text", qz_copyright.get("name", "")) + if isinstance(qz_copyright, str): + pp["qobuz_copyright"] = qz_copyright + qz_album = qz_result.get("album", {}) + if isinstance(qz_album, dict): + qz_label_info = qz_album.get("label", {}) + if isinstance(qz_label_info, dict) and qz_label_info.get("name"): + pp["qobuz_label"] = qz_label_info["name"] + if not pp["release_year"]: + qz_release = str(qz_album.get("release_date_original", "") or "") + if not qz_release: + qz_ts = qz_album.get("released_at") + if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: + import datetime as _dt + qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) + if len(qz_release) >= 4 and qz_release[:4].isdigit(): + pp["release_year"] = qz_release[:4] + except Exception as exc: + logger_.error("Qobuz lookup failed (non-fatal): %s", exc) + continue + + if source_name == "lastfm": + if cfg.get("lastfm.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + lastfm_worker = getattr(runtime, "lastfm_worker", None) + lf_client = lastfm_worker.client if lastfm_worker else None + if not lf_client: + continue + lf_result = lf_client.get_track_info(artist_name, track_title) + if lf_result: + lf_url = lf_result.get("url") + if lf_url: + pp["lastfm_url"] = lf_url + lf_toptags = lf_result.get("toptags", {}) + if isinstance(lf_toptags, dict): + tag_list = lf_toptags.get("tag", []) + if isinstance(tag_list, list): + pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")] + elif isinstance(tag_list, dict) and tag_list.get("name"): + pp["lastfm_tags"] = [tag_list["name"]] + except Exception as exc: + logger_.error("Last.fm lookup failed (non-fatal): %s", exc) + continue + + if source_name == "genius": + if cfg.get("genius.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + import core.genius_client as _genius_module + + if time.time() < _genius_module._rate_limit_until: + logger_.info("Genius rate-limited, skipping (non-blocking)") + continue + genius_worker = getattr(runtime, "genius_worker", None) + g_client = genius_worker.client if genius_worker else None + if not g_client: + continue + g_result = g_client.search_song(artist_name, track_title) + if g_result: + g_id = g_result.get("id") + if g_id: + pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id) + g_url = g_result.get("url") + if g_url: + pp["genius_url"] = g_url + except Exception as exc: + logger_.error("Genius lookup failed (non-fatal): %s", exc) + continue + + if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["audiodb_mood"] and not pp["audiodb_style"]: + return + + filtered_tags: Dict[str, str] = {} + for tag_name, value in pp["id_tags"].items(): + config_path = tag_config.get(tag_name) + if config_path and not _tag_enabled(config_path): + continue + filtered_tags[tag_name] = value + + written = [] + id3_tag_map = { + "MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"), + "MUSICBRAINZ_ARTIST_ID": ("TXXX", "MusicBrainz Artist Id"), + "MUSICBRAINZ_RELEASE_ID": ("TXXX", "MusicBrainz Album Id"), + "MUSICBRAINZ_RELEASEGROUPID": ("TXXX", "MusicBrainz Release Group Id"), + "MUSICBRAINZ_ALBUMARTISTID": ("TXXX", "MusicBrainz Album Artist Id"), + "MUSICBRAINZ_RELEASETRACKID": ("TXXX", "MusicBrainz Release Track Id"), + "RELEASETYPE": ("TXXX", "MusicBrainz Album Type"), + "RELEASESTATUS": ("TXXX", "MusicBrainz Album Status"), + "RELEASECOUNTRY": ("TXXX", "MusicBrainz Album Release Country"), + "ORIGINALDATE": ("TDOR", None), + "MEDIA": ("TMED", None), + } + vorbis_tag_map = { + "MUSICBRAINZ_RECORDING_ID": "MUSICBRAINZ_TRACKID", + "MUSICBRAINZ_ARTIST_ID": "MUSICBRAINZ_ARTISTID", + "MUSICBRAINZ_RELEASE_ID": "MUSICBRAINZ_ALBUMID", + "MUSICBRAINZ_RELEASEGROUPID": "MUSICBRAINZ_RELEASEGROUPID", + "MUSICBRAINZ_ALBUMARTISTID": "MUSICBRAINZ_ALBUMARTISTID", + "MUSICBRAINZ_RELEASETRACKID": "MUSICBRAINZ_RELEASETRACKID", + } + mp4_tag_map = { + "MUSICBRAINZ_RECORDING_ID": "MusicBrainz Track Id", + "MUSICBRAINZ_ARTIST_ID": "MusicBrainz Artist Id", + "MUSICBRAINZ_RELEASE_ID": "MusicBrainz Album Id", + "MUSICBRAINZ_RELEASEGROUPID": "MusicBrainz Release Group Id", + "MUSICBRAINZ_ALBUMARTISTID": "MusicBrainz Album Artist Id", + "MUSICBRAINZ_RELEASETRACKID": "MusicBrainz Release Track Id", + "RELEASETYPE": "MusicBrainz Album Type", + "RELEASESTATUS": "MusicBrainz Album Status", + "RELEASECOUNTRY": "MusicBrainz Album Release Country", + } + + if isinstance(audio_file.tags, symbols.ID3): + for tag_name, value in filtered_tags.items(): + spec = id3_tag_map.get(tag_name) + if spec: + frame_type, desc = spec + if frame_type == "UFID": + audio_file.tags.add(symbols.UFID(owner=desc, data=str(value).encode("ascii"))) + written.append(f"UFID:{desc}") + elif frame_type == "TDOR": + audio_file.tags.add(symbols.TDOR(encoding=3, text=[value])) + written.append("TDOR") + elif frame_type == "TMED": + audio_file.tags.add(symbols.TMED(encoding=3, text=[value])) + written.append("TMED") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=desc, text=[value])) + written.append(f"TXXX:{desc}") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=tag_name, text=[str(value)])) + written.append(f"TXXX:{tag_name}") + elif _is_vorbis_like(audio_file, symbols): + for tag_name, value in filtered_tags.items(): + audio_file[vorbis_tag_map.get(tag_name, tag_name)] = [str(value)] + written.append(vorbis_tag_map.get(tag_name, tag_name)) + elif isinstance(audio_file, symbols.MP4): + for tag_name, value in filtered_tags.items(): + key = f"----:com.apple.iTunes:{mp4_tag_map.get(tag_name, tag_name)}" + audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))] + written.append(key) + + if written: + logger_.info("Embedded IDs: %s", ", ".join(written)) + + release_year = pp["release_year"] + needs_date_tag = bool(release_year and not metadata.get("date")) + if needs_date_tag: + metadata["date"] = release_year + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["date"] = [release_year] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9day"] = [release_year] + logger_.info("Date tag: %s", release_year) + + if _tag_enabled("deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0: + bpm_int = int(pp["deezer_bpm"]) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["BPM"] = [str(bpm_int)] + elif isinstance(audio_file, symbols.MP4): + audio_file["tmpo"] = [bpm_int] + logger_.info("BPM: %s", bpm_int) + + if _tag_enabled("audiodb.tags.mood") and pp["audiodb_mood"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["MOOD"] = [pp["audiodb_mood"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))] + + if _tag_enabled("audiodb.tags.style") and pp["audiodb_style"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["STYLE"] = [pp["audiodb_style"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))] + + if _tag_enabled("metadata_enhancement.tags.genre_merge"): + enrichment_genres = [] + if _tag_enabled("musicbrainz.tags.genres"): + enrichment_genres += pp["mb_genres"] + if pp["audiodb_genre"] and _tag_enabled("audiodb.tags.genre"): + enrichment_genres.append(pp["audiodb_genre"]) + if _tag_enabled("lastfm.tags.genres"): + enrichment_genres += pp["lastfm_tags"] + if enrichment_genres: + from core.genre_filter import filter_genres as _filter_genres + + enrichment_genres = _filter_genres(enrichment_genres, cfg) + source_genres = [g.strip() for g in str(metadata.get("genre", "")).split(",") if g.strip()] + seen = set() + merged = [] + for genre in source_genres + enrichment_genres: + key = genre.strip().lower() + if key and key not in seen: + seen.add(key) + merged.append(genre.strip().title()) + if len(merged) >= 5: + break + if merged: + genre_string = ", ".join(merged) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["GENRE"] = [genre_string] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9gen"] = [genre_string] + logger_.info("Genres merged: %s", genre_string) + + isrc_candidates = [] + if pp["isrc"] and _tag_enabled("musicbrainz.tags.isrc"): + isrc_candidates.append(("MusicBrainz", pp["isrc"])) + if pp["deezer_isrc"] and _tag_enabled("deezer.tags.isrc"): + isrc_candidates.append(("Deezer", pp["deezer_isrc"])) + if pp["tidal_isrc"] and _tag_enabled("tidal.tags.isrc"): + isrc_candidates.append(("Tidal", pp["tidal_isrc"])) + if pp["qobuz_isrc"] and _tag_enabled("qobuz.tags.isrc"): + isrc_candidates.append(("Qobuz", pp["qobuz_isrc"])) + if isrc_candidates: + isrc_source, final_isrc = isrc_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["ISRC"] = [final_isrc] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))] + logger_.info("ISRC (%s): %s", isrc_source, final_isrc) + + copyright_candidates = [] + if pp["tidal_copyright"] and _tag_enabled("tidal.tags.copyright"): + copyright_candidates.append(("Tidal", pp["tidal_copyright"])) + if pp["qobuz_copyright"] and _tag_enabled("qobuz.tags.copyright"): + copyright_candidates.append(("Qobuz", pp["qobuz_copyright"])) + if copyright_candidates: + copyright_source, final_copyright = copyright_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["COPYRIGHT"] = [final_copyright] + elif isinstance(audio_file, symbols.MP4): + audio_file["cprt"] = [final_copyright] + logger_.info("Copyright (%s): %s", copyright_source, final_copyright[:60]) + + if _tag_enabled("qobuz.tags.label") and pp["qobuz_label"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["LABEL"] = [pp["qobuz_label"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))] + + if _tag_enabled("lastfm.tags.url") and pp["lastfm_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["LASTFM_URL"] = [pp["lastfm_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))] + + if _tag_enabled("genius.tags.url") and pp["genius_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["GENIUS_URL"] = [pp["genius_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))] + + release_id = pp["release_mbid"] + if release_id: + metadata["musicbrainz_release_id"] = release_id + if db is not None: + try: + album_name_for_db = metadata.get("album", "") + album_artist_for_db = metadata.get("album_artist", "") or metadata.get("artist", "") + if album_name_for_db and album_artist_for_db: + conn = db._get_connection() + try: + cursor = conn.cursor() + cursor.execute( + """ + UPDATE albums SET year = ? + WHERE (year IS NULL OR year = 0) + AND id IN ( + SELECT al.id FROM albums al + JOIN artists ar ON ar.id = al.artist_id + WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) + ) + """, + (int(release_year), album_name_for_db, album_artist_for_db), + ) + if cursor.rowcount > 0: + conn.commit() + logger_.info("Updated album year to %s in database", release_year) + else: + conn.rollback() + finally: + conn.close() + except Exception as exc: + logger_.error("Could not update album year in DB: %s", exc) + + except Exception as exc: + logger_.error("Error embedding source IDs (non-fatal): %s", exc) + + +def download_cover_art(album_info: dict, target_dir: str, context: dict = None, runtime=None): + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + if cfg.get("metadata_enhancement.cover_art_download", True) is False: + return + + try: + cover_path = os.path.join(target_dir, "cover.jpg") + album_info = album_info or {} + release_mbid = album_info.get("musicbrainz_release_id") + prefer_caa = cfg.get("metadata_enhancement.prefer_caa_art", False) + + if os.path.exists(cover_path): + if release_mbid and prefer_caa: + try: + existing_size = os.path.getsize(cover_path) + if existing_size > 200_000: + return + is_upgrade = True + except Exception: + return + else: + return + else: + is_upgrade = False + + image_data = None + if release_mbid and prefer_caa: + try: + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) + with urllib.request.urlopen(req, timeout=10) as response: + image_data = response.read() + if not image_data or len(image_data) <= 1000: + image_data = None + except Exception: + image_data = None + + if is_upgrade and not image_data: + logger_.error("CAA upgrade failed - keeping existing cover.jpg") + return + + if not image_data: + art_url = album_info.get("album_image_url") + if not art_url and context: + album_ctx = get_import_context_album(context) + art_url = album_ctx.get("image_url") + if not art_url and album_ctx.get("images"): + images = album_ctx.get("images", []) + if images and isinstance(images[0], dict): + art_url = images[0].get("url", "") + if art_url: + logger_.info("Using cover art URL from album context") + if art_url and "i.scdn.co" in art_url: + try: + from core.spotify_client import _upgrade_spotify_image_url + + art_url = _upgrade_spotify_image_url(art_url) + except Exception: + pass + elif art_url and "mzstatic.com" in art_url: + import re as _re + + art_url = _re.sub(r"\d+x\d+bb", "3000x3000bb", art_url) + if not art_url: + logger_.warning("No cover art URL available for download.") + return + with urllib.request.urlopen(art_url, timeout=10) as response: + image_data = response.read() + + if not image_data: + return + + with open(cover_path, "wb") as handle: + handle.write(image_data) + logger_.info("Cover art downloaded to: %s", cover_path) + except Exception as exc: + logger_.error("Error downloading cover.jpg: %s", exc) + + +def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + if cfg.get("metadata_enhancement.lrclib_enabled", True) is False: + return False + + try: + from core.lyrics_client import lyrics_client + + context = normalize_import_context(context) + original_search = get_import_original_search(context) + album_context = get_import_context_album(context) + track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track")) + + if isinstance(artist, dict): + artist_name = artist.get("name", "Unknown Artist") + elif hasattr(artist, "name"): + artist_name = artist.name + else: + artist_name = str(artist) if artist else "Unknown Artist" + + album_name = None + duration_seconds = None + if album_info and album_info.get("is_album"): + album_name = ( + get_import_clean_album(context, album_info=album_info, default="") + or album_info.get("album_name") + or album_context.get("name") + ) + + if original_search.get("duration_ms"): + duration_seconds = int(original_search["duration_ms"] / 1000) + + success = lyrics_client.create_lrc_file( + audio_file_path=file_path, + track_name=track_name, + artist_name=artist_name, + album_name=album_name, + duration_seconds=duration_seconds, + ) + + if success: + logger_.info("LRC file generated for: %s", track_name) + else: + logger_.warning("No lyrics found for: %s", track_name) + return success + except Exception as exc: + logger_.error("Error generating LRC file for %s: %s", file_path, exc) + return False + + +def wipe_source_tags(file_path: str, runtime=None) -> bool: + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + _ = cfg # keep signature parallel with other helpers + + try: + _strip_all_non_audio_tags(file_path, runtime=runtime) + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return False + + audio = symbols.File(file_path) + if audio is None: + return False + if hasattr(audio, "clear_pictures"): + audio.clear_pictures() + if audio.tags is not None: + tag_count = len(audio.tags) + audio.tags.clear() + else: + audio.add_tags() + tag_count = 0 + _save_audio_file(audio, symbols) + if tag_count > 0: + logger_.info("[Tag Wipe] Stripped %s source tags from: %s", tag_count, os.path.basename(file_path)) + return True + except Exception as exc: + logger_.error("[Tag Wipe] Failed (non-fatal): %s", exc) + return False + + +def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + if cfg.get("metadata_enhancement.enabled", True) is False: + logger_.warning("Metadata enhancement disabled in config.") + return True + + if album_info is None: + album_info = {} + + symbols = _get_mutagen_symbols(runtime) + if not symbols: + logger_.error("Mutagen is unavailable, cannot enhance metadata.") + return False + + file_lock = _get_file_lock(file_path) + with file_lock: + logger_.info("Enhancing metadata for: %s", os.path.basename(file_path)) + try: + _strip_all_non_audio_tags(file_path, runtime=runtime) + audio_file = symbols.File(file_path) + if audio_file is None: + logger_.error("Could not load audio file with Mutagen: %s", file_path) + return False + + if hasattr(audio_file, "clear_pictures"): + audio_file.clear_pictures() + + if audio_file.tags is not None: + if len(audio_file.tags) > 0: + tag_keys = list(audio_file.tags.keys())[:15] + logger_.info("Clearing %s existing tags: %s", len(audio_file.tags), ", ".join(str(k) for k in tag_keys)) + audio_file.tags.clear() + else: + audio_file.add_tags() + + _save_audio_file(audio_file, symbols) + + metadata = extract_source_metadata(context, artist, album_info, runtime=runtime) + if not metadata: + logger_.error("Could not extract source metadata, saving with cleared tags.") + _save_audio_file(audio_file, symbols) + return True + + track_num_str = f"{metadata.get('track_number', 1)}/{metadata.get('total_tracks', 1)}" + write_multi = cfg.get("metadata_enhancement.tags.write_multi_artist", False) + artists_list = metadata.get("_artists_list", []) + + if isinstance(audio_file.tags, symbols.ID3): + if metadata.get("title"): + audio_file.tags.add(symbols.TIT2(encoding=3, text=[metadata["title"]])) + if metadata.get("artist"): + audio_file.tags.add(symbols.TPE1(encoding=3, text=[metadata["artist"]])) + if write_multi and len(artists_list) > 1: + audio_file.tags.add(symbols.TPE1(encoding=3, text=artists_list)) + if metadata.get("album_artist"): + audio_file.tags.add(symbols.TPE2(encoding=3, text=[metadata["album_artist"]])) + if metadata.get("album"): + audio_file.tags.add(symbols.TALB(encoding=3, text=[metadata["album"]])) + if metadata.get("date"): + audio_file.tags.add(symbols.TDRC(encoding=3, text=[metadata["date"]])) + if metadata.get("genre"): + audio_file.tags.add(symbols.TCON(encoding=3, text=[metadata["genre"]])) + audio_file.tags.add(symbols.TRCK(encoding=3, text=[track_num_str])) + if metadata.get("disc_number"): + audio_file.tags.add(symbols.TPOS(encoding=3, text=[str(metadata["disc_number"])])) + elif _is_vorbis_like(audio_file, symbols): + if metadata.get("title"): + audio_file["title"] = [metadata["title"]] + if metadata.get("artist"): + audio_file["artist"] = [metadata["artist"]] + if write_multi and len(artists_list) > 1: + audio_file["artists"] = artists_list + if metadata.get("album_artist"): + audio_file["albumartist"] = [metadata["album_artist"]] + if metadata.get("album"): + audio_file["album"] = [metadata["album"]] + if metadata.get("date"): + audio_file["date"] = [metadata["date"]] + if metadata.get("genre"): + audio_file["genre"] = [metadata["genre"]] + audio_file["tracknumber"] = [track_num_str] + if metadata.get("disc_number"): + audio_file["discnumber"] = [str(metadata["disc_number"])] + elif isinstance(audio_file, symbols.MP4): + if metadata.get("title"): + audio_file["\xa9nam"] = [metadata["title"]] + if metadata.get("artist"): + audio_file["\xa9ART"] = artists_list if (write_multi and len(artists_list) > 1) else [metadata["artist"]] + if metadata.get("album_artist"): + audio_file["aART"] = [metadata["album_artist"]] + if metadata.get("album"): + audio_file["\xa9alb"] = [metadata["album"]] + if metadata.get("date"): + audio_file["\xa9day"] = [metadata["date"]] + if metadata.get("genre"): + audio_file["\xa9gen"] = [metadata["genre"]] + audio_file["trkn"] = [(metadata.get("track_number", 1), metadata.get("total_tracks", 1))] + if metadata.get("disc_number"): + audio_file["disk"] = [(metadata["disc_number"], 0)] + + embed_source_ids(audio_file, metadata, context, runtime=runtime) + + if album_info is not None and metadata.get("musicbrainz_release_id"): + album_info["musicbrainz_release_id"] = metadata["musicbrainz_release_id"] + + if cfg.get("metadata_enhancement.embed_album_art", True): + embed_album_art_metadata(audio_file, metadata, runtime=runtime) + + quality = context.get("_audio_quality", "") + if quality and cfg.get("metadata_enhancement.tags.quality_tag", True) is not False: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="QUALITY", text=[quality])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["quality"] = [quality] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:QUALITY"] = [symbols.MP4FreeForm(quality.encode("utf-8"))] + + _save_audio_file(audio_file, symbols) + + verified = _verify_metadata_written(file_path, runtime=runtime) + if verified: + logger_.info("Metadata enhanced successfully.") + else: + logger_.info("Metadata saved but verification found issues (see above).") + return True + except Exception as exc: + import traceback + + logger_.error("Error enhancing metadata for %s: %s", file_path, exc) + logger_.error("[Metadata Debug] Exception type: %s", type(exc).__name__) + logger_.info("[Metadata Debug] File exists: %s", os.path.exists(file_path)) + logger_.warning("[Metadata Debug] Artist: %s", artist.get("name", "MISSING") if artist else "None") + logger_.warning("[Metadata Debug] Album info: %s", album_info.get("album_name", "MISSING") if album_info else "None") + logger_.error("[Metadata Debug] Traceback:\n%s", traceback.format_exc()) + return False + + +def _normalize_album_cache_key(album_name: str) -> str: + result = _EDITION_PAREN_RE.sub("", album_name or "") + result = _EDITION_BARE_RE.sub("", result) + return result.lower().strip() diff --git a/core/metadata_service.py b/core/metadata_service.py index e2636d0f..861f0b08 100644 --- a/core/metadata_service.py +++ b/core/metadata_service.py @@ -719,6 +719,23 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti if not isinstance(images, list): images = list(images) if images else [] + artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) + if not artists and artist_name: + artists = [{'name': artist_name}] + + primary_artist = artists[0] if artists else {} + resolved_artist_name = ( + _extract_lookup_value(primary_artist, 'name', default='') + or artist_name + or _extract_lookup_value(album_data, 'artist_name', 'artist', default='') + or '' + ) + resolved_artist_id = str( + _extract_lookup_value(primary_artist, 'id', default='') + or _extract_lookup_value(album_data, 'artist_id', default='') + or '' + ).strip() + image_url = None if images: image_url = _extract_lookup_value(images[0], 'url') @@ -728,12 +745,15 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti return { 'id': _extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', 'release_id', default=album_id) or album_id, 'name': _extract_lookup_value(album_data, 'name', 'title', default=album_name or album_id) or album_name or album_id, + 'artist': resolved_artist_name or '', + 'artist_name': resolved_artist_name or '', + 'artist_id': resolved_artist_id, + 'artists': artists, 'image_url': image_url, 'images': images, 'release_date': _extract_lookup_value(album_data, 'release_date', default='') or '', 'album_type': _extract_lookup_value(album_data, 'album_type', default='album') or 'album', 'total_tracks': _extract_lookup_value(album_data, 'total_tracks', 'track_count', default=0) or 0, - 'artist_name': artist_name or _extract_lookup_value(album_data, 'artist_name', default='') or '', } @@ -754,6 +774,8 @@ def _build_album_track_entry(track_item: Any, album_info: Dict[str, Any], source 'external_urls': _extract_lookup_value(track_item, 'external_urls', default={}) or {}, 'uri': _extract_lookup_value(track_item, 'uri', default='') or '', 'album': album_info, + 'source': source, + 'provider': source, '_source': source, } @@ -767,6 +789,9 @@ def _build_album_tracks_payload( artist_name: str = '', ) -> Dict[str, Any]: album_info = _build_album_info(album_data, album_id, album_name=album_name, artist_name=artist_name) + album_info['source'] = source + album_info['_source'] = source + album_info['provider'] = source track_items = _extract_album_track_items(album_data, tracks_data) tracks = [_build_album_track_entry(track, album_info, source) for track in track_items] @@ -778,6 +803,369 @@ def _build_album_tracks_payload( } +def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]: + if not artists: + return [] + + if isinstance(artists, (str, bytes)): + artists = [artists] + elif isinstance(artists, dict): + artists = [artists] + else: + try: + artists = list(artists) + except TypeError: + artists = [artists] + + normalized: List[Dict[str, Any]] = [] + for artist in artists: + if isinstance(artist, dict): + name = _extract_lookup_value(artist, 'name', 'artist_name', 'title', default='') or '' + artist_id = _extract_lookup_value(artist, 'id', 'artist_id', default='') or '' + entry: Dict[str, Any] = {} + if name: + entry['name'] = str(name) + if artist_id: + entry['id'] = str(artist_id) + genres = _extract_lookup_value(artist, 'genres', default=None) + if genres is not None: + entry['genres'] = genres + if entry: + normalized.append(entry) + continue + + name = str(artist).strip() + if name: + normalized.append({'name': name}) + + return normalized + + +def _build_single_import_context_payload( + track_data: Any, + source: Optional[str], + source_priority: List[str], + requested_title: str = '', + requested_artist: str = '', +) -> Dict[str, Any]: + album_data = _extract_lookup_value(track_data, 'album', default=None) + + track_id = str(_extract_lookup_value(track_data, 'id', 'track_id', 'trackId', default='') or '') + track_name = _extract_lookup_value(track_data, 'name', 'title', 'trackName', default='') or requested_title or 'Unknown Track' + track_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'artists', default=[])) + if not track_artists and requested_artist: + track_artists = [{'name': requested_artist}] + + primary_track_artist = track_artists[0] if track_artists else {} + primary_artist_name = primary_track_artist.get('name') or requested_artist or 'Unknown Artist' + primary_artist_id = str(primary_track_artist.get('id', '') or _extract_lookup_value(track_data, 'artist_id', 'artistId', default='') or '') + + album_name = _extract_lookup_value(track_data, 'album_name', 'collectionName', default='') or '' + album_id = str(_extract_lookup_value(track_data, 'album_id', 'collectionId', 'albumId', default='') or '') + release_date = str(_extract_lookup_value(track_data, 'release_date', default='') or '') + album_type = str(_extract_lookup_value(track_data, 'album_type', default='album') or 'album') + total_tracks = int(_extract_lookup_value(track_data, 'total_tracks', 'track_count', default=0) or 0) + album_images: List[Dict[str, Any]] = [] + album_image_url = str(_extract_lookup_value(track_data, 'image_url', 'thumb_url', default='') or '') + album_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'album_artists', 'artists', default=[])) + + if isinstance(album_data, dict): + album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name + album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id) + release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date) + album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type) + total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks) + album_images = _extract_lookup_value(album_data, 'images', default=[]) or [] + if not album_image_url: + album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '') + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) + elif album_data: + album_name = album_name or str(album_data) + + if not album_artists and primary_artist_name: + album_artists = [{'name': primary_artist_name}] + + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + + track_info = { + 'id': track_id, + 'name': track_name, + 'track_number': int(_extract_lookup_value(track_data, 'track_number', 'trackNumber', default=1) or 1), + 'disc_number': int(_extract_lookup_value(track_data, 'disc_number', 'discNumber', default=1) or 1), + 'duration_ms': int(_extract_lookup_value(track_data, 'duration_ms', 'duration', 'trackTimeMillis', default=0) or 0), + 'artists': track_artists or [{'name': primary_artist_name}], + 'uri': str(_extract_lookup_value(track_data, 'uri', default='') or ''), + 'album': album_name, + 'album_id': album_id, + 'album_type': album_type, + 'release_date': release_date, + '_source': source or '', + } + + album_payload = { + 'id': album_id, + 'name': album_name, + 'release_date': release_date, + 'total_tracks': total_tracks or 1, + 'album_type': album_type, + 'image_url': album_image_url, + 'images': album_images, + 'artists': album_artists, + '_source': source or '', + } + + artist_payload = { + 'id': primary_artist_id, + 'name': primary_artist_name, + 'genres': [], + '_source': source or '', + } + + original_search = { + 'title': track_name, + 'artist': primary_artist_name, + 'album': album_name, + 'track_number': track_info['track_number'], + 'disc_number': track_info['disc_number'], + 'clean_title': track_name, + 'clean_album': album_name, + 'clean_artist': primary_artist_name, + 'artists': track_info['artists'], + 'duration_ms': track_info['duration_ms'], + 'id': track_id, + '_source': source or '', + } + + return { + 'success': bool(track_id or track_name != requested_title or album_name), + 'source': source, + 'source_priority': source_priority, + 'context': { + 'artist': artist_payload, + 'album': album_payload, + 'track_info': track_info, + 'original_search_result': original_search, + 'is_album_download': False, + 'has_clean_metadata': bool(track_id), + 'has_full_metadata': bool(track_id), + 'source': source, + 'source_priority': source_priority, + }, + } + + +def _build_single_import_fallback_context( + requested_title: str, + requested_artist: str, + source_priority: List[str], +) -> Dict[str, Any]: + artist_name = requested_artist or 'Unknown Artist' + title = requested_title or 'Unknown Track' + return { + 'success': False, + 'source': None, + 'source_priority': source_priority, + 'context': { + 'artist': { + 'id': '', + 'name': artist_name, + 'genres': [], + '_source': '', + }, + 'album': { + 'id': '', + 'name': '', + 'release_date': '', + 'total_tracks': 1, + 'album_type': 'album', + 'image_url': '', + 'images': [], + 'artists': [], + '_source': '', + }, + 'track_info': { + 'id': '', + 'name': title, + 'track_number': 1, + 'disc_number': 1, + 'duration_ms': 0, + 'artists': [{'name': artist_name}], + 'uri': '', + 'album': '', + 'album_id': '', + 'album_type': 'album', + 'release_date': '', + '_source': '', + }, + 'original_search_result': { + 'title': title, + 'artist': artist_name, + 'album': '', + 'track_number': 1, + 'disc_number': 1, + 'clean_title': title, + 'clean_album': '', + 'clean_artist': artist_name, + 'artists': [{'name': artist_name}], + 'duration_ms': 0, + 'id': '', + '_source': '', + }, + 'is_album_download': False, + 'has_clean_metadata': False, + 'has_full_metadata': False, + 'source': None, + 'source_priority': source_priority, + }, + } + + +def _build_track_search_query(source: str, title: str, artist: str) -> str: + base_query = " ".join(part for part in (title, artist) if part).strip() + if source == 'deezer' and title: + if artist: + return f'artist:"{artist}" track:"{title}"' + return f'track:"{title}"' + return base_query or title or artist + + +def _pick_best_track_match(search_results: List[Any], title: str, artist: str = '') -> Optional[Any]: + if not search_results: + return None + + target_title = str(title or '').strip().lower() + target_artist = str(artist or '').strip().lower() + + for candidate in search_results: + candidate_title = str(_extract_lookup_value(candidate, 'name', 'title', 'track_name', default='') or '').strip().lower() + if candidate_title != target_title: + continue + + if not target_artist: + return candidate + + candidate_artists = _normalize_context_artists(_extract_lookup_value(candidate, 'artists', default=[])) + candidate_artist_name = candidate_artists[0]['name'].strip().lower() if candidate_artists else '' + if candidate_artist_name == target_artist: + return candidate + + return search_results[0] + + +def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]: + if not client or not hasattr(client, 'search_tracks'): + return [] + + try: + kwargs = {'limit': limit} + if source == 'spotify': + kwargs['allow_fallback'] = False + return client.search_tracks(query, **kwargs) or [] + except Exception as exc: + logger.debug("Could not search %s for %s: %s", source, query, exc) + return [] + + +def get_single_track_import_context( + title: str, + artist: str = '', + override_id: Optional[str] = None, + override_source: str = 'spotify', + source_override: Optional[str] = None, +) -> Dict[str, Any]: + """Build an import context for singles using source-priority metadata lookup.""" + options = MetadataLookupOptions(source_override=source_override, allow_fallback=True) + source_priority = _get_source_chain_for_lookup(options) + title = (title or '').strip() + artist = (artist or '').strip() + + if override_id: + chosen_source = (override_source or 'spotify').strip().lower() or 'spotify' + client = get_client_for_source(chosen_source) + if client and hasattr(client, 'get_track_details'): + try: + track_data = client.get_track_details(str(override_id)) + if track_data: + payload = _build_single_import_context_payload( + track_data, + chosen_source, + source_priority, + requested_title=title, + requested_artist=artist, + ) + if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): + try: + artist_details = client.get_artist(payload['context']['artist']['id']) + if artist_details: + payload['context']['artist']['genres'] = _extract_lookup_value( + artist_details, + 'genres', + default=[], + ) or [] + except Exception: + pass + return payload + except Exception as exc: + logger.debug("Override track lookup failed on %s for %s: %s", chosen_source, override_id, exc) + + for source in source_priority: + client = get_client_for_source(source) + if not client: + continue + + search_query = _build_track_search_query(source, title, artist) + if not search_query: + continue + + search_results = _search_tracks_for_source(source, client, search_query, limit=5) + if not search_results and search_query != title: + search_results = _search_tracks_for_source(source, client, title, limit=5) + if not search_results and artist and search_query != artist: + search_results = _search_tracks_for_source(source, client, artist, limit=5) + + if not search_results: + continue + + best_match = _pick_best_track_match(search_results, title or search_query, artist) + if not best_match: + continue + + resolved_track_id = str(_extract_lookup_value(best_match, 'id', 'track_id', 'trackId', default='') or '') + resolved_data = best_match + if resolved_track_id and hasattr(client, 'get_track_details'): + try: + detailed = client.get_track_details(resolved_track_id) + if detailed: + resolved_data = detailed + except Exception as exc: + logger.debug("Track detail lookup failed on %s for %s: %s", source, resolved_track_id, exc) + + payload = _build_single_import_context_payload( + resolved_data, + source, + source_priority, + requested_title=title, + requested_artist=artist, + ) + if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): + try: + artist_details = client.get_artist(payload['context']['artist']['id']) + if artist_details: + payload['context']['artist']['genres'] = _extract_lookup_value( + artist_details, + 'genres', + default=[], + ) or [] + except Exception: + pass + return payload + + return _build_single_import_fallback_context(title, artist, source_priority) + + def resolve_album_reference( album_id: str, preferred_source: Optional[str] = None, diff --git a/core/soulseek_client.py b/core/soulseek_client.py index 9b64ada6..e859fd4e 100644 --- a/core/soulseek_client.py +++ b/core/soulseek_client.py @@ -8,6 +8,7 @@ import time from pathlib import Path from utils.logging_config import get_logger from config.settings import config_manager +from core.import_filename import parse_filename_metadata logger = get_logger("soulseek_client") @@ -87,60 +88,17 @@ class TrackResult(SearchResult): def _parse_filename_metadata(self): """Extract artist, title, album from filename patterns""" - import re - import os - - # Get just the filename without extension and path - base_name = os.path.splitext(os.path.basename(self.filename))[0] - - # Common patterns for track naming - patterns = [ - r'^(\d+)\s*[-\.]\s*(.+?)\s*[-–]\s*(.+)$', # "01 - Artist - Title" or "01. Artist - Title" - r'^(.+?)\s*[-–]\s*(.+)$', # "Artist - Title" - r'^(\d+)\s*[-\.]\s*(.+)$', # "01 - Title" or "01. Title" - ] - - for pattern in patterns: - match = re.match(pattern, base_name) - if match: - groups = match.groups() - if len(groups) == 3: # Track number, artist, title - try: - self.track_number = int(groups[0]) - self.artist = self.artist or groups[1].strip() - self.title = self.title or groups[2].strip() - except ValueError: - # First group might not be a number - self.artist = self.artist or groups[0].strip() - self.title = self.title or f"{groups[1]} - {groups[2]}".strip() - elif len(groups) == 2: - if groups[0].isdigit(): # Track number and title - try: - self.track_number = int(groups[0]) - self.title = self.title or groups[1].strip() - except ValueError: - pass - else: # Artist and title - self.artist = self.artist or groups[0].strip() - self.title = self.title or groups[1].strip() - break - - # Fallback: use filename as title if nothing was extracted - if not self.title: - self.title = base_name - - # Try to extract album from directory path - if not self.album and '/' in self.filename: - path_parts = self.filename.split('/') - if len(path_parts) >= 2: - # Look for album-like directory names - for part in reversed(path_parts[:-1]): # Exclude filename - if part and not part.startswith('@'): # Skip system directories - # Clean up common patterns - cleaned = re.sub(r'^\d+\s*[-\.]\s*', '', part) # Remove leading numbers - if len(cleaned) > 3: # Must be substantial - self.album = cleaned - break + parsed = parse_filename_metadata(self.filename) + if not self.artist and parsed.get("artist"): + self.artist = parsed["artist"] + if not self.title and parsed.get("title"): + self.title = parsed["title"] + if not self.album and parsed.get("album"): + self.album = parsed["album"] + if self.track_number is None: + track_number = parsed.get("track_number") + if track_number is not None: + self.track_number = track_number @dataclass class AlbumResult: @@ -1829,4 +1787,4 @@ class SoulseekClient: def __del__(self): # No persistent session to clean up - pass \ No newline at end of file + pass diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index a0cdd66b..c1d9aa86 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -2955,9 +2955,6 @@ class WatchlistScanner: } track_data = { - 'spotify_track_id': track['id'], - 'spotify_album_id': album_data['id'], - 'spotify_artist_id': artist.spotify_artist_id, 'track_name': track['name'], 'artist_name': artist.artist_name, 'album_name': album_data.get('name', 'Unknown Album'), diff --git a/tests/test_downloads_pagination.py b/tests/test_downloads_pagination.py index af2c7b9a..f85dd96d 100644 --- a/tests/test_downloads_pagination.py +++ b/tests/test_downloads_pagination.py @@ -7,7 +7,6 @@ and `status` query params and includes a `total` count. """ import sys -import threading import types from unittest.mock import patch @@ -45,6 +44,7 @@ _install_flask_limiter_stub() from flask import Flask, Blueprint # noqa: E402 from api import downloads as downloads_mod # noqa: E402 +import core.import_runtime_state as runtime_state # noqa: E402 def _make_task(status="downloading", when=None): @@ -63,11 +63,10 @@ def _make_task(status="downloading", when=None): def _make_app_with_tasks(tasks_dict): """Create a minimal Flask app with the downloads blueprint mounted and - a fake `web_server` module exposing the given download_tasks dict.""" - fake_ws = types.ModuleType("web_server") - fake_ws.download_tasks = tasks_dict - fake_ws.tasks_lock = threading.RLock() - sys.modules["web_server"] = fake_ws + the shared import runtime state populated with the given download_tasks dict.""" + original_tasks = dict(runtime_state.download_tasks) + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(tasks_dict) # Bypass API key auth for tests. def _passthrough(f): @@ -83,6 +82,7 @@ def _make_app_with_tasks(tasks_dict): downloads_mod.register_routes(bp) app.register_blueprint(bp) + app._original_download_tasks = original_tasks return app @@ -96,8 +96,12 @@ def client(): for i in range(25) } app = _make_app_with_tasks(tasks) - with app.test_client() as c: - yield c + try: + with app.test_client() as c: + yield c + finally: + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(app._original_download_tasks) def test_default_limit_applied(client): diff --git a/tests/test_import_album.py b/tests/test_import_album.py new file mode 100644 index 00000000..81ab6fb6 --- /dev/null +++ b/tests/test_import_album.py @@ -0,0 +1,186 @@ +import core.import_album as import_album + + +class _FakeEngine: + def normalize_string(self, text): + return str(text or "").strip().lower() + + def similarity_score(self, left, right): + return 1.0 if left == right else 0.0 + + +def test_resolve_album_artist_context_uses_provider_genres(monkeypatch): + class _FakeClient: + def get_artist(self, artist_id): + assert artist_id == "artist-1" + return {"genres": ["rock", "indie"]} + + monkeypatch.setattr(import_album, "get_client_for_source", lambda source: _FakeClient() if source == "spotify" else None) + + context = import_album.resolve_album_artist_context( + { + "id": "album-1", + "name": "Album One", + "artist_id": "artist-1", + "artists": [{"name": "Artist One", "id": "artist-1"}], + }, + source="spotify", + ) + + assert context == { + "id": "artist-1", + "name": "Artist One", + "genres": ["rock", "indie"], + "source": "spotify", + } + + +def test_build_album_import_context_is_neutral(): + context = import_album.build_album_import_context( + { + "id": "album-1", + "name": "Album One", + "artist": "Artist One", + "artist_id": "artist-1", + "artists": [{"name": "Artist One", "id": "artist-1"}], + "release_date": "2024-01-01", + "total_tracks": 12, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + "source": "deezer", + }, + { + "id": "track-1", + "name": "Song One", + "track_number": 3, + "disc_number": 2, + "duration_ms": 180000, + "artists": [{"name": "Artist One"}], + "uri": "deezer:track:track-1", + "album": "Album One", + "source": "deezer", + }, + artist_context={ + "id": "artist-1", + "name": "Artist One", + "genres": ["rock"], + "source": "deezer", + }, + total_discs=2, + source="deezer", + ) + + assert context["artist"]["name"] == "Artist One" + assert context["artist"]["genres"] == ["rock"] + assert context["album"]["name"] == "Album One" + assert context["album"]["total_discs"] == 2 + assert context["track_info"]["name"] == "Song One" + assert context["track_info"]["track_number"] == 3 + assert context["original_search_result"]["clean_title"] == "Song One" + assert context["source"] == "deezer" + assert "spotify_artist" not in context + assert "spotify_album" not in context + + +def test_build_album_import_match_payload_uses_generic_track_keys(monkeypatch, tmp_path): + staging_root = tmp_path / "Staging" + staging_root.mkdir() + (staging_root / "Song One.flac").write_text("fake") + + monkeypatch.setattr(import_album, "get_staging_path", lambda: str(staging_root)) + monkeypatch.setattr(import_album, "_get_matching_engine", lambda: _FakeEngine()) + monkeypatch.setattr( + import_album, + "read_staging_file_metadata", + lambda file_path, filename=None: { + "title": "Song One", + "artist": "Artist One", + "albumartist": "Artist One", + "album": "Album One", + "track_number": 1, + "disc_number": 1, + }, + ) + monkeypatch.setattr( + import_album, + "get_artist_album_tracks", + lambda album_id, artist_name="", album_name="", source=None: { + "success": True, + "album": { + "id": album_id, + "name": "Album One", + "artist": "Artist One", + "artist_name": "Artist One", + "artist_id": "artist-1", + "artists": [{"name": "Artist One", "id": "artist-1"}], + "release_date": "2024-01-01", + "total_tracks": 1, + "total_discs": 1, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + "images": [{"url": "https://img.example/album.jpg"}], + "source": "spotify", + }, + "tracks": [ + { + "id": "track-1", + "name": "Song One", + "track_number": 1, + "disc_number": 1, + "duration_ms": 180000, + "artists": [{"name": "Artist One"}], + "uri": "spotify:track:track-1", + "album": { + "id": album_id, + "name": "Album One", + "artist": "Artist One", + }, + "source": "spotify", + } + ], + "source": "spotify", + "source_priority": ["spotify"], + "resolved_album_id": album_id, + }, + ) + + result = import_album.build_album_import_match_payload( + "album-1", + album_name="Album One", + album_artist="Artist One", + source="spotify", + ) + + assert result["success"] is True + assert result["album"]["artist"] == "Artist One" + assert result["source"] == "spotify" + assert result["matches"] == [ + { + "track": { + "id": "track-1", + "name": "Song One", + "track_number": 1, + "disc_number": 1, + "duration_ms": 180000, + "artists": [{"name": "Artist One"}], + "uri": "spotify:track:track-1", + "album": { + "id": "album-1", + "name": "Album One", + "artist": "Artist One", + }, + "source": "spotify", + }, + "staging_file": { + "filename": "Song One.flac", + "full_path": str(staging_root / "Song One.flac"), + "title": "Song One", + "artist": "Artist One", + "album": "Album One", + "albumartist": "Artist One", + "track_number": 1, + "disc_number": 1, + }, + "confidence": 1.0, + } + ] diff --git a/tests/test_import_context.py b/tests/test_import_context.py new file mode 100644 index 00000000..f91acf19 --- /dev/null +++ b/tests/test_import_context.py @@ -0,0 +1,202 @@ +import pytest + +from core.import_context import ( + build_import_album_info, + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_has_clean_metadata, + get_import_has_full_metadata, + get_import_source, + get_import_source_ids, + get_library_source_id_columns, + get_source_tag_names, + normalize_import_context, +) + + +def test_normalize_import_context_promotes_neutral_fields_without_legacy_aliases(): + context = { + "source": "deezer", + "spotify_artist": {"name": "Artist One", "id": "artist-1"}, + "spotify_album": { + "name": "Album One", + "id": "album-1", + "release_date": "2024-01-01", + "total_tracks": 12, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + }, + "track_info": { + "name": "Song One", + "id": "track-1", + "track_number": 3, + "disc_number": 2, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "spotify_clean_title": "Song One", + "spotify_clean_album": "Album One", + "spotify_clean_artist": "Artist One", + }, + "has_clean_spotify_data": True, + "has_full_spotify_metadata": False, + } + + normalized = normalize_import_context(context) + + assert normalized["artist"]["name"] == "Artist One" + assert normalized["album"]["name"] == "Album One" + assert "spotify_artist" not in normalized + assert "spotify_album" not in normalized + assert normalized["original_search_result"]["clean_title"] == "Song One" + assert normalized["original_search_result"]["clean_album"] == "Album One" + assert normalized["original_search_result"]["clean_artist"] == "Artist One" + assert "spotify_clean_title" not in normalized["original_search_result"] + assert "spotify_clean_album" not in normalized["original_search_result"] + assert "spotify_clean_artist" not in normalized["original_search_result"] + assert get_import_clean_title(normalized) == "Song One" + assert get_import_clean_album(normalized) == "Album One" + assert get_import_clean_artist(normalized) == "Artist One" + assert get_import_source(normalized) == "deezer" + assert get_import_has_clean_metadata(normalized) is True + assert get_import_has_full_metadata(normalized) is False + + +def test_neutral_import_context_helpers_work_without_legacy_aliases(): + context = { + "source": "deezer", + "artist": {"name": "Artist One", "id": "artist-1"}, + "album": { + "name": "Album One", + "id": "album-1", + "release_date": "2024-01-01", + "total_tracks": 12, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + }, + "track_info": { + "name": "Song One", + "id": "track-1", + "track_number": 3, + "disc_number": 2, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "title": "Song One", + "artist": "Artist One", + "album": "Album One", + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + }, + "has_clean_metadata": True, + "has_full_metadata": True, + } + + assert get_import_clean_title(context) == "Song One" + assert get_import_clean_album(context) == "Album One" + assert get_import_clean_artist(context) == "Artist One" + assert get_import_source(context) == "deezer" + + normalized = normalize_import_context(context) + assert normalized["artist"]["name"] == "Artist One" + assert normalized["album"]["name"] == "Album One" + assert normalized["original_search_result"]["clean_title"] == "Song One" + assert "spotify_artist" not in normalized + assert "spotify_album" not in normalized + assert get_import_has_clean_metadata(normalized) is True + assert get_import_has_full_metadata(normalized) is True + + +@pytest.mark.parametrize( + "source,expected_tags,expected_columns", + [ + ( + "spotify", + {"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"}, + {"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"}, + ), + ( + "itunes", + {"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"}, + {"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"}, + ), + ( + "deezer", + {"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None}, + {"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"}, + ), + ( + "hydrabase", + {"track": None, "artist": None, "album": None}, + {"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"}, + ), + ( + "discogs", + {"track": None, "artist": None, "album": None}, + {"artist": "discogs_id", "album": "discogs_id", "track": None}, + ), + ], +) +def test_source_tag_and_library_column_mappings(source, expected_tags, expected_columns): + assert get_source_tag_names(source) == expected_tags + assert get_library_source_id_columns(source) == expected_columns + + +def test_get_import_source_ids_prefers_nested_source_specific_ids(): + context = normalize_import_context( + { + "source": "deezer", + "artist": {"deezer_id": "deezer-artist-1"}, + "album": {"deezer_id": "deezer-album-1"}, + "track_info": {"deezer_id": "deezer-track-1"}, + "original_search_result": {"source_artist_id": "deezer-artist-1"}, + } + ) + + assert get_import_source_ids(context) == { + "track_id": "deezer-track-1", + "artist_id": "deezer-artist-1", + "album_id": "deezer-album-1", + } + + +def test_build_import_album_info_uses_normalized_album_context(): + context = normalize_import_context( + { + "source": "deezer", + "artist": {"name": "Artist One"}, + "album": { + "name": "Album One", + "image_url": "https://img.example/album.jpg", + "release_date": "2024-05-01", + "total_tracks": 8, + "album_type": "album", + }, + "track_info": { + "name": "Song One", + "track_number": 4, + "disc_number": 2, + "duration_ms": 240000, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "title": "Song One", + "album": "Album One", + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + }, + } + ) + + album_info = build_import_album_info(context) + + assert album_info["is_album"] is True + assert album_info["album_name"] == "Album One" + assert album_info["track_number"] == 4 + assert album_info["disc_number"] == 2 + assert album_info["clean_track_name"] == "Song One" + assert album_info["album_image_url"] == "https://img.example/album.jpg" + assert album_info["source"] == "deezer" diff --git a/tests/test_import_file_ops.py b/tests/test_import_file_ops.py new file mode 100644 index 00000000..e9d72901 --- /dev/null +++ b/tests/test_import_file_ops.py @@ -0,0 +1,92 @@ +import sys +import types + +from core.import_file_ops import ( + cleanup_empty_directories, + extract_track_number_from_filename, + safe_move_file, + read_staging_file_metadata, +) + + +def test_extract_track_number_from_filename_handles_common_patterns(): + assert extract_track_number_from_filename("01 - Song.mp3") == 1 + assert extract_track_number_from_filename("1-03 - Song.mp3") == 3 + assert extract_track_number_from_filename("Artist - Song.mp3") == 1 + + +def test_safe_move_file_replaces_existing_destination(tmp_path): + src = tmp_path / "source.flac" + dst_dir = tmp_path / "dest" + dst_dir.mkdir() + dst = dst_dir / "track.flac" + + src.write_text("new") + dst.write_text("old") + + safe_move_file(src, dst) + + assert not src.exists() + assert dst.read_text() == "new" + + +def test_cleanup_empty_directories_removes_nested_empty_paths(tmp_path): + download_root = tmp_path / "downloads" + nested_dir = download_root / "Artist" / "Album" + nested_dir.mkdir(parents=True) + moved_file_path = nested_dir / "track.flac" + + cleanup_empty_directories(str(download_root), str(moved_file_path)) + + assert not nested_dir.exists() + assert not (download_root / "Artist").exists() + assert download_root.exists() + + +def test_read_staging_file_metadata_reads_tags(monkeypatch, tmp_path): + file_path = tmp_path / "Song One.flac" + file_path.write_text("fake") + + class DummyTags: + def __init__(self): + self.values = { + "title": ["Song One"], + "artist": ["Artist One"], + "albumartist": ["Album Artist"], + "album": ["Album One"], + "tracknumber": ["03/12"], + "discnumber": ["2/3"], + } + + def get(self, key, default=None): + return self.values.get(key, default) + + fake_mutagen = types.ModuleType("mutagen") + fake_mutagen.File = lambda path, easy=True: DummyTags() + monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen) + + metadata = read_staging_file_metadata(str(file_path), file_path.name) + + assert metadata == { + "title": "Song One", + "artist": "Artist One", + "albumartist": "Album Artist", + "album": "Album One", + "track_number": 3, + "disc_number": 2, + } + + +def test_read_staging_file_metadata_falls_back_to_filename_track_number(monkeypatch, tmp_path): + file_path = tmp_path / "07 - Song Two.flac" + file_path.write_text("fake") + + fake_mutagen = types.ModuleType("mutagen") + fake_mutagen.File = lambda path, easy=True: None + monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen) + + metadata = read_staging_file_metadata(str(file_path), file_path.name) + + assert metadata["title"] == "07 - Song Two" + assert metadata["track_number"] == 7 + assert metadata["disc_number"] == 1 diff --git a/tests/test_import_filename.py b/tests/test_import_filename.py new file mode 100644 index 00000000..a5b32ff2 --- /dev/null +++ b/tests/test_import_filename.py @@ -0,0 +1,33 @@ +import pytest + +from core.import_filename import parse_filename_metadata + + +@pytest.mark.parametrize( + "filename,expected", + [ + ( + "01 - Artist One - Title One.mp3", + {"artist": "Artist One", "title": "Title One", "album": "", "track_number": 1}, + ), + ( + "Artist Two - Title Two.flac", + {"artist": "Artist Two", "title": "Title Two", "album": "", "track_number": None}, + ), + ( + r"Artist Three\Album Three\03 - Title Three.ogg", + {"artist": "", "title": "Title Three", "album": "Album Three", "track_number": 3}, + ), + ( + "Loose Song.wav", + {"artist": "", "title": "Loose Song", "album": "", "track_number": None}, + ), + ], +) +def test_parse_filename_metadata_handles_common_patterns(filename, expected): + parsed = parse_filename_metadata(filename) + + assert parsed["artist"] == expected["artist"] + assert parsed["title"] == expected["title"] + assert parsed["album"] == expected["album"] + assert parsed["track_number"] == expected["track_number"] diff --git a/tests/test_import_guards.py b/tests/test_import_guards.py new file mode 100644 index 00000000..a7b95372 --- /dev/null +++ b/tests/test_import_guards.py @@ -0,0 +1,47 @@ +from types import SimpleNamespace + +from core import import_guards as guards + + +class _FakeDB: + def __init__(self, quality_profile): + self._quality_profile = quality_profile + + def get_quality_profile(self): + return self._quality_profile + + +def test_check_flac_bit_depth_rejects_strict_mismatch(monkeypatch): + monkeypatch.setattr( + guards, + "MusicDatabase", + lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": False}}}), + ) + monkeypatch.setattr( + guards, + "_get_config_manager", + lambda: SimpleNamespace(get=lambda _key, default=None: False), + ) + + context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}} + + assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) == ( + "FLAC bit depth mismatch: file is 24-bit, preference is 16-bit" + ) + + +def test_check_flac_bit_depth_allows_fallback_when_enabled(monkeypatch): + monkeypatch.setattr( + guards, + "MusicDatabase", + lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": True}}}), + ) + monkeypatch.setattr( + guards, + "_get_config_manager", + lambda: SimpleNamespace(get=lambda _key, default=None: False), + ) + + context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}} + + assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) is None diff --git a/tests/test_import_paths.py b/tests/test_import_paths.py new file mode 100644 index 00000000..f9a39fab --- /dev/null +++ b/tests/test_import_paths.py @@ -0,0 +1,161 @@ +import core.import_paths as import_paths + + +class _Config: + def __init__(self, values): + self._values = values + + def get(self, key, default=None): + return self._values.get(key, default) + + +def test_sanitize_filename_replaces_illegal_characters(): + assert import_paths.sanitize_filename("AC/DC: Song?") == "AC_DC_ Song_" + assert import_paths.sanitize_filename("AUX.txt").startswith("_") + + +def test_build_simple_download_destination_uses_album_folder(monkeypatch, tmp_path): + config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")}) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + + destination, album_name, filename = import_paths.build_simple_download_destination( + { + "search_result": { + "filename": "Album Folder/source.flac", + "album": "Album Folder", + } + }, + str(tmp_path / "source.flac"), + ) + + assert destination == tmp_path / "Transfer" / "Album Folder" / "source.flac" + assert album_name == "Album Folder" + assert filename == "source.flac" + assert destination.parent.exists() + + +def test_build_simple_download_destination_falls_back_to_transfer_root(monkeypatch, tmp_path): + config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")}) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + + destination, album_name, filename = import_paths.build_simple_download_destination( + { + "search_result": { + "filename": "source.flac", + "album": "Unknown Album", + } + }, + str(tmp_path / "source.flac"), + ) + + assert destination == tmp_path / "Transfer" / "source.flac" + assert album_name == "" + assert filename == "source.flac" + assert destination.parent.exists() + + +def test_get_file_path_from_template_raw_handles_quality_and_disc_placeholders(monkeypatch): + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config({})) + + folder_path, filename = import_paths.get_file_path_from_template_raw( + "$artist/$album/$discnum - $title [$quality]", + { + "artist": "Artist One", + "album": "Album One", + "title": "Song One", + "quality": "FLAC 16bit", + "disc_number": 3, + }, + ) + + assert folder_path == "Artist One/Album One" + assert filename == "3 - Song One [FLAC 16bit]" + + +def test_resolve_album_group_upgrades_standard_to_deluxe(): + import_paths._album_name_cache.clear() + import_paths._album_editions.clear() + + artist_context = {"name": "Cache Artist"} + standard_album = {"album_name": "Cache Album"} + deluxe_album = {"album_name": "Cache Album (Deluxe Edition)"} + + assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album" + assert import_paths.resolve_album_group(artist_context, deluxe_album) == "Cache Album (Deluxe Edition)" + assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album (Deluxe Edition)" + + +def test_build_final_path_for_track_uses_template_and_disc_folder(monkeypatch, tmp_path): + config = _Config( + { + "soulseek.transfer_path": str(tmp_path / "Transfer"), + "file_organization.enabled": True, + "file_organization.templates": { + "album_path": "$albumartist/$albumartist - $album/$track - $title", + "single_path": "$artist/$artist - $title", + }, + "file_organization.collab_artist_mode": "first", + "file_organization.disc_label": "Disc", + } + ) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + + calls = [] + + def fake_get_album_tracks_for_source(source, album_id): + calls.append((source, album_id)) + return { + "items": [ + {"disc_number": 1}, + {"disc_number": 2}, + ] + } + + monkeypatch.setattr(import_paths, "_get_album_tracks_for_source", fake_get_album_tracks_for_source) + + context = { + "artist": {"name": "Artist One"}, + "album": { + "name": "Album One", + "id": "album-1", + "release_date": "2026-01-01", + "total_tracks": 12, + "album_type": "album", + "artists": [{"name": "Artist One"}], + }, + "track_info": { + "name": "Song One", + "id": "track-1", + "track_number": 4, + "disc_number": 1, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "title": "Song One", + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + "artists": [{"name": "Artist One"}], + }, + "source": "deezer", + "is_album_download": False, + } + + final_path, created = import_paths.build_final_path_for_track( + context, + {"name": "Artist One"}, + { + "is_album": True, + "album_name": "Album One", + "track_number": 4, + "disc_number": 1, + }, + ".flac", + ) + + assert created is True + assert calls == [("deezer", "album-1")] + assert final_path == str( + tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1" / "04 - Song One.flac" + ) + assert (tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1").is_dir() diff --git a/tests/test_import_pipeline.py b/tests/test_import_pipeline.py new file mode 100644 index 00000000..768a8788 --- /dev/null +++ b/tests/test_import_pipeline.py @@ -0,0 +1,128 @@ +import logging +import sys +import types + +import core.import_pipeline as import_pipeline +import core.import_paths as import_paths +import core.import_runtime_state as runtime_state + + +class _Config: + def __init__(self, transfer_path): + self.transfer_path = transfer_path + + def get(self, key, default=None): + if key == "soulseek.transfer_path": + return self.transfer_path + if key in {"post_processing.replaygain_enabled", "lossy_copy.enabled", "lossy_copy.delete_original", "import.replace_lower_quality"}: + return False + return default + + +class _FakeAcoustidVerifier: + def quick_check_available(self): + return False, "disabled" + + +class _ImmediateThread: + def __init__(self, target=None, daemon=None): + self._target = target + + def start(self): + if self._target: + self._target() + + +def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch): + transfer_root = tmp_path / "Transfer" + transfer_root.mkdir() + source_path = tmp_path / "source.flac" + source_path.write_bytes(b"audio") + + context_key = "ctx-1" + task_id = "task-1" + batch_id = "batch-1" + context = { + "search_result": { + "is_simple_download": True, + "filename": "Album Folder/source.flac", + "album": "Album Folder", + }, + "track_info": {}, + "original_search_result": {}, + "is_album_download": False, + "task_id": task_id, + "batch_id": batch_id, + } + + mark_calls = [] + completion_calls = [] + scan_calls = [] + activity_calls = [] + + original_matched_context = dict(runtime_state.matched_downloads_context) + original_download_tasks = dict(runtime_state.download_tasks) + original_download_batches = dict(runtime_state.download_batches) + original_processed_ids = set(runtime_state._processed_download_ids) + original_post_process_locks = dict(runtime_state._post_process_locks) + + runtime_state.matched_downloads_context.clear() + runtime_state.download_tasks.clear() + runtime_state.download_batches.clear() + runtime_state._processed_download_ids.clear() + runtime_state._post_process_locks.clear() + + runtime = types.SimpleNamespace( + automation_engine=None, + on_download_completed=lambda batch, task, success: completion_calls.append((batch, task, success)), + web_scan_manager=types.SimpleNamespace(request_scan=lambda reason: scan_calls.append(reason)), + repair_worker=None, + ) + + fake_acoustid = types.ModuleType("core.acoustid_verification") + fake_acoustid.AcoustIDVerification = _FakeAcoustidVerifier + fake_acoustid.VerificationResult = types.SimpleNamespace(FAIL="FAIL") + + monkeypatch.setitem(sys.modules, "core.acoustid_verification", fake_acoustid) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config(str(transfer_root))) + monkeypatch.setattr(import_pipeline, "add_activity_item", lambda *args, **kwargs: activity_calls.append((args, kwargs))) + monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "record_library_history_download", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "record_download_provenance", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "_mark_task_completed", lambda task, track_info: mark_calls.append((task, track_info))) + monkeypatch.setattr(import_pipeline.threading, "Thread", _ImmediateThread) + + runtime_state.matched_downloads_context[context_key] = context + runtime_state.download_tasks[task_id] = {"track_info": {}, "status": "running"} + + try: + import_pipeline.post_process_matched_download_with_verification( + context_key, + context, + str(source_path), + task_id, + batch_id, + runtime, + ) + + expected_path = transfer_root / "Album Folder" / "source.flac" + assert expected_path.exists() + assert not source_path.exists() + assert context["_simple_download_completed"] is True + assert context["_final_path"] == str(expected_path) + assert mark_calls == [(task_id, {})] + assert completion_calls == [(batch_id, task_id, True)] + assert context_key not in runtime_state.matched_downloads_context + assert scan_calls == ["Simple download completed"] + assert activity_calls + finally: + runtime_state.matched_downloads_context.clear() + runtime_state.matched_downloads_context.update(original_matched_context) + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(original_download_tasks) + runtime_state.download_batches.clear() + runtime_state.download_batches.update(original_download_batches) + runtime_state._processed_download_ids.clear() + runtime_state._processed_download_ids.update(original_processed_ids) + runtime_state._post_process_locks.clear() + runtime_state._post_process_locks.update(original_post_process_locks) diff --git a/tests/test_import_side_effects.py b/tests/test_import_side_effects.py new file mode 100644 index 00000000..510a9ab0 --- /dev/null +++ b/tests/test_import_side_effects.py @@ -0,0 +1,144 @@ +import sqlite3 +from types import SimpleNamespace + +from core import import_side_effects as side_effects + + +class _FakeDB: + def __init__(self, conn): + self._conn = conn + + def _get_connection(self): + return self._conn + + +def _make_soulsync_db(): + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute( + """ + CREATE TABLE artists ( + id TEXT PRIMARY KEY, + name TEXT, + genres TEXT, + thumb_url TEXT, + server_source TEXT, + created_at TEXT, + updated_at TEXT, + spotify_artist_id TEXT + ) + """ + ) + conn.execute( + """ + CREATE TABLE albums ( + id TEXT PRIMARY KEY, + artist_id TEXT, + title TEXT, + year INTEGER, + thumb_url TEXT, + genres TEXT, + track_count INTEGER, + duration INTEGER, + server_source TEXT, + created_at TEXT, + updated_at TEXT, + spotify_album_id TEXT + ) + """ + ) + conn.execute( + """ + CREATE TABLE tracks ( + id TEXT PRIMARY KEY, + album_id TEXT, + artist_id TEXT, + title TEXT, + track_number INTEGER, + duration INTEGER, + file_path TEXT, + bitrate INTEGER, + track_artist TEXT, + server_source TEXT, + created_at TEXT, + updated_at TEXT, + spotify_track_id TEXT + ) + """ + ) + return conn + + +def test_record_soulsync_library_entry_writes_artist_album_and_track(tmp_path, monkeypatch): + conn = _make_soulsync_db() + fake_db = _FakeDB(conn) + final_path = tmp_path / "track.flac" + final_path.write_bytes(b"audio") + + monkeypatch.setattr(side_effects, "get_database", lambda: fake_db) + monkeypatch.setattr( + side_effects, + "_get_config_manager", + lambda: SimpleNamespace(get_active_media_server=lambda: "soulsync"), + ) + + import core.genre_filter as genre_filter + + monkeypatch.setattr(genre_filter, "filter_genres", lambda genres, _cfg: [genre.upper() for genre in genres]) + + context = { + "source": "spotify", + "artist": {"id": "sp-artist", "name": "Artist One"}, + "album": { + "id": "sp-album", + "name": "Album One", + "release_date": "2024-02-03", + "total_tracks": 12, + "image_url": "https://img.example/album.jpg", + }, + "track_info": { + "id": "sp-track", + "name": "Song One", + "track_number": 7, + "duration_ms": 210000, + "artists": [{"name": "Guest Artist"}], + "_source": "spotify", + }, + "original_search_result": { + "title": "Song One", + "artists": [{"name": "Guest Artist"}], + "_source": "spotify", + }, + "_final_processed_path": str(final_path), + } + + artist_context = {"name": "Artist One", "genres": ["rock", "indie"]} + album_info = {"is_album": True, "album_name": "Album One", "track_number": 7} + + side_effects.record_soulsync_library_entry(context, artist_context, album_info) + + artist_row = conn.execute("SELECT * FROM artists").fetchone() + album_row = conn.execute("SELECT * FROM albums").fetchone() + track_row = conn.execute("SELECT * FROM tracks").fetchone() + + assert artist_row["name"] == "Artist One" + assert artist_row["server_source"] == "soulsync" + assert artist_row["spotify_artist_id"] == "sp-artist" + assert artist_row["genres"] == '["ROCK", "INDIE"]' + + assert album_row["title"] == "Album One" + assert album_row["server_source"] == "soulsync" + assert album_row["spotify_album_id"] == "sp-album" + assert album_row["year"] == 2024 + assert album_row["track_count"] == 12 + assert album_row["duration"] == 210000 + assert album_row["artist_id"] == artist_row["id"] + + assert track_row["title"] == "Song One" + assert track_row["server_source"] == "soulsync" + assert track_row["spotify_track_id"] == "sp-track" + assert track_row["track_number"] == 7 + assert track_row["duration"] == 210000 + assert track_row["track_artist"] == "Guest Artist" + assert track_row["album_id"] == album_row["id"] + assert track_row["file_path"] == str(final_path) diff --git a/tests/test_import_staging.py b/tests/test_import_staging.py new file mode 100644 index 00000000..1cd5e15b --- /dev/null +++ b/tests/test_import_staging.py @@ -0,0 +1,265 @@ +from types import SimpleNamespace + +import core.import_staging as import_staging + + +class FakeClient: + def __init__(self, results=None): + self.results = results or [] + self.calls = [] + + def search_albums(self, query, **kwargs): + self.calls.append((query, kwargs)) + return self.results + + def search_tracks(self, query, **kwargs): + self.calls.append((query, kwargs)) + return self.results + + +def _album_result(album_id, name, artist, release_date="2024-01-01", total_tracks=10, image_url="https://img.example/album.jpg", album_type="album"): + return SimpleNamespace( + id=album_id, + name=name, + artists=[artist], + release_date=release_date, + total_tracks=total_tracks, + image_url=image_url, + album_type=album_type, + ) + + +def test_search_import_albums_prefers_primary_source(monkeypatch): + deezer_client = FakeClient([ + _album_result("deezer-1", "Album One", "Artist One"), + ]) + spotify_client = FakeClient([ + _album_result("spotify-1", "Album One", "Artist One"), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_albums_for_source", + lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit), + ) + + results = import_staging.search_import_albums("Album One", limit=2) + + assert results == [ + { + "id": "deezer-1", + "name": "Album One", + "artist": "Artist One", + "release_date": "2024-01-01", + "total_tracks": 10, + "image_url": "https://img.example/album.jpg", + "album_type": "album", + "source": "deezer", + } + ] + assert deezer_client.calls == [("Album One", {"limit": 2})] + assert spotify_client.calls == [] + + +def test_search_import_albums_falls_back_when_primary_has_no_results(monkeypatch): + deezer_client = FakeClient([]) + spotify_client = FakeClient([ + _album_result("spotify-1", "Album Two", "Artist Two"), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_albums_for_source", + lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit), + ) + + results = import_staging.search_import_albums("Album Two", limit=2) + + assert results == [ + { + "id": "spotify-1", + "name": "Album Two", + "artist": "Artist Two", + "release_date": "2024-01-01", + "total_tracks": 10, + "image_url": "https://img.example/album.jpg", + "album_type": "album", + "source": "spotify", + } + ] + assert deezer_client.calls == [("Album Two", {"limit": 2})] + assert spotify_client.calls == [("Album Two", {"limit": 2, "allow_fallback": False})] + + +def test_search_import_tracks_prefers_primary_source(monkeypatch): + deezer_client = FakeClient([ + SimpleNamespace( + id="deezer-track-1", + name="Song One", + artists=[{"name": "Artist One"}], + album={"id": "deezer-album-1", "name": "Album One"}, + duration_ms=210000, + image_url="https://img.example/track.jpg", + track_number=7, + ), + ]) + spotify_client = FakeClient([ + SimpleNamespace( + id="spotify-track-1", + name="Song One", + artists=["Artist One"], + album="Album One", + duration_ms=210000, + image_url="https://img.example/track.jpg", + track_number=7, + ), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_tracks_for_source", + lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit), + ) + + results = import_staging.search_import_tracks("Song One", limit=2) + + assert results == [ + { + "id": "deezer-track-1", + "name": "Song One", + "artist": "Artist One", + "album": "Album One", + "album_id": "deezer-album-1", + "duration_ms": 210000, + "image_url": "https://img.example/track.jpg", + "track_number": 7, + "source": "deezer", + } + ] + assert deezer_client.calls == [("Song One", {"limit": 2})] + assert spotify_client.calls == [] + + +def test_search_import_tracks_falls_back_when_primary_has_no_results(monkeypatch): + deezer_client = FakeClient([]) + spotify_client = FakeClient([ + SimpleNamespace( + id="spotify-track-1", + name="Song Two", + artists=["Artist Two"], + album="Album Two", + duration_ms=180000, + image_url="", + track_number=3, + ), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_tracks_for_source", + lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit), + ) + + results = import_staging.search_import_tracks("Song Two", limit=2) + + assert results == [ + { + "id": "spotify-track-1", + "name": "Song Two", + "artist": "Artist Two", + "album": "Album Two", + "album_id": "", + "duration_ms": 180000, + "image_url": "", + "track_number": 3, + "source": "spotify", + } + ] + assert deezer_client.calls == [("Song Two", {"limit": 2})] + assert spotify_client.calls == [("Song Two", {"limit": 2, "allow_fallback": False})] + + +def test_build_import_suggestions_background_uses_collected_queries(monkeypatch, tmp_path): + staging_root = tmp_path / "Staging" + staging_root.mkdir() + + cache = import_staging.get_import_suggestions_cache() + original_cache = dict(cache) + cache["suggestions"] = [] + cache["building"] = False + cache["built"] = False + + monkeypatch.setattr(import_staging, "get_staging_path", lambda: str(staging_root)) + monkeypatch.setattr(import_staging, "_collect_import_suggestion_queries", lambda staging_path: ["Album One", "Folder Hint"]) + monkeypatch.setattr( + import_staging, + "search_import_albums", + lambda query, limit=2: [ + { + "id": query.lower().replace(" ", "-"), + "name": query, + "artist": f"{query} Artist", + "release_date": "2024-02-01", + "total_tracks": 12, + "image_url": "", + "album_type": "album", + } + ], + ) + + try: + import_staging._build_import_suggestions_background() + + assert cache["built"] is True + assert cache["building"] is False + assert cache["suggestions"] == [ + { + "id": "album-one", + "name": "Album One", + "artist": "Album One Artist", + "release_date": "2024-02-01", + "total_tracks": 12, + "image_url": "", + "album_type": "album", + }, + { + "id": "folder-hint", + "name": "Folder Hint", + "artist": "Folder Hint Artist", + "release_date": "2024-02-01", + "total_tracks": 12, + "image_url": "", + "album_type": "album", + }, + ] + finally: + cache.clear() + cache.update(original_cache) diff --git a/tests/test_metadata_enrichment.py b/tests/test_metadata_enrichment.py new file mode 100644 index 00000000..49cbe630 --- /dev/null +++ b/tests/test_metadata_enrichment.py @@ -0,0 +1,333 @@ +import logging +import types + +import pytest + +from core import metadata_enrichment as me + + +class _Config: + def __init__(self, values=None): + self.values = values or {} + + def get(self, key, default=None): + return self.values.get(key, default) + + +class _FakeTag: + def __init__(self, kind, **kwargs): + self.kind = kind + self.kwargs = kwargs + + +class _FakeID3Tags: + def __init__(self): + self.added = [] + + def add(self, frame): + self.added.append(frame) + + def clear(self): + self.added.clear() + + def getall(self, _key): + return [] + + def keys(self): + return [frame.kind for frame in self.added] + + def __len__(self): + return len(self.added) + + +class _FakeAudio: + def __init__(self): + self.tags = _FakeID3Tags() + self.save_calls = [] + self.clear_pictures_calls = 0 + + def clear_pictures(self): + self.clear_pictures_calls += 1 + + def add_tags(self): + self.tags = _FakeID3Tags() + + def save(self, **kwargs): + self.save_calls.append(kwargs) + + +class _FakeResponse: + def __init__(self, payload, content_type="image/jpeg"): + self._payload = payload + self._content_type = content_type + + def read(self): + return self._payload + + def info(self): + return types.SimpleNamespace(get_content_type=lambda: self._content_type) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + +def _fake_symbols(audio): + def _tag_factory(kind): + return lambda **kwargs: _FakeTag(kind, **kwargs) + + return types.SimpleNamespace( + File=lambda _path: audio, + APEv2=type("FakeAPEv2", (), {}), + APENoHeaderError=Exception, + FLAC=type("FakeFLAC", (), {}), + Picture=type("FakePicture", (), {"__init__": lambda self: None}), + ID3=_FakeID3Tags, + APIC=_tag_factory("APIC"), + TBPM=_tag_factory("TBPM"), + TCOP=_tag_factory("TCOP"), + TDOR=_tag_factory("TDOR"), + TDRC=_tag_factory("TDRC"), + TCON=_tag_factory("TCON"), + TIT2=_tag_factory("TIT2"), + TALB=_tag_factory("TALB"), + TPE1=_tag_factory("TPE1"), + TPE2=_tag_factory("TPE2"), + TPOS=_tag_factory("TPOS"), + TPUB=_tag_factory("TPUB"), + TRCK=_tag_factory("TRCK"), + TSRC=_tag_factory("TSRC"), + TXXX=_tag_factory("TXXX"), + UFID=_tag_factory("UFID"), + TMED=_tag_factory("TMED"), + MP4=type("FakeMP4", (), {}), + MP4Cover=types.SimpleNamespace(FORMAT_JPEG=1, FORMAT_PNG=2, __call__=lambda *args, **kwargs: ("cover", args, kwargs)), + MP4FreeForm=lambda data: ("freeform", data), + OggVorbis=type("FakeOggVorbis", (), {}), + OggOpus=None, + ) + + +def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_for_non_itunes_sources(): + class _ItunesClient: + def __init__(self): + self.called = False + + def resolve_primary_artist(self, artist_id): + self.called = True + raise AssertionError("itunes fallback should not run for non-itunes sources") + + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config({"file_organization.collab_artist_mode": "first"}), + itunes_enrichment_worker=types.SimpleNamespace(client=_ItunesClient()), + ) + + context = { + "source": "spotify", + "artist": {"name": "Artist One & Artist Two", "id": "123", "genres": ["rock", "indie"]}, + "album": { + "name": "Album One", + "total_tracks": 12, + "release_date": "2024-01-02", + "images": [{"url": "https://img.example/album.jpg"}], + }, + "track_info": { + "artists": [{"name": "Artist One"}], + "_source": "spotify", + "track_number": 3, + "disc_number": 2, + "total_tracks": 12, + }, + "original_search_result": { + "title": "Song One", + "artists": [{"name": "Artist One"}], + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + "disc_number": 2, + "duration_ms": 180000, + }, + } + + metadata = me.extract_source_metadata( + context, + context["artist"], + {"is_album": True, "album_name": "Album One", "track_number": 3, "disc_number": 2, "album_image_url": "https://img.example/album.jpg"}, + runtime=runtime, + ) + + assert metadata["source"] == "spotify" + assert metadata["title"] == "Song One" + assert metadata["artist"] == "Artist One" + assert metadata["album_artist"] == "Artist One & Artist Two" + assert metadata["album"] == "Album One" + assert metadata["track_number"] == 3 + assert metadata["total_tracks"] == 12 + assert metadata["disc_number"] == 2 + assert metadata["album_art_url"] == "https://img.example/album.jpg" + assert runtime.itunes_enrichment_worker.client.called is False + + +def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatch): + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config(), + mb_worker=None, + deezer_worker=None, + audiodb_worker=None, + tidal_client=None, + qobuz_enrichment_worker=None, + lastfm_worker=None, + genius_worker=None, + itunes_enrichment_worker=None, + get_database=lambda: None, + ) + + audio = _FakeAudio() + symbols = _fake_symbols(audio) + monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + + current_metadata = { + "source": "deezer", + "source_track_id": "dz-track", + "source_artist_id": "dz-artist", + "source_album_id": "dz-album", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + } + me.embed_source_ids(audio, current_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime) + + current_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"] + assert "DEEZER_TRACK_ID" in current_descs + assert "DEEZER_ARTIST_ID" in current_descs + + audio = _FakeAudio() + symbols = _fake_symbols(audio) + monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + + legacy_metadata = { + "source": "", + "spotify_track_id": "sp-track", + "spotify_artist_id": "sp-artist", + "spotify_album_id": "sp-album", + "itunes_track_id": "it-track", + "itunes_artist_id": "it-artist", + "itunes_album_id": "it-album", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + } + me.embed_source_ids(audio, legacy_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime) + + legacy_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"] + assert "SPOTIFY_TRACK_ID" in legacy_descs + assert "SPOTIFY_ARTIST_ID" in legacy_descs + assert "SPOTIFY_ALBUM_ID" in legacy_descs + assert "ITUNES_TRACK_ID" in legacy_descs + assert "ITUNES_ARTIST_ID" in legacy_descs + assert "ITUNES_ALBUM_ID" in legacy_descs + + +def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch): + audio = _FakeAudio() + symbols = _fake_symbols(audio) + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config( + { + "metadata_enhancement.enabled": True, + "metadata_enhancement.embed_album_art": False, + "metadata_enhancement.tags.write_multi_artist": False, + } + ), + mb_worker=None, + deezer_worker=None, + audiodb_worker=None, + tidal_client=None, + qobuz_enrichment_worker=None, + lastfm_worker=None, + genius_worker=None, + itunes_enrichment_worker=None, + get_database=lambda: None, + ) + + strip_calls = [] + verify_calls = [] + + monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + monkeypatch.setattr(me, "_strip_all_non_audio_tags", lambda file_path, runtime=None: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0}) + monkeypatch.setattr( + me, + "extract_source_metadata", + lambda context, artist, album_info, runtime=None: { + "source": "deezer", + "source_track_id": "dz-track", + "source_artist_id": "dz-artist", + "source_album_id": "dz-album", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + "track_number": 3, + "total_tracks": 12, + "disc_number": 2, + "date": "2024", + "genre": "Rock", + "musicbrainz_release_id": "mb-release-1", + }, + ) + monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None) + monkeypatch.setattr(me, "_verify_metadata_written", lambda file_path, runtime=None: verify_calls.append(file_path) or True) + + album_info = {} + result = me.enhance_file_metadata( + "song.flac", + {"_audio_quality": ""}, + {"name": "Artist One"}, + album_info, + runtime=runtime, + ) + + assert result is True + assert strip_calls == ["song.flac"] + assert verify_calls == ["song.flac"] + assert audio.clear_pictures_calls == 1 + assert len(audio.save_calls) == 2 + assert album_info["musicbrainz_release_id"] == "mb-release-1" + assert any(frame.kind == "TIT2" for frame in audio.tags.added) + assert any(frame.kind == "TPE1" for frame in audio.tags.added) + assert any(frame.kind == "TXXX" and frame.kwargs.get("desc") == "DEEZER_TRACK_ID" for frame in audio.tags.added) + + +def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch): + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config( + { + "metadata_enhancement.cover_art_download": True, + "metadata_enhancement.prefer_caa_art": False, + } + ), + ) + + monkeypatch.setattr(me.urllib.request, "urlopen", lambda *args, **kwargs: _FakeResponse(b"cover-bytes")) + + target_dir = tmp_path / "Album One" + target_dir.mkdir() + + me.download_cover_art( + {}, + str(target_dir), + {"album": {"image_url": "https://img.example/album.jpg"}}, + runtime=runtime, + ) + + cover_path = target_dir / "cover.jpg" + assert cover_path.exists() + assert cover_path.read_bytes() == b"cover-bytes" diff --git a/tests/test_metadata_service_single_import_context.py b/tests/test_metadata_service_single_import_context.py new file mode 100644 index 00000000..ed7ae567 --- /dev/null +++ b/tests/test_metadata_service_single_import_context.py @@ -0,0 +1,193 @@ +from types import SimpleNamespace + +from core import metadata_service + + +class FakeClient: + def __init__(self, search_results=None, details=None, artist_details=None): + self.search_results = search_results or [] + self.details = details or {} + self.artist_details = artist_details or {} + self.calls = [] + + def search_tracks(self, query, limit=1, allow_fallback=True): + self.calls.append(("search_tracks", query, limit, allow_fallback)) + return self.search_results + + def get_track_details(self, track_id): + self.calls.append(("get_track_details", track_id)) + return self.details.get(str(track_id)) + + def get_artist(self, artist_id): + self.calls.append(("get_artist", artist_id)) + return self.artist_details.get(str(artist_id)) + + +def _track_result(track_id="track-1", name="Song One", artist="Artist One"): + return SimpleNamespace( + id=track_id, + name=name, + artists=[artist], + album="Album One", + duration_ms=123000, + track_number=1, + disc_number=1, + image_url="https://img.example/track.jpg", + ) + + +def _track_details(source, track_id="track-1", name="Song One", artist_name="Artist One", artist_id="artist-1"): + return { + "id": track_id, + "name": name, + "track_number": 7, + "disc_number": 1, + "duration_ms": 210000, + "explicit": True, + "uri": f"{source}:track:{track_id}", + "artists": [{"name": artist_name, "id": artist_id}], + "album": { + "id": f"{source}-album-1", + "name": "Album One", + "release_date": "2024-01-01", + "album_type": "album", + "total_tracks": 10, + "images": [{"url": f"https://img.example/{source}-album.jpg"}], + "artists": [{"name": artist_name, "id": artist_id}], + }, + } + + +def test_get_single_track_import_context_uses_primary_source_priority(monkeypatch): + deezer_client = FakeClient( + search_results=[_track_result(track_id="deezer-track-1")], + details={"deezer-track-1": _track_details("deezer", track_id="deezer-track-1", artist_name="Artist One", artist_id="deezer-artist-1")}, + artist_details={"deezer-artist-1": {"id": "deezer-artist-1", "genres": ["electronic"]}}, + ) + spotify_client = FakeClient() + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), + ) + + result = metadata_service.get_single_track_import_context("Song One", "Artist One") + + assert result["success"] is True + assert result["source"] == "deezer" + assert result["source_priority"] == ["deezer", "spotify", "itunes"] + assert result["context"]["track_info"]["name"] == "Song One" + assert result["context"]["track_info"]["album_id"] == "deezer-album-1" + assert result["context"]["album"]["image_url"] == "https://img.example/deezer-album.jpg" + assert result["context"]["artist"]["genres"] == ["electronic"] + assert result["context"]["original_search_result"]["clean_title"] == "Song One" + assert deezer_client.calls == [ + ('search_tracks', 'artist:"Artist One" track:"Song One"', 5, True), + ("get_track_details", "deezer-track-1"), + ("get_artist", "deezer-artist-1"), + ] + assert spotify_client.calls == [] + + +def test_get_single_track_import_context_falls_back_to_next_source(monkeypatch): + deezer_client = FakeClient(search_results=[]) + spotify_client = FakeClient( + search_results=[_track_result(track_id="spotify-track-1")], + details={"spotify-track-1": _track_details("spotify", track_id="spotify-track-1", artist_name="Artist Two", artist_id="spotify-artist-1")}, + artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["indie"]}}, + ) + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), + ) + + result = metadata_service.get_single_track_import_context("Song Two", "Artist Two") + + assert result["success"] is True + assert result["source"] == "spotify" + assert result["context"]["track_info"]["id"] == "spotify-track-1" + assert result["context"]["album"]["name"] == "Album One" + assert deezer_client.calls == [ + ('search_tracks', 'artist:"Artist Two" track:"Song Two"', 5, True), + ('search_tracks', 'Song Two', 5, True), + ('search_tracks', 'Artist Two', 5, True), + ] + assert spotify_client.calls == [ + ("search_tracks", "Song Two Artist Two", 5, False), + ("get_track_details", "spotify-track-1"), + ("get_artist", "spotify-artist-1"), + ] + + +def test_get_single_track_import_context_uses_explicit_override_first(monkeypatch): + spotify_client = FakeClient( + details={"override-track-1": _track_details("spotify", track_id="override-track-1", artist_name="Override Artist", artist_id="spotify-artist-1")}, + artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["pop"]}}, + ) + deezer_client = FakeClient() + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), + ) + + result = metadata_service.get_single_track_import_context( + "Ignored Title", + "Ignored Artist", + override_id="override-track-1", + ) + + assert result["success"] is True + assert result["source"] == "spotify" + assert result["context"]["track_info"]["id"] == "override-track-1" + assert result["context"]["artist"]["genres"] == ["pop"] + assert spotify_client.calls == [ + ("get_track_details", "override-track-1"), + ("get_artist", "spotify-artist-1"), + ] + assert deezer_client.calls == [] + + +def test_get_single_track_import_context_uses_explicit_override_source(monkeypatch): + itunes_client = FakeClient( + details={"override-track-2": _track_details("itunes", track_id="override-track-2", artist_name="Override Artist Two", artist_id="itunes-artist-1")}, + artist_details={"itunes-artist-1": {"id": "itunes-artist-1", "genres": ["singer-songwriter"]}}, + ) + spotify_client = FakeClient() + deezer_client = FakeClient() + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": itunes_client}.get(source), + ) + + result = metadata_service.get_single_track_import_context( + "Ignored Title", + "Ignored Artist", + override_id="override-track-2", + override_source="itunes", + ) + + assert result["success"] is True + assert result["source"] == "itunes" + assert result["context"]["track_info"]["id"] == "override-track-2" + assert result["context"]["artist"]["genres"] == ["singer-songwriter"] + assert itunes_client.calls == [ + ("get_track_details", "override-track-2"), + ("get_artist", "itunes-artist-1"), + ] + assert deezer_client.calls == [] + assert spotify_client.calls == [] diff --git a/tests/test_watchlist_scanner_scan.py b/tests/test_watchlist_scanner_scan.py index dcddd3df..6c08efa0 100644 --- a/tests/test_watchlist_scanner_scan.py +++ b/tests/test_watchlist_scanner_scan.py @@ -2,10 +2,8 @@ import sys import types from datetime import datetime, timedelta - _RECENT_RELEASE_DATE = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d") - if "spotipy" not in sys.modules: spotipy = types.ModuleType("spotipy") @@ -1045,6 +1043,10 @@ def test_update_discovery_pool_incremental_uses_source_priority(monkeypatch): assert scanner.database.discovery_pool_calls assert scanner.database.discovery_pool_calls[0][1] == "deezer" + assert scanner.database.discovery_pool_calls[0][0]["deezer_track_id"] == "dz-track-1" + assert scanner.database.discovery_pool_calls[0][0]["deezer_album_id"] == "dz-release-1" + assert scanner.database.discovery_pool_calls[0][0]["deezer_artist_id"] == "dz-artist" + assert scanner.database.discovery_pool_calls[0][0].get("spotify_track_id") is None assert deezer_client.search_calls == [("Incremental Artist", 1, {})] assert deezer_client.album_calls assert spotify_client.search_calls == [] diff --git a/web_server.py b/web_server.py index 8d2a06eb..504ba7b5 100644 --- a/web_server.py +++ b/web_server.py @@ -26,6 +26,9 @@ from flask import Flask, render_template, request, jsonify, redirect, send_file, from flask_socketio import SocketIO, emit, join_room, leave_room from utils.logging_config import get_logger, setup_logging from utils.async_helpers import run_async +from mutagen.flac import FLAC +from mutagen.mp4 import MP4 +from mutagen.oggvorbis import OggVorbis # --- Core Application Imports --- # Import the same core clients and config manager used by the GUI app @@ -94,6 +97,56 @@ from core.database_update_worker import DatabaseUpdateWorker from core.web_scan_manager import WebScanManager from core.lyrics_client import lyrics_client from core.metadata_cache import get_metadata_cache +from core.import_context import ( + build_import_album_info, + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_context_album, + get_import_context_artist, + get_import_has_clean_metadata, + get_import_has_full_metadata, + get_import_original_search, + get_import_source, + get_import_source_ids, + get_import_track_info, + get_library_source_id_columns, + get_source_tag_names, + normalize_import_context, +) +from core.import_album import ( + build_album_import_context, + build_album_import_match_payload, + resolve_album_artist_context, +) +from core.import_album_naming import resolve_album_group as _resolve_album_group +from core.import_filename import extract_track_number_from_filename, parse_filename_metadata +from core.import_staging import ( + get_import_suggestions_cache, + get_primary_source, + get_staging_path, + read_staging_file_metadata, + refresh_import_suggestions_cache, + search_import_albums, + search_import_tracks, + start_import_suggestions_cache, +) +from core.import_paths import build_final_path_for_track as _build_final_path_for_track +from core.metadata_common import get_file_lock +from core.import_runtime_state import ( + activity_feed, + activity_feed_lock, + add_activity_item, + download_batches, + download_tasks, + matched_context_lock, + matched_downloads_context, + mark_task_completed as _core_mark_task_completed, + set_activity_toast_emitter, + tasks_lock, +) +from core import metadata_enrichment +from core.metadata_source import normalize_album_cache_key from database.music_database import get_database, MusicDatabase from services.sync_service import PlaylistSyncService @@ -225,6 +278,7 @@ _socketio_cors_origins = _resolve_socketio_cors_origins(config_manager) socketio = SocketIO(app, async_mode='threading', cors_allowed_origins=_socketio_cors_origins) _log_socketio_startup_status(_socketio_cors_origins, logger) _socketio_rejection_logger = _SocketIORejectionLogger(logger) +set_activity_toast_emitter(socketio.emit) # Plex PIN auth requests stored in memory for polling _plex_pin_requests = {} @@ -1883,7 +1937,7 @@ def _register_automation_handlers(): # --- 4. Sweep empty staging directories --- _update_automation_progress(automation_id, phase='Sweeping import folder...', progress=60) - staging_path = _get_staging_path() + staging_path = get_staging_path() s_removed = 0 if os.path.isdir(staging_path): for dirpath, _dirnames, _filenames in os.walk(staging_path, topdown=False): @@ -2162,346 +2216,6 @@ def _register_automation_handlers(): logger.info("Automation action handlers registered") - -def _emit_track_downloaded(context): - """Emit track_downloaded event for automation engine. Safe to call anywhere.""" - try: - if not automation_engine: - return - ti = context.get('track_info') or context.get('search_result') or {} - artist_name = '' - artists = ti.get('artists', []) - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - automation_engine.emit('track_downloaded', { - 'artist': artist_name, - 'title': ti.get('name', ti.get('title', '')), - 'album': ti.get('album', ''), - 'quality': context.get('_audio_quality', 'Unknown'), - }) - except Exception: - pass - - -def _record_library_history_download(context): - """Record a completed download to the library_history table. Non-blocking.""" - try: - # Determine download source - search_result = context.get('original_search_result') or context.get('search_result') or {} - username = search_result.get('username', context.get('_download_username', '')) - _svc_map = {'youtube': 'YouTube', 'tidal': 'Tidal', 'qobuz': 'Qobuz', 'hifi': 'HiFi', 'deezer_dl': 'Deezer', 'lidarr': 'Lidarr'} - download_source = _svc_map.get(username, 'Soulseek') - - ti = context.get('track_info') or context.get('search_result') or {} - artist_name = '' - artists = ti.get('artists', []) - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - if not artist_name: - artist_name = ti.get('artist', '') - - album_raw = ti.get('album', '') - album_name = album_raw.get('name', '') if isinstance(album_raw, dict) else str(album_raw or '') - - title = ti.get('name', ti.get('title', '')) - quality = context.get('_audio_quality', '') - file_path = context.get('_final_processed_path', context.get('_final_path', '')) - - # Try to get album art URL - thumb_url = '' - spotify_album = context.get('spotify_album') - if spotify_album and isinstance(spotify_album, dict): - thumb_url = spotify_album.get('image_url', '') - if not thumb_url: - images = spotify_album.get('images', []) - if images: - thumb_url = images[0].get('url', '') - if not thumb_url: - album_info = context.get('album_info', {}) - if isinstance(album_info, dict): - thumb_url = album_info.get('album_image_url', '') - - # Source provenance — what file/track was actually downloaded - source_filename = search_result.get('filename', '') - # Track ID: try search result first, then track_info (Spotify ID used for streaming lookups) - source_track_id = (search_result.get('track_id', '') - or search_result.get('id', '') - or ti.get('id', '')) - - # Source title/artist — what the download source said the track was. - # For Soulseek: parsed from the peer's filename by TrackResult._parse_filename_metadata() - # For Tidal/YouTube/Qobuz: from the streaming API's own metadata - # These live on the candidate's original fields, NOT the spotify_clean_* fields - source_track_title = search_result.get('title', '') or search_result.get('name', '') - source_artist = search_result.get('artist', '') - # For streaming sources, track ID is encoded in filename as "id||display_name" - if source_filename and '||' in source_filename and username in ('tidal', 'youtube', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): - _stream_id = source_filename.split('||')[0] - if _stream_id and not source_track_id: - source_track_id = _stream_id - - # AcoustID verification result - acoustid_result = context.get('_acoustid_result', '') - - db = get_database() - db.add_library_history_entry( - event_type='download', - title=title, - artist_name=artist_name, - album_name=album_name, - quality=quality, - file_path=file_path, - thumb_url=thumb_url, - download_source=download_source, - source_track_id=source_track_id, - source_track_title=source_track_title, - source_filename=source_filename, - acoustid_result=acoustid_result, - source_artist=source_artist - ) - except Exception: - pass # Non-critical, never block download flow - - -def _record_download_provenance(context): - """Record download source provenance for track lineage tracking. Non-blocking.""" - try: - # Extract source info - search_result = context.get('original_search_result') or context.get('search_result') or {} - username = search_result.get('username', context.get('_download_username', '')) - filename = search_result.get('filename', '') - - # Determine source service from username - service_map = {'youtube': 'youtube', 'tidal': 'tidal', 'qobuz': 'qobuz', 'hifi': 'hifi', 'deezer_dl': 'deezer', 'lidarr': 'lidarr'} - source_service = service_map.get(username, 'soulseek') - - # Track metadata - ti = context.get('track_info') or context.get('search_result') or {} - artist_name = '' - artists = ti.get('artists', []) - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - if not artist_name: - artist_name = ti.get('artist', '') - - album_raw = ti.get('album', '') - album_name = album_raw.get('name', '') if isinstance(album_raw, dict) else str(album_raw or '') - title = ti.get('name', ti.get('title', '')) - - file_path = context.get('_final_processed_path', context.get('_final_path', '')) - quality = context.get('_audio_quality', '') - size = search_result.get('size', 0) - - # Read audio details from the file for provenance (survives transcoding) - bit_depth = None - sample_rate = None - bitrate = None - try: - if file_path and os.path.isfile(file_path): - from mutagen import File as MutagenFile - audio = MutagenFile(file_path) - if audio and audio.info: - sample_rate = getattr(audio.info, 'sample_rate', None) - bitrate = getattr(audio.info, 'bitrate', None) - bit_depth = getattr(audio.info, 'bits_per_sample', None) - except Exception: - pass - - db = get_database() - db.record_track_download( - file_path=file_path, - source_service=source_service, - source_username=username, - source_filename=filename, - source_size=size or 0, - audio_quality=quality, - track_title=title, - track_artist=artist_name, - track_album=album_name, - bit_depth=bit_depth, - sample_rate=sample_rate, - bitrate=bitrate, - ) - except Exception: - pass # Non-critical, never block download flow - - -def _record_soulsync_library_entry(context, spotify_artist, album_info): - """Write artist/album/track to library DB after successful download/import. - - Only runs when active server is 'soulsync' (standalone mode). Creates - DB records with server_source='soulsync' and pre-populates enrichment - IDs so enrichment workers don't need to re-discover them. - """ - try: - active_server = config_manager.get_active_media_server() - if active_server != 'soulsync': - return - - final_path = context.get('_final_processed_path') - if not final_path: - return - - spotify_album = context.get('spotify_album', {}) or {} - track_info = context.get('track_info', {}) or {} - original_search = context.get('original_search_result', {}) or {} - - artist_name = (spotify_artist or {}).get('name', '') - if not artist_name: - artist_name = original_search.get('spotify_clean_artist', '') or original_search.get('artist', '') - if not artist_name or artist_name in ('Unknown', 'Unknown Artist'): - return - - album_name = '' - if album_info and isinstance(album_info, dict): - album_name = album_info.get('album_name', '') - if not album_name: - album_name = spotify_album.get('name', '') or original_search.get('album', '') - if not album_name: - album_name = track_info.get('name', 'Unknown') - - track_name = original_search.get('spotify_clean_title', '') or track_info.get('name', '') or original_search.get('title', '') - track_number = (track_info.get('track_number') or (album_info.get('track_number') if isinstance(album_info, dict) else None)) or 1 - duration_ms = track_info.get('duration_ms', 0) or 0 - - year = None - release_date = spotify_album.get('release_date', '') - if release_date and len(release_date) >= 4: - try: - year = int(release_date[:4]) - except ValueError: - pass - - image_url = spotify_album.get('image_url', '') - if not image_url: - images = spotify_album.get('images', []) - if images and isinstance(images, list) and len(images) > 0: - img = images[0] - image_url = img.get('url', '') if isinstance(img, dict) else str(img) - - # Enrichment IDs from context — saves enrichment workers from re-discovering - spotify_artist_id = (spotify_artist or {}).get('id', '') - if spotify_artist_id in ('auto_import', 'from_sync_modal', 'explicit_artist', ''): - spotify_artist_id = '' - spotify_album_id = spotify_album.get('id', '') - if spotify_album_id in ('from_sync_modal', 'explicit_album', ''): - spotify_album_id = '' - spotify_track_id = track_info.get('id', '') or original_search.get('id', '') - - genres = (spotify_artist or {}).get('genres', []) - if genres: - from core.genre_filter import filter_genres as _gf2 - genres = _gf2(genres, config_manager) - genres_json = json.dumps(genres) if genres else '' - - bitrate = 0 - try: - from mutagen import File as MutagenFile - audio = MutagenFile(final_path) - if audio and hasattr(audio, 'info') and audio.info and hasattr(audio.info, 'bitrate'): - bitrate = int(audio.info.bitrate / 1000) if audio.info.bitrate else 0 - except Exception: - pass - - import hashlib - def _sid(text): - return str(abs(int(hashlib.md5(text.encode('utf-8', errors='replace')).hexdigest(), 16)) % (10 ** 9)) - - artist_id = _sid(artist_name.lower().strip()) - album_id = _sid(f"{artist_name}::{album_name}".lower().strip()) - track_id = _sid(final_path) - total_tracks = spotify_album.get('total_tracks', 0) or 0 - - db = get_database() - with db._get_connection() as conn: - cursor = conn.cursor() - - # ── Artist: find existing soulsync record or create ── - cursor.execute("SELECT id FROM artists WHERE id = ? AND server_source = 'soulsync'", (artist_id,)) - if not cursor.fetchone(): - # Check if soulsync artist exists by name - cursor.execute("SELECT id FROM artists WHERE name COLLATE NOCASE = ? AND server_source = 'soulsync' LIMIT 1", (artist_name,)) - existing_by_name = cursor.fetchone() - if existing_by_name: - artist_id = existing_by_name[0] - else: - # Avoid PK collision with other server sources - cursor.execute("SELECT id FROM artists WHERE id = ?", (artist_id,)) - if cursor.fetchone(): - # ID taken by another source — append suffix - artist_id = _sid(artist_name.lower().strip() + '::soulsync') - cursor.execute(""" - INSERT INTO artists (id, name, genres, thumb_url, server_source, created_at, updated_at) - VALUES (?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - """, (artist_id, artist_name, genres_json, image_url)) - if spotify_artist_id: - try: - cursor.execute("UPDATE artists SET spotify_artist_id = ? WHERE id = ?", (spotify_artist_id, artist_id)) - except Exception: - pass - - # ── Album: find existing soulsync record or create ── - cursor.execute("SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'", (album_id,)) - if not cursor.fetchone(): - cursor.execute("SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? AND server_source = 'soulsync' LIMIT 1", - (album_name, artist_id)) - existing_album_by_name = cursor.fetchone() - if existing_album_by_name: - album_id = existing_album_by_name[0] - else: - # Avoid PK collision - cursor.execute("SELECT id FROM albums WHERE id = ?", (album_id,)) - if cursor.fetchone(): - album_id = _sid(f"{artist_name}::{album_name}::soulsync".lower().strip()) - cursor.execute(""" - INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, track_count, - duration, server_source, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - """, (album_id, artist_id, album_name, year, image_url, genres_json, total_tracks, duration_ms)) - if spotify_album_id: - try: - cursor.execute("UPDATE albums SET spotify_album_id = ? WHERE id = ?", (spotify_album_id, album_id)) - except Exception: - pass - - # ── Track ── - # Determine per-track artist (for compilations/features where track artist != album artist) - track_artist = None - track_artists_list = track_info.get('artists', []) or original_search.get('artists', []) - if track_artists_list: - first_track_artist = track_artists_list[0] - if isinstance(first_track_artist, dict): - ta_name = first_track_artist.get('name', '') - else: - ta_name = str(first_track_artist) - if ta_name and ta_name.lower() != artist_name.lower(): - track_artist = ta_name # Only store when different from album artist - - cursor.execute("SELECT id FROM tracks WHERE file_path = ?", (final_path,)) - if not cursor.fetchone(): - cursor.execute(""" - INSERT INTO tracks (id, album_id, artist_id, title, track_number, - duration, file_path, bitrate, track_artist, server_source, - created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - """, (track_id, album_id, artist_id, track_name, track_number, - duration_ms, final_path, bitrate, track_artist)) - if spotify_track_id: - try: - cursor.execute("UPDATE tracks SET spotify_track_id = ? WHERE id = ?", (spotify_track_id, track_id)) - except Exception: - pass - - conn.commit() - logger.info(f"[SoulSync Library] Added: {artist_name} / {album_name} / {track_name}") - - except Exception as e: - logger.debug(f"[SoulSync Library] Non-critical error: {e}") - - # --- Register Public REST API Blueprint (v1) --- try: from api import create_api_blueprint, limiter @@ -2652,30 +2366,13 @@ def _update_automation_progress(automation_id, **kwargs): pass # --- Global Matched Downloads Context Management --- -# Thread-safe storage for matched download contexts -# Key: slskd download ID, Value: dict containing Spotify artist/album data -matched_downloads_context = {} -matched_context_lock = threading.Lock() +# Shared with core.import_runtime_state so the refactored pipeline and web +# server operate on the same context registry. _orphaned_download_keys = set() # Context keys of downloads abandoned during retry -# --- File-Level Metadata Write Locking --- -# Prevents concurrent threads from writing metadata to the same file simultaneously -_metadata_write_locks = {} # file_path -> threading.Lock() -_metadata_locks_lock = threading.Lock() # Lock for the locks dict - -def _get_file_lock(file_path): - """Get or create a lock for a specific file path to prevent concurrent metadata writes.""" - with _metadata_locks_lock: - if file_path not in _metadata_write_locks: - _metadata_write_locks[file_path] = threading.Lock() - return _metadata_write_locks[file_path] - # --- Download Missing Tracks Modal State Management --- # Thread-safe state tracking for modal download functionality with batch management missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker") -download_tasks = {} # task_id -> task state dict -download_batches = {} # batch_id -> {queue, active_count, max_concurrent} -tasks_lock = threading.Lock() batch_locks = {} # batch_id -> Lock() for atomic batch operations def _get_max_concurrent(): @@ -2707,8 +2404,7 @@ def _mark_task_completed(task_id, track_info=None): Assumes task_id exists in download_tasks (should be called within tasks_lock). """ global session_completed_downloads - - download_tasks[task_id]['status'] = 'completed' + _core_mark_task_completed(task_id, track_info) # Increment session counter (matches dashboard.py behavior) with session_stats_lock: @@ -4159,12 +3855,6 @@ _EDITION_BARE_RE = _re.compile( _re.IGNORECASE ) -def _normalize_album_cache_key(album_name): - """Normalize album name for cache key: strip edition suffixes, lowercase, strip whitespace.""" - result = _EDITION_PAREN_RE.sub('', album_name) - result = _EDITION_BARE_RE.sub('', result) - return result.lower().strip() - def _prepare_stream_task(track_data): """ Background streaming task that downloads track to Stream folder and updates global state. @@ -6225,10 +5915,6 @@ def get_debug_info(): return jsonify(info) -# Global activity tracking storage -activity_feed = [] -activity_feed_lock = threading.Lock() - @app.route('/api/activity/feed') def get_activity_feed(): """Get recent activity feed for dashboard""" @@ -6297,37 +5983,6 @@ def get_activity_logs(): except Exception as e: return jsonify({'logs': [f'Error reading activity feed: {str(e)}']}) -def add_activity_item(icon: str, title: str, subtitle: str, time_ago: str = "Now", show_toast: bool = True): - """Add activity item to the feed (replicates dashboard.py functionality)""" - try: - import time - from datetime import datetime, timezone - activity_item = { - 'icon': icon, - 'title': title, - 'subtitle': subtitle, - 'time': datetime.now(timezone.utc).isoformat(), - 'timestamp': time.time(), - 'show_toast': show_toast - } - - with activity_feed_lock: - activity_feed.append(activity_item) - # Keep only last 20 items to prevent memory growth - if len(activity_feed) > 20: - activity_feed.pop(0) - - # Instant toast push via WebSocket (replaces 3-second polling) - if show_toast: - try: - socketio.emit('dashboard:toast', activity_item) - except Exception: - pass - - logger.info(f"Activity: {icon} {title} - {subtitle}") - except Exception as e: - logger.error(f"Error adding activity item: {e}") - # --- Internal API Key Management (browser-only, no auth) --- @app.route('/api/v1/api-keys-internal', methods=['GET']) def list_api_keys_internal(): @@ -13238,7 +12893,7 @@ def write_track_tags(track_id): cover_url = thumb # Use file lock for thread safety - file_lock = _get_file_lock(resolved_path) + file_lock = get_file_lock(resolved_path) with file_lock: result = write_tags_to_file(resolved_path, db_data, embed_cover=embed_cover, cover_url=cover_url) @@ -13396,7 +13051,7 @@ def write_tracks_tags_batch(): if thumb and thumb.startswith('http'): art_data = cover_cache.get(thumb) - file_lock = _get_file_lock(resolved_path) + file_lock = get_file_lock(resolved_path) with file_lock: write_result = write_tags_to_file( resolved_path, db_data, @@ -13523,7 +13178,7 @@ def analyze_track_replaygain(track_id): track_gain_db = _RG_REFERENCE_LUFS - lufs - file_lock = _get_file_lock(file_path) + file_lock = get_file_lock(file_path) with file_lock: ok = _rg_write_tags(file_path, track_gain_db, peak_dbfs) @@ -13625,7 +13280,7 @@ def analyze_album_replaygain(album_id): continue file_path, track_gain_db, peak_dbfs = entry try: - file_lock = _get_file_lock(file_path) + file_lock = get_file_lock(file_path) with file_lock: _rg_write_tags(file_path, track_gain_db, peak_dbfs, album_gain_db, album_peak_dbfs) @@ -13711,7 +13366,7 @@ def analyze_tracks_replaygain_batch(): try: lufs, peak_dbfs = _rg_analyze_track(file_path) track_gain_db = _RG_REFERENCE_LUFS - lufs - file_lock = _get_file_lock(file_path) + file_lock = get_file_lock(file_path) with file_lock: _rg_write_tags(file_path, track_gain_db, peak_dbfs) with _rg_batch_lock: @@ -16562,7 +16217,7 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() with _mb_release_cache_lock: - _mb_release_cache[(_normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid + _mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid _mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid with _mb_release_detail_cache_lock: _mb_release_detail_cache[_pf_mbid] = _pf_release @@ -16703,7 +16358,7 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album): _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() with _mb_release_cache_lock: - _mb_release_cache[(_normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid + _mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid _mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid with _mb_release_detail_cache_lock: _mb_release_detail_cache[_pf_mbid] = _pf_release @@ -16776,9 +16431,6 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album): return started_count - - - @app.route('/api/download/matched', methods=['POST']) def start_matched_download(): """ @@ -16903,82 +16555,12 @@ def start_matched_download(): traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 - - - - - def _parse_filename_metadata(filename: str) -> dict: """ A direct port of the metadata parsing logic from the GUI's soulseek_client.py. This is the crucial missing step that cleans filenames BEFORE Spotify matching. """ - import re - import os - - metadata = { - 'artist': None, - 'title': None, - 'album': None, - 'track_number': None - } - - # Get just the filename without extension and path - base_name = os.path.splitext(os.path.basename(filename))[0] - - # --- Logic from soulseek_client.py --- - patterns = [ - # Pattern: 01 - Artist - Title (three-part with track number) - r'^(?P\d{1,2})\s*[-\.]\s*(?P.+?)\s*[-–]\s*(?P.+)$', - # Pattern: 01 - Title (track number + title — must come before Artist - Title - # to prevent "08 - Kilburn Market Dub" matching as artist="08") - r'^(?P<track_number>\d{1,2})\s*[-\.]\s*(?P<title>.+)$', - # Pattern: Artist - Title - r'^(?P<artist>.+?)\s*[-–]\s*(?P<title>.+)$', - ] - - for pattern in patterns: - match = re.match(pattern, base_name) - if match: - match_dict = match.groupdict() - metadata['track_number'] = int(match_dict['track_number']) if match_dict.get('track_number') else None - metadata['artist'] = match_dict.get('artist', '').strip() or None - metadata['title'] = match_dict.get('title', '').strip() or None - break # Stop after first successful match - - # If title is still missing, use the whole base_name - if not metadata['title']: - metadata['title'] = base_name.strip() - - # Fallback for underscore formats like 'Artist_Album_01_Title' - if not metadata['artist'] and '_' in base_name: - parts = base_name.split('_') - if len(parts) >= 3: - # A common pattern is Artist_Album_TrackNum_Title - if parts[-2].isdigit(): - metadata['artist'] = parts[0].strip() - metadata['title'] = parts[-1].strip() - metadata['track_number'] = int(parts[-2]) - metadata['album'] = parts[1].strip() - - # Final cleanup on title if it contains the artist - if metadata['artist'] and metadata['title'] and metadata['artist'].lower() in metadata['title'].lower(): - metadata['title'] = metadata['title'].replace(metadata['artist'], '').lstrip(' -–_').strip() - - - # Try to extract album from the full directory path - if '/' in filename or '\\' in filename: - path_parts = filename.replace('\\', '/').split('/') - if len(path_parts) >= 2: - # The parent directory is often the album - potential_album = path_parts[-2] - # Clean common prefixes like '2024 - ' - cleaned_album = re.sub(r'^\d{4}\s*-\s*', '', potential_album).strip() - metadata['album'] = cleaned_album - - logger.info(f"Parsed Filename '{base_name}': Artist='{metadata['artist']}', Title='{metadata['title']}', Album='{metadata['album']}', Track#='{metadata['track_number']}'") - return metadata - + return parse_filename_metadata(filename) def _read_staging_file_metadata(full_path: str, filename: str) -> dict: """Read metadata from a staging file — tags first, filename parsing as fallback. @@ -16986,57 +16568,7 @@ def _read_staging_file_metadata(full_path: str, filename: str) -> dict: Returns dict with: title, artist, albumartist, album, track_number, disc_number. Only falls back to filename parsing when BOTH title AND artist tags are empty. """ - meta = { - 'title': None, 'artist': None, 'albumartist': None, - 'album': None, 'track_number': None, 'disc_number': None, - } - - # Phase 1: Read embedded tags (most reliable) - try: - from mutagen import File as MutagenFile - tags = MutagenFile(full_path, easy=True) - if tags: - def _first(tag_list): - if isinstance(tag_list, list) and tag_list: - val = str(tag_list[0]).strip() - return val if val else None - return None - - meta['title'] = _first(tags.get('title')) - meta['artist'] = _first(tags.get('artist')) - meta['albumartist'] = _first(tags.get('albumartist')) - meta['album'] = _first(tags.get('album')) - - tn = _first(tags.get('tracknumber')) - if tn: - try: - meta['track_number'] = int(tn.split('/')[0]) - except (ValueError, IndexError): - pass - - dn = _first(tags.get('discnumber')) - if dn: - try: - meta['disc_number'] = int(dn.split('/')[0]) - except (ValueError, IndexError): - pass - except Exception: - pass - - # Phase 2: Only fall back to filename parsing when tags are genuinely empty - if not meta['title'] and not meta['artist']: - parsed = _parse_filename_metadata(filename) - meta['title'] = parsed.get('title') or os.path.splitext(os.path.basename(filename))[0] - meta['artist'] = parsed.get('artist') - if not meta['track_number']: - meta['track_number'] = parsed.get('track_number') - if not meta['album']: - meta['album'] = parsed.get('album') - elif not meta['title']: - # Has artist tag but no title — use filename for title only - meta['title'] = os.path.splitext(os.path.basename(filename))[0] - - return meta + return read_staging_file_metadata(full_path, filename) # =================================================================== @@ -17160,223 +16692,6 @@ def _search_track_in_album_context(original_search: dict, artist: dict) -> dict: return None - - -def _detect_album_info_web(context: dict, artist: dict) -> dict: - """ - Enhanced album detection with GUI parity - multi-priority logic. - (Updated to match GUI downloads.py logic exactly) - """ - try: - # Log available data for debugging (GUI PARITY) - original_search = context.get("original_search_result", {}) - logger.info( - "[Album Detection] start: track=%r clean_spotify_title=%r clean_spotify_album=%r " - "filename_album=%r artist=%r clean_data=%s album_download=%s", - original_search.get('title', 'Unknown'), - original_search.get('spotify_clean_title', 'None'), - original_search.get('spotify_clean_album', 'None'), - original_search.get('album', 'None'), - artist.get('name', 'Unknown'), - context.get('has_clean_spotify_data', False), - context.get('is_album_download', False), - ) - spotify_album_context = context.get("spotify_album") - is_album_download = context.get("is_album_download", False) - artist_name = artist['name'] - - logger.info( - "[Album Detection] track=%r artist=%r has_album_attr=%s album=%r", - original_search.get('title', 'Unknown'), - artist_name, - bool(original_search.get('album')), - original_search.get('album'), - ) - - # --- THIS IS THE CRITICAL FIX --- - # If this is part of a matched album download, we TRUST the context data completely. - # This is the exact logic from downloads.py. - if is_album_download and spotify_album_context: - # We exclusively use the track number and title that were matched - # *before* the download started. We do not try to re-parse the filename. - track_number = original_search.get('track_number', 1) - clean_track_name = original_search.get('title', 'Unknown Track') - - logger.info( - "[Album Detection] using matched context: track_number=%s title=%r album=%r", - track_number, - clean_track_name, - spotify_album_context['name'], - ) - - return { - 'is_album': True, - 'album_name': spotify_album_context['name'], - 'track_number': track_number, - 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album_context.get('image_url') - } - - # PRIORITY 1: Try album-aware search using clean Spotify album name (GUI PARITY) - # Prioritize clean Spotify album name over filename-parsed album - clean_album_name = original_search.get('spotify_clean_album') - fallback_album_name = original_search.get('album') - - album_name_to_use = None - album_source = None - - if clean_album_name and clean_album_name.strip() and clean_album_name != "Unknown Album": - album_name_to_use = clean_album_name - album_source = "CLEAN_SPOTIFY" - elif fallback_album_name and fallback_album_name.strip() and fallback_album_name != "Unknown Album": - album_name_to_use = fallback_album_name - album_source = "FILENAME_PARSED" - - if album_name_to_use: - track_title = original_search.get('spotify_clean_title') or original_search.get('title', 'Unknown') - logger.info(f"ALBUM-AWARE SEARCH ({album_source}): Looking for '{track_title}' in album '{album_name_to_use}'") - - # Temporarily set the album for the search - original_album = original_search.get('album') - original_search['album'] = album_name_to_use - - try: - album_result = _search_track_in_album_context_web(context, artist) - if album_result: - logger.info(f"PRIORITY 1 SUCCESS: Found track using {album_source} album name - FORCING album classification") - return album_result - else: - logger.error(f"PRIORITY 1 FAILED: Track not found using {album_source} album name") - finally: - # Restore original album value - if original_album is not None: - original_search['album'] = original_album - else: - original_search.pop('album', None) - - # PRIORITY 2: Fallback to individual track search for clean metadata - logger.info("Searching Spotify for individual track info (PRIORITY 2)...") - - # Clean the track title before searching - remove artist prefix - # Prioritize clean Spotify title over filename-parsed title - track_title_to_use = original_search.get('spotify_clean_title') or original_search.get('title', '') - clean_title = _clean_track_title_web(track_title_to_use, artist_name) - logger.info(f"Cleaned title: '{track_title_to_use}' -> '{clean_title}'") - - # Search for the track by artist and cleaned title - query = f"artist:{artist_name} track:{clean_title}" - tracks = spotify_client.search_tracks(query, limit=5) - - # Find the best matching track (prefer album versions over singles) - best_match = None - best_confidence = 0 - - if tracks: - from core.matching_engine import MusicMatchingEngine - matching_engine = MusicMatchingEngine() - for track in tracks: - # Calculate confidence based on artist and title similarity - artist_confidence = matching_engine.similarity_score( - matching_engine.normalize_string(artist_name), - matching_engine.normalize_string(track.artists[0] if track.artists else '') - ) - title_confidence = matching_engine.similarity_score( - matching_engine.normalize_string(clean_title), - matching_engine.normalize_string(track.name) - ) - - combined_confidence = (artist_confidence * 0.6 + title_confidence * 0.4) - - # Small bonus for album tracks so they win ties over singles/EPs - album_type = getattr(track, 'album_type', None) or '' - if album_type == 'album': - combined_confidence += 0.02 - elif album_type == 'ep': - combined_confidence += 0.01 - - if combined_confidence > best_confidence and combined_confidence > 0.75: # Higher threshold to avoid bad matches - best_match = track - best_confidence = combined_confidence - - # If we found a good Spotify match, use it for clean metadata - if best_match and best_confidence > 0.75: - logger.info(f"Found matching Spotify track: '{best_match.name}' - Album: '{best_match.album}' (confidence: {best_confidence:.2f})") - - # Get detailed track information using Spotify's track API - detailed_track = None - if hasattr(best_match, 'id') and best_match.id: - logger.info(f"Getting detailed track info from Spotify API for track ID: {best_match.id}") - detailed_track = spotify_client.get_track_details(best_match.id) - - # Use detailed track data if available - if detailed_track: - logger.info("Got detailed track data from Spotify API") - album_name = _clean_album_title_web(detailed_track['album']['name'], artist_name) - clean_track_name = detailed_track['name'] # Use Spotify's clean track name - album_type = detailed_track['album'].get('album_type', 'album') - total_tracks = detailed_track['album'].get('total_tracks', 1) - spotify_track_number = detailed_track.get('track_number', 1) - - logger.info(f"Spotify album info: '{album_name}' (type: {album_type}, total_tracks: {total_tracks}, track#: {spotify_track_number})") - logger.info(f"Clean track name from Spotify: '{clean_track_name}'") - - # Enhanced album detection using detailed API data (GUI PARITY) - is_album = ( - # Album type is 'album' (not 'single') - album_type == 'album' and - # Album has multiple tracks - total_tracks > 1 and - # Album name different from track name - matching_engine.normalize_string(album_name) != matching_engine.normalize_string(clean_track_name) and - # Album name is not just the artist name - matching_engine.normalize_string(album_name) != matching_engine.normalize_string(artist_name) - ) - - album_image_url = None - if detailed_track['album'].get('images'): - album_image_url = detailed_track['album']['images'][0].get('url') - - logger.info(f"Album classification: {is_album} (type={album_type}, tracks={total_tracks})") - - return { - 'is_album': is_album, - 'album_name': album_name, - 'track_number': spotify_track_number, - 'clean_track_name': clean_track_name, - 'album_image_url': album_image_url, - 'confidence': best_confidence, - 'source': 'spotify_api_detailed' - } - - # Fallback: Use original data with basic cleaning - logger.warning("No good Spotify match found, using original data") - fallback_title = _clean_track_title_web(original_search.get('title', 'Unknown Track'), artist_name) - - # Preserve track_number from context if available (playlist sync tracks have it) - _ctx_track_number = (original_search.get('track_number') - or context.get('track_info', {}).get('track_number') - or 1) - - return { - 'is_album': False, - 'clean_track_name': fallback_title, - 'album_name': fallback_title, - 'track_number': _ctx_track_number, - 'confidence': 0.0, - 'source': 'fallback_original' - } - - except Exception as e: - logger.error(f"Error in _detect_album_info_web: {e}") - clean_title = _clean_track_title_web(context.get("original_search_result", {}).get('title', 'Unknown'), artist.get('name', '')) - _err_tn = (context.get("original_search_result", {}).get('track_number') - or context.get('track_info', {}).get('track_number') - or 1) - return {'is_album': False, 'clean_track_name': clean_title, 'album_name': clean_title, 'track_number': _err_tn} - - - - def _cleanup_empty_directories(download_path, moved_file_path): """Cleans up empty directories after a file move, ignoring hidden files.""" import os @@ -18520,333 +17835,6 @@ def _get_audio_quality_string(file_path): logger.debug(f"Could not determine audio quality for {file_path}: {e}") return '' -def _downsample_hires_flac(final_path, context): - """Downsample a 24-bit hi-res FLAC to 16-bit/44.1kHz CD quality. - - Only runs when downsample_hires is enabled and the file is a 24-bit FLAC. - Replaces the original file in-place (write to temp, verify, swap). - - Returns the (possibly renamed) final_path, or None if no conversion needed. - """ - if not config_manager.get('lossy_copy.downsample_hires', False): - return None - - ext = os.path.splitext(final_path)[1].lower() - if ext != '.flac': - return None - - # Check current bit depth — only downsample if hi-res (>16 bit) - try: - from mutagen.flac import FLAC - audio = FLAC(final_path) - original_bits = audio.info.bits_per_sample - original_rate = audio.info.sample_rate - except Exception as e: - logger.error(f"[Downsample] Could not read FLAC info: {e}") - return None - - if original_bits <= 16 and original_rate <= 44100: - return None # Already CD quality or below - - logger.info(f"[Downsample] Converting {original_bits}-bit/{original_rate}Hz → 16-bit/44100Hz: {os.path.basename(final_path)}") - - ffmpeg_bin = shutil.which('ffmpeg') - if not ffmpeg_bin: - local = os.path.join(os.path.dirname(__file__), 'tools', 'ffmpeg') - if os.path.isfile(local): - ffmpeg_bin = local - else: - logger.warning("[Downsample] ffmpeg not found — skipping hi-res conversion") - return None - - temp_path = final_path + '.tmp.flac' - try: - result = subprocess.run([ - ffmpeg_bin, '-i', final_path, - '-sample_fmt', 's16', - '-ar', '44100', - '-map_metadata', '0', - '-compression_level', '8', - '-y', temp_path - ], capture_output=True, text=True, timeout=300) - - if result.returncode != 0: - logger.error(f"[Downsample] ffmpeg failed: {result.stderr[:200]}") - if os.path.exists(temp_path): - os.remove(temp_path) - return None - - # Verify the output is a valid 16-bit FLAC - if not os.path.isfile(temp_path) or os.path.getsize(temp_path) == 0: - logger.warning("[Downsample] Output file missing or empty") - if os.path.exists(temp_path): - os.remove(temp_path) - return None - - verify_audio = FLAC(temp_path) - if verify_audio.info.bits_per_sample != 16: - logger.info(f"[Downsample] Output not 16-bit ({verify_audio.info.bits_per_sample}-bit), aborting") - os.remove(temp_path) - return None - - # Atomic swap — replace original with downsampled version - os.replace(temp_path, final_path) - logger.info(f"[Downsample] Converted to 16-bit/44.1kHz: {os.path.basename(final_path)}") - - # Update QUALITY tag in the new file - new_quality = 'FLAC 16bit' - try: - updated_audio = FLAC(final_path) - updated_audio['QUALITY'] = new_quality - updated_audio.save() - except Exception as tag_err: - logger.error(f"[Downsample] Could not update QUALITY tag: {tag_err}") - - # Update context so downstream (lossy copy, metadata) reflects new quality - old_quality = context.get('_audio_quality', '') - context['_audio_quality'] = new_quality - - # If filename contains old quality string (from $quality template), rename - if old_quality and old_quality != new_quality and old_quality in os.path.basename(final_path): - new_basename = os.path.basename(final_path).replace(old_quality, new_quality) - new_path = os.path.join(os.path.dirname(final_path), new_basename) - try: - os.rename(final_path, new_path) - logger.info(f"[Downsample] Renamed: {os.path.basename(final_path)} → {new_basename}") - # Rename matching lyrics sidecar file if it exists (.lrc or .txt) - for lyrics_ext in ('.lrc', '.txt'): - old_lyrics = os.path.splitext(final_path)[0] + lyrics_ext - if os.path.isfile(old_lyrics): - new_lyrics = os.path.splitext(new_path)[0] + lyrics_ext - os.rename(old_lyrics, new_lyrics) - return new_path - except Exception as rename_err: - logger.error(f"[Downsample] Could not rename file: {rename_err}") - - return final_path - - except subprocess.TimeoutExpired: - logger.info(f"[Downsample] Conversion timed out for: {os.path.basename(final_path)}") - if os.path.exists(temp_path): - os.remove(temp_path) - except Exception as e: - logger.error(f"[Downsample] Conversion error: {e}") - if os.path.exists(temp_path): - try: - os.remove(temp_path) - except Exception: - pass - return None - - -def _create_lossy_copy(final_path): - """Convert a FLAC file to a lossy codec at the user's configured bitrate. - - Supported codecs: mp3 (libmp3lame), opus (libopus), aac (aac/libfdk_aac). - Only runs when lossy_copy is enabled and the file is a FLAC. - Places the output alongside the FLAC with the same basename. - - Returns the output path if Blasphemy Mode deleted the original, else None. - """ - if not config_manager.get('lossy_copy.enabled', False): - return None - - ext = os.path.splitext(final_path)[1].lower() - if ext != '.flac': - return None - - codec = config_manager.get('lossy_copy.codec', 'mp3').lower() - bitrate = config_manager.get('lossy_copy.bitrate', '320') - - # Opus max per-channel bitrate is 256kbps — cap to avoid encoding failures - if codec == 'opus' and int(bitrate) > 256: - bitrate = '256' - - # Codec configuration: (ffmpeg_codec, extension, quality_label, extra_args) - # -vn strips video/image streams (embedded cover art) which can cause - # conversion failures when the output muxer can't handle image streams - codec_map = { - 'mp3': ('libmp3lame', '.mp3', f'MP3-{bitrate}', ['-vn', '-id3v2_version', '3']), - 'opus': ('libopus', '.opus', f'OPUS-{bitrate}', ['-vn', '-map', '0:a', '-vbr', 'on']), - 'aac': ('aac', '.m4a', f'AAC-{bitrate}', ['-vn', '-movflags', '+faststart']), - } - - if codec not in codec_map: - logger.info(f"[Lossy Copy] Unknown codec '{codec}' — skipping conversion") - return None - - ffmpeg_codec, out_ext, quality_label, extra_args = codec_map[codec] - out_path = os.path.splitext(final_path)[0] + out_ext - - # If $quality was used in filename, swap FLAC quality for lossy quality - original_quality = _get_audio_quality_string(final_path) - if original_quality: - out_basename = os.path.basename(out_path) - if original_quality in out_basename: - out_basename = out_basename.replace(original_quality, quality_label) - out_path = os.path.join(os.path.dirname(out_path), out_basename) - - ffmpeg_bin = shutil.which('ffmpeg') - if not ffmpeg_bin: - local = os.path.join(os.path.dirname(__file__), 'tools', 'ffmpeg') - if os.path.isfile(local): - ffmpeg_bin = local - else: - logger.warning(f"[Lossy Copy] ffmpeg not found — skipping {codec.upper()} conversion") - return None - - try: - logger.info(f"[Lossy Copy] Converting to {quality_label}: {os.path.basename(final_path)}") - cmd = [ - ffmpeg_bin, '-i', final_path, - '-codec:a', ffmpeg_codec, - '-b:a', f'{bitrate}k', - '-map_metadata', '0', - ] + extra_args + ['-y', out_path] - - result = subprocess.run(cmd, capture_output=True, text=True, timeout=120) - - if result.returncode == 0: - logger.info(f"[Lossy Copy] Created {quality_label} copy: {os.path.basename(out_path)}") - - # Fix QUALITY tag — the FLAC's tag was copied verbatim by ffmpeg - try: - from mutagen import File as MutagenFile - audio = MutagenFile(out_path) - if audio is not None: - if codec == 'mp3': - from mutagen.id3 import TXXX - audio.tags.add(TXXX(encoding=3, desc='QUALITY', text=[quality_label])) - elif codec == 'opus': - audio['QUALITY'] = [quality_label] - elif codec == 'aac': - from mutagen.mp4 import MP4FreeForm - audio['----:com.apple.iTunes:QUALITY'] = [MP4FreeForm(quality_label.encode('utf-8'))] - audio.save() - except Exception as tag_err: - logger.error(f"[Lossy Copy] Could not update QUALITY tag: {tag_err}") - - # Embed cover art from source FLAC into the lossy copy - # Opus/OGG can't inherit FLAC cover art via ffmpeg -map_metadata alone - if codec in ('opus', 'aac'): - try: - from mutagen import File as MutagenFile - from mutagen.flac import FLAC as MutagenFLAC - source_audio = MutagenFLAC(final_path) - pic = None - if source_audio and source_audio.pictures: - pic = source_audio.pictures[0] - - # Fallback: read cover.jpg from the same directory - if not pic: - cover_path = os.path.join(os.path.dirname(final_path), 'cover.jpg') - if os.path.isfile(cover_path): - try: - from mutagen.flac import Picture - with open(cover_path, 'rb') as f: - img_data = f.read() - pic = Picture() - pic.type = 3 # Cover (front) - pic.mime = 'image/jpeg' - pic.desc = 'Cover' - pic.width = 0 - pic.height = 0 - pic.depth = 0 - pic.colors = 0 - pic.data = img_data - logger.warning("[Lossy Copy] Using cover.jpg as art source (FLAC had no embedded art)") - except Exception: - pass - - if pic: - dest_audio = MutagenFile(out_path) - if dest_audio is not None: - if codec == 'opus': - import base64 - from mutagen.oggopus import OggOpus - if isinstance(dest_audio, OggOpus): - # OGG stores pictures as base64-encoded METADATA_BLOCK_PICTURE - import struct - # Build METADATA_BLOCK_PICTURE block - picture_data = ( - struct.pack('>II', pic.type, len(pic.mime.encode('utf-8'))) - + pic.mime.encode('utf-8') - + struct.pack('>I', len(pic.desc.encode('utf-8'))) - + pic.desc.encode('utf-8') - + struct.pack('>IIII', pic.width, pic.height, pic.depth, pic.colors) - + struct.pack('>I', len(pic.data)) - + pic.data - ) - dest_audio['METADATA_BLOCK_PICTURE'] = [base64.b64encode(picture_data).decode('ascii')] - dest_audio.save() - logger.info("[Lossy Copy] Embedded cover art in Opus file") - elif codec == 'aac': - from mutagen.mp4 import MP4Cover - fmt = MP4Cover.FORMAT_JPEG if 'jpeg' in pic.mime else MP4Cover.FORMAT_PNG - dest_audio['covr'] = [MP4Cover(pic.data, imageformat=fmt)] - dest_audio.save() - logger.info("[Lossy Copy] Embedded cover art in M4A file") - except Exception as art_err: - logger.error(f"[Lossy Copy] Could not embed cover art: {art_err}") - - # Blasphemy Mode: delete original FLAC if enabled and output is verified - if config_manager.get('lossy_copy.delete_original', False): - try: - if os.path.isfile(out_path) and os.path.getsize(out_path) > 0: - from mutagen import File as MutagenFile - test_audio = MutagenFile(out_path) - if test_audio is not None: - # Update provenance record to point to the new transcoded file - try: - db = get_database() - db.update_provenance_file_path(final_path, out_path) - except Exception: - pass - os.remove(final_path) - logger.info(f"[Blasphemy Mode] Deleted original: {os.path.basename(final_path)}") - # Rename lyrics sidecar file to match the output filename - for lyrics_ext in ('.lrc', '.txt'): - src_lyrics = os.path.splitext(final_path)[0] + lyrics_ext - if os.path.isfile(src_lyrics): - dst_lyrics = os.path.splitext(out_path)[0] + lyrics_ext - try: - os.rename(src_lyrics, dst_lyrics) - logger.info(f"[Blasphemy Mode] Renamed {lyrics_ext}: {os.path.basename(src_lyrics)} -> {os.path.basename(dst_lyrics)}") - except Exception as lrc_err: - logger.error(f"[Blasphemy Mode] Could not rename {lyrics_ext}: {lrc_err}") - return out_path - else: - logger.error(f"[Blasphemy Mode] Output failed audio validation, keeping original: {os.path.basename(final_path)}") - else: - logger.warning(f"[Blasphemy Mode] Output missing or empty, keeping original: {os.path.basename(final_path)}") - except Exception as del_err: - logger.error(f"[Blasphemy Mode] Error during original deletion, keeping original: {del_err}") - else: - # ffmpeg always prints its version banner to stderr (~300 chars). - # Strip it so the actual error is visible, and show more than 200 chars. - stderr = result.stderr or '' - # Remove the version/config preamble (ends after the first empty line) - stderr_lines = stderr.split('\n') - error_lines = [] - past_banner = False - for line in stderr_lines: - if past_banner: - error_lines.append(line) - elif line.strip() == '': - past_banner = True - error_msg = '\n'.join(error_lines).strip() if error_lines else stderr[-500:] - logger.error(f"[Lossy Copy] ffmpeg failed (exit code {result.returncode}): {error_msg[:500]}") - # Clean up empty/broken output file - if os.path.isfile(out_path) and os.path.getsize(out_path) == 0: - os.remove(out_path) - logger.warning(f"[Lossy Copy] Removed empty output file: {os.path.basename(out_path)}") - except subprocess.TimeoutExpired: - logger.info(f"[Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}") - if os.path.isfile(out_path) and os.path.getsize(out_path) == 0: - os.remove(out_path) - except Exception as e: - logger.error(f"[Lossy Copy] Conversion error: {e}") - return None def _get_album_type_display(raw_type, track_count) -> str: """ @@ -19096,516 +18084,10 @@ from mutagen.apev2 import APEv2, APENoHeaderError import urllib.request def _wipe_source_tags(file_path: str) -> bool: - """Emergency tag wipe — clears ALL tags from a file without writing new ones. - Used when full metadata enhancement is skipped or fails, to prevent original - Soulseek source tags (especially MusicBrainz IDs from the uploader) from - persisting and causing album splits in media servers like Navidrome.""" - try: - _strip_all_non_audio_tags(file_path) - audio = MutagenFile(file_path) - if audio is None: - return False - if hasattr(audio, 'clear_pictures'): - audio.clear_pictures() - if audio.tags is not None: - tag_count = len(audio.tags) - audio.tags.clear() - else: - audio.add_tags() - tag_count = 0 - if isinstance(audio.tags, ID3): - audio.save(v1=0, v2_version=4) - elif isinstance(audio, FLAC): - audio.save(deleteid3=True) - else: - audio.save() - if tag_count > 0: - logger.info(f"[Tag Wipe] Stripped {tag_count} source tags from: {os.path.basename(file_path)}") - return True - except Exception as e: - logger.error(f"[Tag Wipe] Failed (non-fatal): {e}") - return False - - -def _strip_all_non_audio_tags(file_path: str) -> dict: - """ - Strip ALL non-audio tag containers from a file before metadata rewriting. - MP3 files from Soulseek commonly carry APEv2 tags (foobar2000 users) - with stale metadata that Mutagen's ID3 handler cannot see or clear. - Must run BEFORE MutagenFile() opens the file. - """ - summary = {'apev2_stripped': False, 'apev2_tag_count': 0} - ext = os.path.splitext(file_path)[1].lower() - if ext != '.mp3': - return summary - try: - apev2_tags = APEv2(file_path) - tag_count = len(apev2_tags) - tag_keys = list(apev2_tags.keys()) - apev2_tags.delete(file_path) - summary['apev2_stripped'] = True - summary['apev2_tag_count'] = tag_count - logger.info(f"Stripped {tag_count} APEv2 tags: {', '.join(tag_keys[:10])}") - except APENoHeaderError: - pass # No APEv2 tags — common case - except Exception as e: - logger.error(f"Could not strip APEv2 tags (non-fatal): {e}") - return summary - -def _verify_metadata_written(file_path: str) -> bool: - """Re-open file and verify core metadata fields are present.""" - try: - check = MutagenFile(file_path) - if check is None or check.tags is None: - logger.info(f"[VERIFY] Tags are None after save: {file_path}") - return False - title_found = False - artist_found = False - if isinstance(check.tags, ID3): - title_found = bool(check.tags.getall('TIT2')) - artist_found = bool(check.tags.getall('TPE1')) - # Confirm APEv2 is gone - try: - APEv2(file_path) - logger.info("[VERIFY] APEv2 tags still present after processing!") - return False - except APENoHeaderError: - pass - elif isinstance(check, (FLAC, OggVorbis)) or _is_ogg_opus(check): - title_found = bool(check.get('title')) - artist_found = bool(check.get('artist')) - elif isinstance(check, MP4): - title_found = bool(check.get('\xa9nam')) - artist_found = bool(check.get('\xa9ART')) - if not title_found or not artist_found: - logger.warning(f"[VERIFY] Missing metadata - title:{title_found} artist:{artist_found}") - return False - logger.info("[VERIFY] Metadata verified OK") - return True - except Exception as e: - logger.error(f"[VERIFY] Verification error (non-fatal): {e}") - return False - -def _is_ogg_opus(audio_file): - """Check if a Mutagen file object is OggOpus (uses VorbisComment tags like FLAC/OGG).""" - return type(audio_file).__name__ == 'OggOpus' + return metadata_enrichment.wipe_source_tags(file_path) def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: - """ - Core function to enhance audio file metadata using Spotify data. - Thread-safe with per-file locking to prevent concurrent metadata writes. - - Opens the file once in non-easy mode, clears all tags in memory, writes - new tags using format-specific frames/keys, embeds album art and source - IDs, then saves once. This avoids the old clear→save→reopen pattern - which stripped the ID3v2 header from MP3 files, leaving them tagless. - """ - if not config_manager.get('metadata_enhancement.enabled', True): - logger.warning("Metadata enhancement disabled in config.") - return True - - # Normalize None album_info to empty dict to prevent AttributeError on .get() calls - if album_info is None: - album_info = {} - - # Acquire per-file lock to prevent concurrent metadata writes to the same file - file_lock = _get_file_lock(file_path) - with file_lock: - logger.info(f"Enhancing metadata for: {os.path.basename(file_path)}") - try: - # Strip APEv2 tags from MP3 (invisible to ID3 handler) - strip_summary = _strip_all_non_audio_tags(file_path) - - audio_file = MutagenFile(file_path) - if audio_file is None: - logger.error(f"Could not load audio file with Mutagen: {file_path}") - return False - - # ── Wipe ALL existing tags and save immediately ── - # Files from Soulseek carry random metadata (wrong comments, - # encoder info, ReplayGain, old album art, random TXXX frames). - # Save the cleared state FIRST so that if anything below throws, - # the file at least has clean (empty) tags instead of junk that - # causes album fragmentation in media servers. - if hasattr(audio_file, 'clear_pictures'): - audio_file.clear_pictures() - - if audio_file.tags is not None: - if len(audio_file.tags) > 0: - tag_keys = list(audio_file.tags.keys())[:15] - logger.info(f"Clearing {len(audio_file.tags)} existing tags: " - f"{', '.join(str(k) for k in tag_keys)}") - audio_file.tags.clear() - else: - audio_file.add_tags() - - # Persist the wipe — guarantees junk tags are gone even if later steps fail - if isinstance(audio_file.tags, ID3): - audio_file.save(v1=0, v2_version=4) - elif isinstance(audio_file, FLAC): - audio_file.save(deleteid3=True) - else: - audio_file.save() - - metadata = _extract_spotify_metadata(context, artist, album_info) - if not metadata: - logger.error("Could not extract Spotify metadata, saving with cleared tags.") - if isinstance(audio_file.tags, ID3): - audio_file.save(v1=0, v2_version=4) - elif isinstance(audio_file, FLAC): - audio_file.save(deleteid3=True) - else: - audio_file.save() - return True - - # ── Write standard tags using format-specific API ── - track_num_str = f"{metadata.get('track_number', 1)}/{metadata.get('total_tracks', 1)}" - - _write_multi = config_manager.get('metadata_enhancement.tags.write_multi_artist', False) - _artists_list = metadata.get('_artists_list', []) - - if isinstance(audio_file.tags, ID3): - # MP3: write ID3 frames directly - if metadata.get('title'): - audio_file.tags.add(TIT2(encoding=3, text=[metadata['title']])) - if metadata.get('artist'): - audio_file.tags.add(TPE1(encoding=3, text=[metadata['artist']])) - # Multi-value: write each artist as separate TPE1 text value - if _write_multi and len(_artists_list) > 1: - audio_file.tags.add(TPE1(encoding=3, text=_artists_list)) - if metadata.get('album_artist'): - audio_file.tags.add(TPE2(encoding=3, text=[metadata['album_artist']])) - if metadata.get('album'): - audio_file.tags.add(TALB(encoding=3, text=[metadata['album']])) - if metadata.get('date'): - audio_file.tags.add(TDRC(encoding=3, text=[metadata['date']])) - if metadata.get('genre'): - audio_file.tags.add(TCON(encoding=3, text=[metadata['genre']])) - audio_file.tags.add(TRCK(encoding=3, text=[track_num_str])) - if metadata.get('disc_number'): - audio_file.tags.add(TPOS(encoding=3, text=[str(metadata['disc_number'])])) - - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - # FLAC / OGG Vorbis / OGG Opus: dict-style VorbisComment tags - if metadata.get('title'): - audio_file['title'] = [metadata['title']] - if metadata.get('artist'): - audio_file['artist'] = [metadata['artist']] - # Multi-value: write ARTISTS tag with individual values - if _write_multi and len(_artists_list) > 1: - audio_file['artists'] = _artists_list - if metadata.get('album_artist'): - audio_file['albumartist'] = [metadata['album_artist']] - if metadata.get('album'): - audio_file['album'] = [metadata['album']] - if metadata.get('date'): - audio_file['date'] = [metadata['date']] - if metadata.get('genre'): - audio_file['genre'] = [metadata['genre']] - audio_file['tracknumber'] = [track_num_str] - if metadata.get('disc_number'): - audio_file['discnumber'] = [str(metadata['disc_number'])] - - elif isinstance(audio_file, MP4): - # MP4 / M4A: Apple-style tag keys - if metadata.get('title'): - audio_file['\xa9nam'] = [metadata['title']] - if metadata.get('artist'): - # Multi-value: write each artist as separate list entry - if _write_multi and len(_artists_list) > 1: - audio_file['\xa9ART'] = _artists_list - else: - audio_file['\xa9ART'] = [metadata['artist']] - if metadata.get('album_artist'): - audio_file['aART'] = [metadata['album_artist']] - if metadata.get('album'): - audio_file['\xa9alb'] = [metadata['album']] - if metadata.get('date'): - audio_file['\xa9day'] = [metadata['date']] - if metadata.get('genre'): - audio_file['\xa9gen'] = [metadata['genre']] - track_num = metadata.get('track_number', 1) - total_tracks = metadata.get('total_tracks', 1) - audio_file['trkn'] = [(track_num, total_tracks)] - if metadata.get('disc_number'): - audio_file['disk'] = [(metadata['disc_number'], 0)] - - # ── Embed source IDs (Spotify, MusicBrainz, etc.) on the same object ── - # Runs before album art so MusicBrainz release ID is available for - # Cover Art Archive high-resolution lookup. - _embed_source_ids(audio_file, metadata, context) - - # Propagate MusicBrainz release ID to album_info so _download_cover_art - # can use it for Cover Art Archive high-res cover.jpg - if album_info is not None and metadata.get('musicbrainz_release_id'): - album_info['musicbrainz_release_id'] = metadata['musicbrainz_release_id'] - - # ── Embed album art on the same object ── - if config_manager.get('metadata_enhancement.embed_album_art', True): - _embed_album_art_metadata(audio_file, metadata) - - # ── Embed audio quality tag ── - quality = context.get('_audio_quality', '') - if quality and config_manager.get('metadata_enhancement.tags.quality_tag', True) is not False: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='QUALITY', text=[quality])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['quality'] = [quality] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:QUALITY'] = [MP4FreeForm(quality.encode('utf-8'))] - - # ── Single save for everything ── - if isinstance(audio_file.tags, ID3): - audio_file.save(v1=0, v2_version=4) - elif isinstance(audio_file, FLAC): - audio_file.save(deleteid3=True) - else: - audio_file.save() - - # Verify metadata was written - verified = _verify_metadata_written(file_path) - if verified: - logger.info("Metadata enhanced successfully.") - else: - logger.info("Metadata saved but verification found issues (see above).") - return True - except Exception as e: - import traceback - logger.error(f"Error enhancing metadata for {file_path}: {e}") - logger.error(f"[Metadata Debug] Exception type: {type(e).__name__}") - logger.info(f"[Metadata Debug] File exists: {os.path.exists(file_path)}") - logger.warning(f"[Metadata Debug] Artist: {artist.get('name', 'MISSING') if artist else 'None'}") - logger.warning(f"[Metadata Debug] Album info: {album_info.get('album_name', 'MISSING') if album_info else 'None'}") - logger.error(f"[Metadata Debug] Traceback:\n{traceback.format_exc()}") - return False - -def _generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: - """ - Generate LRC lyrics file using LRClib API. - Elegant addition to post-processing - extracts metadata from existing context. - """ - if not config_manager.get('metadata_enhancement.lrclib_enabled', True): - return False - try: - # Extract track information from existing context (same as metadata enhancement) - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album") - - # Get track metadata - track_name = (original_search.get('spotify_clean_title') or - original_search.get('title', 'Unknown Track')) - - # Handle artist parameter (can be dict or object) - if isinstance(artist, dict): - artist_name = artist.get('name', 'Unknown Artist') - elif hasattr(artist, 'name'): - artist_name = artist.name - else: - artist_name = str(artist) if artist else 'Unknown Artist' - album_name = None - duration_seconds = None - - # Get album name if available - if album_info.get('is_album'): - album_name = (original_search.get('spotify_clean_album') or - album_info.get('album_name') or - (spotify_album.get('name') if spotify_album else None)) - - # Get duration from original search context - if original_search.get('duration_ms'): - duration_seconds = int(original_search['duration_ms'] / 1000) - - # Generate LRC file using lyrics client - success = lyrics_client.create_lrc_file( - audio_file_path=file_path, - track_name=track_name, - artist_name=artist_name, - album_name=album_name, - duration_seconds=duration_seconds - ) - - if success: - logger.info(f"LRC file generated for: {track_name}") - else: - logger.warning(f"No lyrics found for: {track_name}") - - return success - - except Exception as e: - logger.error(f"Error generating LRC file for {file_path}: {e}") - return False - -def _extract_spotify_metadata(context: dict, artist: dict, album_info: dict) -> dict: - """Extracts a comprehensive metadata dictionary from the provided context.""" - metadata = {} - if album_info is None: - album_info = {} - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album") - - # Priority 1: Spotify clean title from context - if original_search.get('spotify_clean_title'): - metadata['title'] = original_search['spotify_clean_title'] - logger.info(f"Metadata: Using Spotify clean title: '{metadata['title']}'") - # Priority 2: Album info clean name - elif album_info.get('clean_track_name'): - metadata['title'] = album_info['clean_track_name'] - logger.info(f"Metadata: Using album info clean name: '{metadata['title']}'") - # Priority 3: Original title as fallback - else: - metadata['title'] = original_search.get('title', '') - logger.warning(f"Metadata: Using original title as fallback: '{metadata['title']}'") - # Handle multiple artists from Spotify data - original_search = context.get("original_search_result", {}) - if 'artists' in original_search and isinstance(original_search['artists'], list) and len(original_search['artists']) > 0: - all_artists = [] - for a in original_search['artists']: - if isinstance(a, dict) and 'name' in a: - all_artists.append(a['name']) - elif isinstance(a, str): - all_artists.append(a) - else: - all_artists.append(str(a)) - metadata['artist'] = ', '.join(all_artists) - logger.info(f"Metadata: Using all artists: '{metadata['artist']}'") - else: - # Fallback to single artist - metadata['artist'] = artist.get('name', '') - logger.info(f"Metadata: Using primary artist: '{metadata['artist']}'") - - # Resolve album_artist for consistent tagging across all tracks in an album. - # Priority: 1) explicit batch artist context (same artist for whole album) - # 2) album-level artists from spotify_album - # 3) context-level artist (spotify_artist parameter) - # 4) collab mode first-artist resolution (per-track, last resort) - # Using album-level artist prevents media server album splits when an artist - # changed names (Kanye West → Ye) and Spotify returns different per-track artists. - _raw_album_artist = artist.get('name', '') - _track_info_ctx = context.get('track_info', {}) or {} - _explicit_aa = _track_info_ctx.get('_explicit_artist_context') if isinstance(_track_info_ctx, dict) else None - - # Build album-level artists list for collab mode resolution. - # Using album-level artists (instead of per-track) ensures collab mode produces - # the SAME album_artist tag for every track, preventing media server album splits. - _album_artists_for_collab = None - if isinstance(_explicit_aa, dict) and _explicit_aa.get('name'): - _raw_album_artist = _explicit_aa['name'] - _album_artists_for_collab = [_explicit_aa] - elif isinstance(_explicit_aa, str) and _explicit_aa: - _raw_album_artist = _explicit_aa - _album_artists_for_collab = [{'name': _explicit_aa}] - elif spotify_album and isinstance(spotify_album, dict): - _sa_aa = spotify_album.get('artists', []) - if _sa_aa: - _first_aa = _sa_aa[0] - if isinstance(_first_aa, dict) and _first_aa.get('name'): - _raw_album_artist = _first_aa['name'] - elif isinstance(_first_aa, str) and _first_aa: - _raw_album_artist = _first_aa - _album_artists_for_collab = _sa_aa - - collab_mode = config_manager.get('file_organization.collab_artist_mode', 'first') - if collab_mode == 'first' and _raw_album_artist: - original_search = context.get("original_search_result", {}) - # Prefer album-level artists for collab resolution (consistent per album) - _ctx_artists = _album_artists_for_collab or original_search.get('artists') or _track_info_ctx.get('artists') or [] - if len(_ctx_artists) > 1: - # Multiple artist objects (Spotify) — use first - first = _ctx_artists[0] - _raw_album_artist = first.get('name', first) if isinstance(first, dict) else str(first) - elif len(_ctx_artists) == 1 and (',' in _raw_album_artist or ' & ' in _raw_album_artist): - # Single combined string (iTunes) — resolve via artist ID - _aid = str(artist.get('id', '')) - _src = original_search.get('_source') or _track_info_ctx.get('_source', '') - if _aid.isdigit() and _src != 'deezer': - try: - resolved = _get_itunes_client().resolve_primary_artist(_aid) - if resolved and resolved != _raw_album_artist: - _raw_album_artist = resolved - except Exception: - pass - metadata['album_artist'] = _raw_album_artist # Crucial for library organization - - if album_info.get('is_album'): - metadata['album'] = album_info.get('album_name', 'Unknown Album') - track_num = album_info.get('track_number', 1) - metadata['track_number'] = track_num - metadata['total_tracks'] = spotify_album.get('total_tracks', 1) if spotify_album else 1 - logger.info(f"[METADATA] Album track - track_number: {track_num}, album: {metadata['album']}") - else: - # SAFEGUARD: If we have spotify_album context, never use track title as album name - # This prevents album tracks from being tagged as singles due to classification errors - if spotify_album and spotify_album.get('name'): - logger.info("[SAFEGUARD] Using spotify_album name instead of track title for album metadata") - metadata['album'] = spotify_album['name'] - # Use corrected track_number from album_info (which should be updated by post-processing) - corrected_track_number = album_info.get('track_number', 1) if album_info else 1 - metadata['track_number'] = corrected_track_number - metadata['total_tracks'] = spotify_album.get('total_tracks', 1) - logger.info(f"[SAFEGUARD] Using track_number: {corrected_track_number}") - else: - metadata['album'] = metadata['title'] # For true singles, album is the title - metadata['track_number'] = 1 - metadata['total_tracks'] = 1 - - # Always write disc_number to overwrite any stale tags from the soulseek source. - # Without this, original disc tags persist and can cause media servers (Plex) to - # split a single album into standard/deluxe based on differing disc numbers. - # Priority: original_search context (from API) > album_info > default to 1 - disc_num = original_search.get('disc_number') - if disc_num is None and album_info: - disc_num = album_info.get('disc_number') - if disc_num is None: - disc_num = 1 - metadata['disc_number'] = disc_num - - if spotify_album and spotify_album.get('release_date'): - metadata['date'] = spotify_album['release_date'][:4] - - if artist.get('genres'): - from core.genre_filter import filter_genres - _genre_list = filter_genres(list(artist['genres'][:2]), config_manager) - if _genre_list: - metadata['genre'] = ', '.join(_genre_list) - - metadata['album_art_url'] = album_info.get('album_image_url') if album_info else None - - # Playlist mode fallback: album_info is None, try to get art from spotify_album context - if not metadata['album_art_url']: - _spa = context.get('spotify_album', {}) - if _spa: - _spa_img = _spa.get('image_url') - if not _spa_img and _spa.get('images'): - _spa_img = _spa['images'][0].get('url') if isinstance(_spa['images'][0], dict) else None - metadata['album_art_url'] = _spa_img - - # Extract source IDs (Spotify or iTunes) for tag embedding - track_info = context.get("track_info", {}) - if track_info and track_info.get('id'): - # Spotify track IDs are alphanumeric strings; iTunes IDs are numeric - # Beatport IDs (beatport_*) are neither — skip them for external ID tagging - track_id = str(track_info['id']) - if track_id.isdigit(): - metadata['itunes_track_id'] = track_id - elif not track_id.startswith('beatport_'): - metadata['spotify_track_id'] = track_id - if artist.get('id'): - artist_id = str(artist['id']) - if artist_id.isdigit(): - metadata['itunes_artist_id'] = artist_id - elif not artist_id.startswith('beatport_'): - metadata['spotify_artist_id'] = artist_id - if spotify_album and spotify_album.get('id'): - album_id = str(spotify_album['id']) - if album_id.isdigit(): - metadata['itunes_album_id'] = album_id - elif not album_id.startswith('beatport_'): - metadata['spotify_album_id'] = album_id - - # Summary log for debugging metadata issues (e.g. wrong album_artist / track_number) - logger.info(f"[Metadata Summary] title='{metadata.get('title')}' | artist='{metadata.get('artist')}' | album_artist='{metadata.get('album_artist')}' | album='{metadata.get('album')}' | track={metadata.get('track_number')}/{metadata.get('total_tracks')} | disc={metadata.get('disc_number')}") - - return metadata + return metadata_enrichment.enhance_file_metadata(file_path, context, artist, album_info) def _get_image_dimensions(data: bytes): """Extract width/height from JPEG or PNG image data without PIL.""" @@ -19633,870 +18115,12 @@ def _get_image_dimensions(data: bytes): return None, None -def _embed_album_art_metadata(audio_file, metadata: dict): - """Downloads and embeds album art — tries Cover Art Archive (full resolution) - first if MusicBrainz release ID is available, falls back to Spotify/iTunes URL.""" - try: - image_data = None - mime_type = None - - # Try Cover Art Archive first (often 1200x1200+, original quality) — opt-in - release_mbid = metadata.get('musicbrainz_release_id') - if release_mbid and config_manager.get('metadata_enhancement.prefer_caa_art', False): - try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" - req = urllib.request.Request(caa_url, headers={'Accept': 'image/*'}) - with urllib.request.urlopen(req, timeout=10) as response: - image_data = response.read() - mime_type = response.info().get_content_type() or 'image/jpeg' - if image_data and len(image_data) > 1000: - logger.info(f"Cover art from Cover Art Archive ({len(image_data) // 1024}KB)") - else: - image_data = None # Too small, likely an error page - except Exception: - image_data = None # Fall through to Spotify/iTunes URL - - # Fallback to Spotify/iTunes/Deezer URL (typically 640x640) - if not image_data: - art_url = metadata.get('album_art_url') - if not art_url: - logger.warning("No album art URL available for embedding.") - return - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - mime_type = response.info().get_content_type() - - if not image_data: - logger.error("Failed to download album art data.") - return - - # MP3 (ID3) - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(APIC(encoding=3, mime=mime_type, type=3, desc='Cover', data=image_data)) - # FLAC - elif isinstance(audio_file, FLAC): - picture = Picture() - picture.data = image_data - picture.type = 3 - picture.mime = mime_type - # Detect actual dimensions from image data - _img_w, _img_h = _get_image_dimensions(image_data) - picture.width = _img_w or 640 - picture.height = _img_h or 640 - picture.depth = 24 - audio_file.add_picture(picture) - # MP4/M4A - elif isinstance(audio_file, MP4): - fmt = MP4Cover.FORMAT_JPEG if 'jpeg' in mime_type else MP4Cover.FORMAT_PNG - audio_file['covr'] = [MP4Cover(image_data, imageformat=fmt)] - - logger.info("Album art successfully embedded.") - except Exception as e: - logger.error(f"Error embedding album art: {e}") - -def _embed_source_ids(audio_file, metadata: dict, context: dict = None): - """ - Lookup MusicBrainz, Deezer, AudioDB, Tidal, Qobuz, Last.fm, and Genius - metadata, then embed them along with Spotify/iTunes source IDs as custom - tags into the audio file. - Tags written: source IDs, BPM (Deezer), mood/style (AudioDB), ISRC - (MB→Deezer→Tidal→Qobuz fallback), copyright (Tidal→Qobuz), - label (Qobuz), URLs (Last.fm/Genius), - and merged genres (Spotify+MB+AudioDB+Last.fm). - One file write, one shot. Concurrent calls are safe — each service has - its own global rate limiter. - Operates on a non-easy-mode MutagenFile object (caller must save). - """ - try: - # ── Per-tag config: maps internal tag name → config path ── - # Each tag can be individually toggled via {service}.tags.{tag_name} - _TAG_CONFIG = { - # Spotify (from metadata, no API call) - 'SPOTIFY_TRACK_ID': 'spotify.tags.track_id', - 'SPOTIFY_ARTIST_ID': 'spotify.tags.artist_id', - 'SPOTIFY_ALBUM_ID': 'spotify.tags.album_id', - # iTunes (from metadata, no API call) - 'ITUNES_TRACK_ID': 'itunes.tags.track_id', - 'ITUNES_ARTIST_ID': 'itunes.tags.artist_id', - 'ITUNES_ALBUM_ID': 'itunes.tags.album_id', - # MusicBrainz IDs - 'MUSICBRAINZ_RECORDING_ID': 'musicbrainz.tags.recording_id', - 'MUSICBRAINZ_ARTIST_ID': 'musicbrainz.tags.artist_id', - 'MUSICBRAINZ_RELEASE_ID': 'musicbrainz.tags.release_id', - 'MUSICBRAINZ_RELEASEGROUPID': 'musicbrainz.tags.release_group_id', - 'MUSICBRAINZ_ALBUMARTISTID': 'musicbrainz.tags.album_artist_id', - 'MUSICBRAINZ_RELEASETRACKID': 'musicbrainz.tags.release_track_id', - # MusicBrainz Release Info - 'RELEASETYPE': 'musicbrainz.tags.release_type', - 'ORIGINALDATE': 'musicbrainz.tags.original_date', - 'RELEASESTATUS': 'musicbrainz.tags.release_status', - 'RELEASECOUNTRY': 'musicbrainz.tags.release_country', - 'BARCODE': 'musicbrainz.tags.barcode', - 'MEDIA': 'musicbrainz.tags.media', - 'TOTALDISCS': 'musicbrainz.tags.total_discs', - 'CATALOGNUMBER': 'musicbrainz.tags.catalog_number', - 'SCRIPT': 'musicbrainz.tags.script', - 'ASIN': 'musicbrainz.tags.asin', - # Deezer - 'DEEZER_TRACK_ID': 'deezer.tags.track_id', - 'DEEZER_ARTIST_ID': 'deezer.tags.artist_id', - # AudioDB - 'AUDIODB_TRACK_ID': 'audiodb.tags.track_id', - # Tidal - 'TIDAL_TRACK_ID': 'tidal.tags.track_id', - 'TIDAL_ARTIST_ID': 'tidal.tags.artist_id', - # Qobuz - 'QOBUZ_TRACK_ID': 'qobuz.tags.track_id', - 'QOBUZ_ARTIST_ID': 'qobuz.tags.artist_id', - # Genius - 'GENIUS_TRACK_ID': 'genius.tags.track_id', - } - - def _tag_enabled(config_path): - """Check if an individual tag is enabled (defaults to True).""" - return config_manager.get(config_path, True) is not False - - # ── Helper: normalize + compare names (same logic as enrichment workers) ── - from difflib import SequenceMatcher - def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: - if not a or not b: - return False - norm = lambda s: re.sub(r'[^a-z0-9 ]', '', re.sub(r'\(.*?\)', '', s).lower()).strip() - return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold - - # ── 1. Collect Spotify / iTunes IDs already in metadata ── - id_tags = {} - if config_manager.get('spotify.embed_tags', True) is not False: - if metadata.get('spotify_track_id'): - id_tags['SPOTIFY_TRACK_ID'] = metadata['spotify_track_id'] - if metadata.get('spotify_artist_id'): - id_tags['SPOTIFY_ARTIST_ID'] = metadata['spotify_artist_id'] - if metadata.get('spotify_album_id'): - id_tags['SPOTIFY_ALBUM_ID'] = metadata['spotify_album_id'] - if config_manager.get('itunes.embed_tags', True) is not False: - if metadata.get('itunes_track_id'): - id_tags['ITUNES_TRACK_ID'] = metadata['itunes_track_id'] - if metadata.get('itunes_artist_id'): - id_tags['ITUNES_ARTIST_ID'] = metadata['itunes_artist_id'] - if metadata.get('itunes_album_id'): - id_tags['ITUNES_ALBUM_ID'] = metadata['itunes_album_id'] - - # Shared post-processing context for modular lookups - track_title = metadata.get('title', '') - artist_name = metadata.get('album_artist', '') or metadata.get('artist', '') - - # Extract batch-level artist name for stable MB release cache keys. - # When downloading an album batch, all tracks should use the same artist key - # to guarantee they hit the same preflight-cached release MBID. - _track_info_for_pp = (context or {}).get('track_info', {}) or {} - _explicit_artist_for_pp = _track_info_for_pp.get('_explicit_artist_context') if isinstance(_track_info_for_pp, dict) else None - _batch_artist_name = None - if isinstance(_explicit_artist_for_pp, dict) and _explicit_artist_for_pp.get('name'): - _batch_artist_name = _explicit_artist_for_pp['name'] - elif isinstance(_explicit_artist_for_pp, str) and _explicit_artist_for_pp: - _batch_artist_name = _explicit_artist_for_pp - - pp = { - 'id_tags': id_tags, - 'track_title': track_title, - 'artist_name': artist_name, - 'batch_artist_name': _batch_artist_name, - 'metadata': metadata, - 'recording_mbid': None, - 'artist_mbid': None, - 'release_mbid': '', - 'mb_genres': [], - 'isrc': None, - 'deezer_bpm': None, 'deezer_isrc': None, - 'audiodb_mood': None, 'audiodb_style': None, 'audiodb_genre': None, - 'tidal_isrc': None, 'tidal_copyright': None, - 'qobuz_isrc': None, 'qobuz_copyright': None, 'qobuz_label': None, - 'lastfm_tags': [], 'lastfm_url': None, - 'genius_url': None, - 'release_year': None, # First source to find a year wins - } - - # Run each metadata source lookup in configured order - _pp_source_order = config_manager.get('metadata_enhancement.post_process_order', None) - if not _pp_source_order or not isinstance(_pp_source_order, list): - _pp_source_order = ['musicbrainz', 'deezer', 'audiodb', 'tidal', 'qobuz', 'lastfm', 'genius'] - - _pp_lookup_map = { - 'musicbrainz': _pp_lookup_musicbrainz, - 'deezer': _pp_lookup_deezer, - 'audiodb': _pp_lookup_audiodb, - 'tidal': _pp_lookup_tidal, - 'qobuz': _pp_lookup_qobuz, - 'lastfm': _pp_lookup_lastfm, - 'genius': _pp_lookup_genius, - } - - for source_name in _pp_source_order: - fn = _pp_lookup_map.get(source_name) - if fn: - fn(pp, _names_match) - - # Extract results from shared context after all lookups - recording_mbid = pp['recording_mbid'] - artist_mbid = pp['artist_mbid'] - _rc_mbid = pp['release_mbid'] - mb_genres = pp['mb_genres'] - isrc = pp['isrc'] - deezer_bpm = pp['deezer_bpm'] - deezer_isrc = pp['deezer_isrc'] - audiodb_mood = pp['audiodb_mood'] - audiodb_style = pp['audiodb_style'] - audiodb_genre = pp['audiodb_genre'] - tidal_isrc = pp['tidal_isrc'] - tidal_copyright = pp['tidal_copyright'] - qobuz_isrc = pp['qobuz_isrc'] - qobuz_copyright = pp['qobuz_copyright'] - qobuz_label = pp['qobuz_label'] - lastfm_tags = pp['lastfm_tags'] - lastfm_url = pp['lastfm_url'] - genius_url = pp['genius_url'] - id_tags = pp['id_tags'] - release_year = pp['release_year'] - - # If metadata already has a date from Spotify context, use that as fallback - if not release_year and metadata.get('date'): - yr = str(metadata['date'])[:4] - if yr.isdigit(): - release_year = yr - - # Store release MBID in metadata for downstream use (e.g. Cover Art Archive) - if _rc_mbid: - metadata['musicbrainz_release_id'] = _rc_mbid - - # Write release year to file tags if not already present - if release_year and 'ORIGINALDATE' not in id_tags: - id_tags['ORIGINALDATE'] = release_year - # If the file was written without a date tag, flag it for writing below - _needs_date_tag = release_year and not metadata.get('date') - if _needs_date_tag: - metadata['date'] = release_year - - # Update DB album year if currently missing - if release_year: - try: - _pp_album_name = metadata.get('album', '') - _pp_artist_name = metadata.get('album_artist', '') or metadata.get('artist', '') - if _pp_album_name and _pp_artist_name: - conn = get_database()._get_connection() - try: - cursor = conn.cursor() - cursor.execute(""" - UPDATE albums SET year = ? - WHERE (year IS NULL OR year = 0) - AND id IN ( - SELECT al.id FROM albums al - JOIN artists ar ON ar.id = al.artist_id - WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) - ) - """, (int(release_year), _pp_album_name, _pp_artist_name)) - if cursor.rowcount > 0: - conn.commit() - logger.info(f"Updated album year to {release_year} in database") - else: - conn.rollback() - finally: - conn.close() - except Exception as e: - logger.error(f"Could not update album year in DB: {e}") - - # (All source lookups now handled by _pp_lookup_* functions called via configurable order above) - if False: # Dead code — old inline blocks preserved for reference during transition - try: - mb_service_for_detail = mb_worker.mb_service if mb_worker else None - if mb_service_for_detail: - with _mb_release_detail_cache_lock: - release_detail = _mb_release_detail_cache.get(_rc_mbid) - if release_detail is None: - release_detail = mb_service_for_detail.mb_client.get_release( - _rc_mbid, includes=['release-groups', 'labels', 'media', 'artist-credits', 'recordings'] - ) or {} - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[_rc_mbid] = release_detail - if release_detail: - rg = release_detail.get('release-group', {}) - if rg.get('id'): - id_tags['MUSICBRAINZ_RELEASEGROUPID'] = rg['id'] - ac = release_detail.get('artist-credit', []) - if ac and isinstance(ac[0], dict): - aa_artist = ac[0].get('artist', {}) - if aa_artist.get('id'): - id_tags['MUSICBRAINZ_ALBUMARTISTID'] = aa_artist['id'] - primary_type = rg.get('primary-type', '') - if primary_type: - id_tags['RELEASETYPE'] = primary_type - orig_date = rg.get('first-release-date', '') - if orig_date: - id_tags['ORIGINALDATE'] = orig_date - status = release_detail.get('status', '') - if status: - id_tags['RELEASESTATUS'] = status - country = release_detail.get('country', '') - if country: - id_tags['RELEASECOUNTRY'] = country - barcode = release_detail.get('barcode', '') - if barcode: - id_tags['BARCODE'] = barcode - media_list = release_detail.get('media', []) - if media_list: - media_format = media_list[0].get('format', '') - if media_format: - id_tags['MEDIA'] = media_format - id_tags['TOTALDISCS'] = str(len(media_list)) - label_info = release_detail.get('label-info', []) - if label_info and isinstance(label_info[0], dict): - cat_num = label_info[0].get('catalog-number', '') - if cat_num: - id_tags['CATALOGNUMBER'] = cat_num - text_rep = release_detail.get('text-representation', {}) - if isinstance(text_rep, dict) and text_rep.get('script'): - id_tags['SCRIPT'] = text_rep['script'] - asin = release_detail.get('asin', '') - if asin: - id_tags['ASIN'] = asin - # Release Track ID — match by disc + track position - _trk_num = metadata.get('track_number') - _disc_num = metadata.get('disc_number') or 1 - if _trk_num and media_list: - try: - _trk_num_int = int(_trk_num) - _disc_num_int = int(_disc_num) - for medium in media_list: - if medium.get('position', 1) == _disc_num_int: - for mtrack in (medium.get('tracks') or medium.get('track-list', [])): - if mtrack.get('position') == _trk_num_int and mtrack.get('id'): - id_tags['MUSICBRAINZ_RELEASETRACKID'] = mtrack['id'] - break - break - except (ValueError, TypeError): - pass - logger.info(f"MusicBrainz release details: type={primary_type or '?'}, " - f"country={country or '?'}, media={id_tags.get('MEDIA', '?')}") - except Exception as e: - logger.error(f"MusicBrainz release detail lookup failed (non-fatal): {e}") - - # ── 2b. Deezer lookup for BPM, ISRC fallback, and source IDs ── - deezer_bpm = None - deezer_isrc = None - if not config_manager.get('deezer.embed_tags', True): - pass - elif track_title and artist_name: - try: - dz_client = deezer_worker.client if deezer_worker else None - if dz_client: - dz_result = dz_client.search_track(artist_name, track_title) - if dz_result and _names_match(dz_result.get('title', ''), track_title) and \ - _names_match(dz_result.get('artist', {}).get('name', ''), artist_name): - dz_track_id = dz_result['id'] - id_tags['DEEZER_TRACK_ID'] = str(dz_track_id) - dz_artist_id = dz_result.get('artist', {}).get('id') - if dz_artist_id: - id_tags['DEEZER_ARTIST_ID'] = str(dz_artist_id) - logger.info(f"Deezer track matched: {dz_track_id}") - - # Get full track details for BPM and ISRC - dz_details = dz_client.get_track_details(dz_track_id) - if dz_details: - bpm_val = dz_details.get('bpm') - if bpm_val and bpm_val > 0: - deezer_bpm = bpm_val - dz_isrc = dz_details.get('isrc') - if dz_isrc: - deezer_isrc = dz_isrc - else: - logger.info("Deezer worker not available, skipping Deezer lookup") - except Exception as e: - logger.error(f"Deezer lookup failed (non-fatal): {e}") - - # ── 2c. AudioDB lookup for mood, style, genre, and source ID ── - audiodb_mood = None - audiodb_style = None - audiodb_genre = None - if not config_manager.get('audiodb.embed_tags', True): - pass - elif track_title and artist_name: - try: - adb_client = audiodb_worker.client if audiodb_worker else None - if adb_client: - adb_result = adb_client.search_track(artist_name, track_title) - if adb_result and _names_match(adb_result.get('strTrack', ''), track_title) and \ - _names_match(adb_result.get('strArtist', ''), artist_name): - adb_track_id = adb_result.get('idTrack') - if adb_track_id: - id_tags['AUDIODB_TRACK_ID'] = str(adb_track_id) - logger.info(f"AudioDB track matched: {adb_track_id}") - # Use AudioDB's MusicBrainz IDs as fallbacks for any missing from MB lookup - adb_mb_track = adb_result.get('strMusicBrainzID') - if adb_mb_track and 'MUSICBRAINZ_RECORDING_ID' not in id_tags: - id_tags['MUSICBRAINZ_RECORDING_ID'] = adb_mb_track - recording_mbid = adb_mb_track - logger.warning(f"MusicBrainz recording ID from AudioDB fallback: {adb_mb_track}") - # NOTE: AudioDB's strMusicBrainzAlbumID is intentionally - # NOT used as a fallback for MUSICBRAINZ_RELEASE_ID. - # AudioDB links each track to its original album in MB, - # which differs per track on compilations and splits - # albums in players like Navidrome. Album MBID must come - # from match_release (cached) to stay consistent. - adb_mb_artist = adb_result.get('strMusicBrainzArtistID') - if adb_mb_artist and 'MUSICBRAINZ_ARTIST_ID' not in id_tags: - id_tags['MUSICBRAINZ_ARTIST_ID'] = adb_mb_artist - artist_mbid = adb_mb_artist - logger.warning(f"MusicBrainz artist ID from AudioDB fallback: {adb_mb_artist}") - audiodb_mood = adb_result.get('strMood') or None - audiodb_style = adb_result.get('strStyle') or None - audiodb_genre = adb_result.get('strGenre') or None - else: - logger.info("AudioDB worker not available, skipping AudioDB lookup") - except Exception as e: - logger.error(f"AudioDB lookup failed (non-fatal): {e}") - - # ── 2d. Tidal lookup for ISRC fallback, copyright, and source IDs ── - tidal_isrc = None - tidal_copyright = None - if not config_manager.get('tidal.embed_tags', True): - pass - elif track_title and artist_name: - try: - if tidal_client and tidal_client.is_authenticated(): - td_result = tidal_client.search_track(artist_name, track_title) - if td_result and _names_match(td_result.get('title', ''), track_title): - td_track_id = td_result.get('id') - if td_track_id: - id_tags['TIDAL_TRACK_ID'] = str(td_track_id) - logger.info(f"Tidal track matched: {td_track_id}") - td_artist = td_result.get('artist', {}) - if isinstance(td_artist, dict) and td_artist.get('id'): - id_tags['TIDAL_ARTIST_ID'] = str(td_artist['id']) - # Get full details for ISRC and copyright - if td_track_id: - td_details = tidal_client.get_track(str(td_track_id)) - if td_details: - td_isrc = td_details.get('isrc') - if td_isrc: - tidal_isrc = td_isrc - td_copyright = td_details.get('copyright') - if isinstance(td_copyright, dict): - td_copyright = td_copyright.get('text', td_copyright.get('name', '')) - if td_copyright: - tidal_copyright = td_copyright - except Exception as e: - logger.error(f"Tidal lookup failed (non-fatal): {e}") - - # ── 2e. Qobuz lookup for ISRC fallback, copyright, label, and source IDs ── - qobuz_isrc = None - qobuz_copyright = None - qobuz_label = None - if not config_manager.get('qobuz.embed_tags', True): - pass - elif track_title and artist_name: - try: - qz_client = qobuz_enrichment_worker.client if qobuz_enrichment_worker else None - if qz_client and qz_client.is_authenticated(): - qz_result = qz_client.search_track(artist_name, track_title) - if qz_result: - qz_performer = (qz_result.get('performer') or {}) - if not isinstance(qz_performer, dict): - qz_performer = {} - qz_artist_name = qz_performer.get('name', '') - if _names_match(qz_result.get('title', ''), track_title) and \ - _names_match(qz_artist_name, artist_name): - qz_track_id = qz_result.get('id') - if qz_track_id: - id_tags['QOBUZ_TRACK_ID'] = str(qz_track_id) - logger.info(f"Qobuz track matched: {qz_track_id}") - if isinstance(qz_performer, dict) and qz_performer.get('id'): - id_tags['QOBUZ_ARTIST_ID'] = str(qz_performer['id']) - qz_isrc = qz_result.get('isrc') - if isinstance(qz_isrc, dict): - qz_isrc = qz_isrc.get('value', qz_isrc.get('id', '')) - if qz_isrc: - qobuz_isrc = qz_isrc - qz_copyright = qz_result.get('copyright') - if isinstance(qz_copyright, dict): - qz_copyright = qz_copyright.get('text', qz_copyright.get('name', '')) - if qz_copyright and isinstance(qz_copyright, str): - qobuz_copyright = qz_copyright - qz_album = qz_result.get('album', {}) - if isinstance(qz_album, dict): - qz_label_info = qz_album.get('label', {}) - if isinstance(qz_label_info, dict) and qz_label_info.get('name'): - qobuz_label = qz_label_info['name'] - except Exception as e: - logger.error(f"Qobuz lookup failed (non-fatal): {e}") - - # ── 2f. Last.fm lookup for tags (genre merge) and URL ── - lastfm_tags = [] - lastfm_url = None - if not config_manager.get('lastfm.embed_tags', True): - pass - elif track_title and artist_name: - try: - lf_client = lastfm_worker.client if lastfm_worker else None - if lf_client: - lf_result = lf_client.get_track_info(artist_name, track_title) - if lf_result: - lf_url = lf_result.get('url') - if lf_url: - lastfm_url = lf_url - lf_toptags = lf_result.get('toptags', {}) - if isinstance(lf_toptags, dict): - tag_list = lf_toptags.get('tag', []) - if isinstance(tag_list, list): - lastfm_tags = [t.get('name', '') for t in tag_list if isinstance(t, dict) and t.get('name')] - elif isinstance(tag_list, dict) and tag_list.get('name'): - lastfm_tags = [tag_list['name']] - logger.info(f"Last.fm track info found: {len(lastfm_tags)} tags") - except Exception as e: - logger.error(f"Last.fm lookup failed (non-fatal): {e}") - - # ── 2g. Genius lookup for source ID and URL ── - # Genius has an aggressive global rate limiter (30→60→120s backoff) that - # blocks ALL callers including post-processing. We check the backoff - # state directly and skip immediately if Genius is rate-limited, rather - # than entering search_song which would sleep for up to 120s. - genius_url = None - if not config_manager.get('genius.embed_tags', True): - pass - elif track_title and artist_name: - try: - import core.genius_client as _genius_module - if time.time() < _genius_module._rate_limit_until: - logger.info("Genius rate-limited, skipping (non-blocking)") - else: - g_client = genius_worker.client if genius_worker else None - if g_client: - g_result = g_client.search_song(artist_name, track_title) - if g_result: - g_id = g_result.get('id') - if g_id: - id_tags['GENIUS_TRACK_ID'] = str(g_id) - logger.info(f"Genius song matched: {g_id}") - g_url = g_result.get('url') - if g_url: - genius_url = g_url - except Exception as e: - logger.error(f"Genius lookup failed (non-fatal): {e}") - - if not id_tags and not deezer_bpm and not deezer_isrc and not audiodb_mood and not audiodb_style: - return - - # ── 3. Filter tags by per-tag config, then write ── - filtered_tags = {} - for tag_name, value in id_tags.items(): - config_path = _TAG_CONFIG.get(tag_name) - if config_path and not _tag_enabled(config_path): - continue - filtered_tags[tag_name] = value - - # ── 3a. Write ID tags (MusicBrainz, source IDs, release info) ── - written = [] - - # Format-specific tag name mappings for Picard-compatible output - _ID3_TAG_MAP = { - 'MUSICBRAINZ_RECORDING_ID': ('UFID', 'http://musicbrainz.org'), - 'MUSICBRAINZ_ARTIST_ID': ('TXXX', 'MusicBrainz Artist Id'), - 'MUSICBRAINZ_RELEASE_ID': ('TXXX', 'MusicBrainz Album Id'), - 'MUSICBRAINZ_RELEASEGROUPID': ('TXXX', 'MusicBrainz Release Group Id'), - 'MUSICBRAINZ_ALBUMARTISTID': ('TXXX', 'MusicBrainz Album Artist Id'), - 'MUSICBRAINZ_RELEASETRACKID': ('TXXX', 'MusicBrainz Release Track Id'), - 'RELEASETYPE': ('TXXX', 'MusicBrainz Album Type'), - 'RELEASESTATUS': ('TXXX', 'MusicBrainz Album Status'), - 'RELEASECOUNTRY': ('TXXX', 'MusicBrainz Album Release Country'), - 'ORIGINALDATE': ('TDOR', None), - 'MEDIA': ('TMED', None), - } - _VORBIS_TAG_MAP = { - 'MUSICBRAINZ_RECORDING_ID': 'MUSICBRAINZ_TRACKID', - 'MUSICBRAINZ_ARTIST_ID': 'MUSICBRAINZ_ARTISTID', - 'MUSICBRAINZ_RELEASE_ID': 'MUSICBRAINZ_ALBUMID', - 'MUSICBRAINZ_RELEASEGROUPID': 'MUSICBRAINZ_RELEASEGROUPID', - 'MUSICBRAINZ_ALBUMARTISTID': 'MUSICBRAINZ_ALBUMARTISTID', - 'MUSICBRAINZ_RELEASETRACKID': 'MUSICBRAINZ_RELEASETRACKID', - } - _MP4_TAG_MAP = { - 'MUSICBRAINZ_RECORDING_ID': 'MusicBrainz Track Id', - 'MUSICBRAINZ_ARTIST_ID': 'MusicBrainz Artist Id', - 'MUSICBRAINZ_RELEASE_ID': 'MusicBrainz Album Id', - 'MUSICBRAINZ_RELEASEGROUPID': 'MusicBrainz Release Group Id', - 'MUSICBRAINZ_ALBUMARTISTID': 'MusicBrainz Album Artist Id', - 'MUSICBRAINZ_RELEASETRACKID': 'MusicBrainz Release Track Id', - 'RELEASETYPE': 'MusicBrainz Album Type', - 'RELEASESTATUS': 'MusicBrainz Album Status', - 'RELEASECOUNTRY': 'MusicBrainz Album Release Country', - } - - # MP3 (ID3) - if isinstance(audio_file.tags, ID3): - for tag_name, value in filtered_tags.items(): - id3_spec = _ID3_TAG_MAP.get(tag_name) - if id3_spec: - frame_type, desc = id3_spec - if frame_type == 'UFID': - audio_file.tags.add(UFID(owner=desc, data=value.encode('ascii'))) - written.append(f'UFID:{desc}') - elif frame_type == 'TDOR': - audio_file.tags.add(TDOR(encoding=3, text=[value])) - written.append('TDOR') - elif frame_type == 'TMED': - audio_file.tags.add(TMED(encoding=3, text=[value])) - written.append('TMED') - else: # TXXX - audio_file.tags.add(TXXX(encoding=3, desc=desc, text=[value])) - written.append(f'TXXX:{desc}') - else: - audio_file.tags.add(TXXX(encoding=3, desc=tag_name, text=[str(value)])) - written.append(f'TXXX:{tag_name}') - - # FLAC / OGG Vorbis - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - for tag_name, value in filtered_tags.items(): - vorbis_key = _VORBIS_TAG_MAP.get(tag_name, tag_name) - audio_file[vorbis_key] = [str(value)] - written.append(vorbis_key) - - # MP4 (M4A/AAC) - elif isinstance(audio_file, MP4): - for tag_name, value in filtered_tags.items(): - mp4_desc = _MP4_TAG_MAP.get(tag_name, tag_name) - key = f'----:com.apple.iTunes:{mp4_desc}' - audio_file[key] = [MP4FreeForm(str(value).encode('utf-8'))] - written.append(key) - - if written: - logger.info(f"Embedded IDs: {', '.join(written)}") - - # ── 3a½. Write date tag if discovered during lookups (initial write had no date) ── - if _needs_date_tag and release_year: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TDRC(encoding=3, text=[release_year])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['date'] = [release_year] - elif isinstance(audio_file, MP4): - audio_file['\xa9day'] = [release_year] - logger.info(f"Date tag: {release_year}") - - # ── 3b. Write BPM tag (from Deezer) ── - if _tag_enabled('deezer.tags.bpm') and deezer_bpm and deezer_bpm > 0: - bpm_int = int(deezer_bpm) - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TBPM(encoding=3, text=[str(bpm_int)])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['BPM'] = [str(bpm_int)] - elif isinstance(audio_file, MP4): - audio_file['tmpo'] = [bpm_int] - logger.info(f"BPM: {bpm_int}") - - # ── 3c. Write mood tag (from AudioDB) ── - if _tag_enabled('audiodb.tags.mood') and audiodb_mood: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='MOOD', text=[audiodb_mood])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['MOOD'] = [audiodb_mood] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:MOOD'] = [MP4FreeForm(audiodb_mood.encode('utf-8'))] - logger.info(f"Mood: {audiodb_mood}") - - # ── 3d. Write style tag (from AudioDB) ── - if _tag_enabled('audiodb.tags.style') and audiodb_style: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='STYLE', text=[audiodb_style])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['STYLE'] = [audiodb_style] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:STYLE'] = [MP4FreeForm(audiodb_style.encode('utf-8'))] - logger.info(f"Style: {audiodb_style}") - - # ── 4. Merge genres (Spotify + MusicBrainz + AudioDB + Last.fm) and overwrite tag ── - if _tag_enabled('metadata_enhancement.tags.genre_merge'): - enrichment_genres = (mb_genres if _tag_enabled('musicbrainz.tags.genres') else []) + \ - ([audiodb_genre] if audiodb_genre and _tag_enabled('audiodb.tags.genre') else []) + \ - (lastfm_tags if _tag_enabled('lastfm.tags.genres') else []) - if enrichment_genres: - from core.genre_filter import filter_genres as _gf - enrichment_genres = _gf(enrichment_genres, config_manager) - spotify_genres = [g.strip() for g in metadata.get('genre', '').split(',') if g.strip()] - seen = set() - merged = [] - for g in spotify_genres + enrichment_genres: - key = g.strip().lower() - if key and key not in seen: - seen.add(key) - merged.append(g.strip().title()) - if len(merged) >= 5: - break - - if merged: - genre_string = ', '.join(merged) - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TCON(encoding=3, text=[genre_string])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['GENRE'] = [genre_string] - elif isinstance(audio_file, MP4): - audio_file['\xa9gen'] = [genre_string] - logger.info(f"Genres merged: {genre_string}") - - # ── 5. Write ISRC if available (per-source fallback chain) ── - _isrc_candidates = [] - if isrc and _tag_enabled('musicbrainz.tags.isrc'): - _isrc_candidates.append(('MusicBrainz', isrc)) - if deezer_isrc and _tag_enabled('deezer.tags.isrc'): - _isrc_candidates.append(('Deezer', deezer_isrc)) - if tidal_isrc and _tag_enabled('tidal.tags.isrc'): - _isrc_candidates.append(('Tidal', tidal_isrc)) - if qobuz_isrc and _tag_enabled('qobuz.tags.isrc'): - _isrc_candidates.append(('Qobuz', qobuz_isrc)) - if _isrc_candidates: - source, final_isrc = _isrc_candidates[0] - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TSRC(encoding=3, text=[final_isrc])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['ISRC'] = [final_isrc] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:ISRC'] = [MP4FreeForm(final_isrc.encode('utf-8'))] - logger.info(f"ISRC ({source}): {final_isrc}") - - # ── 6. Write copyright tag (Tidal → Qobuz fallback) ── - _copyright_candidates = [] - if tidal_copyright and _tag_enabled('tidal.tags.copyright'): - _copyright_candidates.append(('Tidal', tidal_copyright)) - if qobuz_copyright and _tag_enabled('qobuz.tags.copyright'): - _copyright_candidates.append(('Qobuz', qobuz_copyright)) - if _copyright_candidates: - source, final_copyright = _copyright_candidates[0] - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TCOP(encoding=3, text=[final_copyright])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['COPYRIGHT'] = [final_copyright] - elif isinstance(audio_file, MP4): - audio_file['cprt'] = [final_copyright] - logger.info(f"©️ Copyright ({source}): {final_copyright[:60]}") - - # ── 7. Write label/publisher tag (from Qobuz) ── - if _tag_enabled('qobuz.tags.label') and qobuz_label: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TPUB(encoding=3, text=[qobuz_label])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['LABEL'] = [qobuz_label] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:LABEL'] = [MP4FreeForm(qobuz_label.encode('utf-8'))] - logger.info(f"Label (Qobuz): {qobuz_label}") - - # ── 8. Write Last.fm and Genius URLs as custom tags ── - if _tag_enabled('lastfm.tags.url') and lastfm_url: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='LASTFM_URL', text=[lastfm_url])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['LASTFM_URL'] = [lastfm_url] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:LASTFM_URL'] = [MP4FreeForm(lastfm_url.encode('utf-8'))] - - if _tag_enabled('genius.tags.url') and genius_url: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='GENIUS_URL', text=[genius_url])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['GENIUS_URL'] = [genius_url] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:GENIUS_URL'] = [MP4FreeForm(genius_url.encode('utf-8'))] - - except Exception as e: - logger.error(f"Error embedding source IDs (non-fatal): {e}") - def _download_cover_art(album_info: dict, target_dir: str, context: dict = None): - """Downloads cover.jpg into the specified directory. - Tries Cover Art Archive first (high-res) if MusicBrainz release ID is available - (set by _enhance_file_metadata during post-processing), falls back to source URL. - Accepts optional context to extract image URL from spotify_album when album_info lacks it.""" - if not config_manager.get('metadata_enhancement.cover_art_download', True): - return - try: - cover_path = os.path.join(target_dir, "cover.jpg") - release_mbid = album_info.get('musicbrainz_release_id') if album_info else None - prefer_caa = config_manager.get('metadata_enhancement.prefer_caa_art', False) - - # If cover.jpg exists but we now have a CAA MBID, check if we should upgrade - if os.path.exists(cover_path): - if release_mbid and prefer_caa: - try: - existing_size = os.path.getsize(cover_path) - # Typical Spotify/iTunes cover is ~50-150KB at 640x640 - # CAA covers are usually 300KB+ at 1200x1200 - if existing_size > 200_000: - return # Already high-res, skip - # Low-res cover exists — try to upgrade from CAA - is_upgrade = True - logger.info(f"Existing cover.jpg is {existing_size // 1024}KB — attempting CAA upgrade...") - except Exception: - return - else: - return - else: - is_upgrade = False - - image_data = None - - # Try Cover Art Archive first (often 1200x1200+, original quality) — opt-in - # The MBID is stored in album_info by _enhance_file_metadata before this is called - if release_mbid and prefer_caa: - try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" - req = urllib.request.Request(caa_url, headers={'Accept': 'image/*'}) - with urllib.request.urlopen(req, timeout=10) as response: - image_data = response.read() - if image_data and len(image_data) > 1000: - logger.info(f"Cover art from Cover Art Archive ({len(image_data) // 1024}KB)") - else: - image_data = None - except Exception: - image_data = None - - # If upgrading and CAA failed, keep existing cover — don't overwrite with same low-res - if is_upgrade and not image_data: - logger.error("CAA upgrade failed — keeping existing cover.jpg") - return - - # Fallback to Spotify/iTunes/Deezer URL - if not image_data: - art_url = album_info.get('album_image_url') - # If album_info lacks the URL, try the context's spotify_album - if not art_url and context: - spotify_album = context.get('spotify_album') or {} - art_url = spotify_album.get('image_url') - # Also try images array (raw Spotify API format) - if not art_url: - images = spotify_album.get('images', []) - if images and isinstance(images, list) and len(images) > 0: - art_url = images[0].get('url') if isinstance(images[0], dict) else None - if art_url: - logger.info("Using cover art URL from spotify_album context") - # Upgrade to highest available resolution before fetching - if art_url and 'i.scdn.co' in art_url: - from core.spotify_client import _upgrade_spotify_image_url - art_url = _upgrade_spotify_image_url(art_url) - elif art_url and 'mzstatic.com' in art_url: - import re as _re - art_url = _re.sub(r'\d+x\d+bb', '3000x3000bb', art_url) - if not art_url: - logger.warning("No cover art URL available for download.") - return - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - - if not image_data: - return - - with open(cover_path, 'wb') as f: - f.write(image_data) - - logger.info(f"Cover art downloaded to: {cover_path}") - except Exception as e: - logger.error(f"Error downloading cover.jpg: {e}") - - - + return metadata_enrichment.download_cover_art( + album_info, + target_dir, + context, + ) def _get_spotify_album_tracks(spotify_album: dict) -> list: """Fetches all tracks for a given Spotify album ID.""" @@ -20568,695 +18192,29 @@ def _match_track_to_spotify_title(slsk_track_meta: dict, spotify_tracks: list) - return slsk_track_meta # Fallback to original - -# --- Post-Processing Logic --- -# ── Modular post-processing metadata lookup functions ── -# Each function receives a shared `pp` context dict and writes its results to it. -# The orchestrator in _post_process_matched_download calls them in configurable order. - -def _pp_lookup_musicbrainz(pp, _names_match): - """MusicBrainz: recording MBID, artist MBID, release MBID, ISRC, genres, release details.""" - if not config_manager.get('musicbrainz.embed_tags', True): - return - track_title = pp['track_title'] - artist_name = pp['artist_name'] - metadata = pp['metadata'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - mb_service = mb_worker.mb_service if mb_worker else None - if not mb_service: - logger.info("MusicBrainz worker not available, skipping MBID lookup") - return - result = mb_service.match_recording(track_title, artist_name) - if result and result.get('mbid'): - pp['recording_mbid'] = result['mbid'] - id_tags['MUSICBRAINZ_RECORDING_ID'] = pp['recording_mbid'] - logger.info(f"MusicBrainz recording matched: {pp['recording_mbid']}") - details = mb_service.mb_client.get_recording(pp['recording_mbid'], includes=['isrcs', 'genres']) - if details: - isrcs = details.get('isrcs', []) - if isrcs: - pp['isrc'] = isrcs[0] - pp['mb_genres'] = [g['name'] for g in sorted(details.get('genres', []), key=lambda x: x.get('count', 0), reverse=True)] - - track_artist_name = metadata.get('artist', '') or artist_name - if ', ' in track_artist_name: - track_artist_name = track_artist_name.split(', ')[0] - artist_result = mb_service.match_artist(track_artist_name) - if artist_result and artist_result.get('mbid'): - pp['artist_mbid'] = artist_result['mbid'] - id_tags['MUSICBRAINZ_ARTIST_ID'] = pp['artist_mbid'] - - album_name_for_mb = metadata.get('album', '') - if album_name_for_mb: - # Use normalized key (strips edition suffixes) so "Album (Deluxe Edition)" - # and "Album (Deluxe)" and "Album" all share the same cached MBID. - # This prevents Navidrome from splitting one album into multiple entries. - # Prefer batch-level artist name (from album download context) for the cache key - # so all tracks in the batch hit the same preflight-cached entry, even if - # per-track metadata spells the artist slightly differently. - _artist_key = (pp.get('batch_artist_name') or artist_name).lower().strip() - _rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), _artist_key) - _rc_key_exact = (album_name_for_mb.lower().strip(), _artist_key) - with _mb_release_cache_lock: - # Check normalized key first (catches edition variants) - cached = _mb_release_cache.get(_rc_key_norm) - if cached is None: - cached = _mb_release_cache.get(_rc_key_exact) - if cached is not None: - pp['release_mbid'] = cached - else: - try: - _rc_result = mb_service.match_release(album_name_for_mb, artist_name) - pp['release_mbid'] = _rc_result.get('mbid', '') if _rc_result else '' - except Exception: - pp['release_mbid'] = '' - # Cache under both normalized and exact keys - _mb_release_cache[_rc_key_norm] = pp['release_mbid'] - _mb_release_cache[_rc_key_exact] = pp['release_mbid'] - if pp['release_mbid']: - id_tags['MUSICBRAINZ_RELEASE_ID'] = pp['release_mbid'] - - # Release details (group, barcode, media, etc.) - if pp['release_mbid']: - with _mb_release_detail_cache_lock: - release_detail = _mb_release_detail_cache.get(pp['release_mbid']) - if release_detail is None: - release_detail = mb_service.mb_client.get_release( - pp['release_mbid'], includes=['release-groups', 'labels', 'media', 'artist-credits', 'recordings'] - ) or {} - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[pp['release_mbid']] = release_detail - if release_detail: - rg = release_detail.get('release-group', {}) - if rg.get('id'): id_tags['MUSICBRAINZ_RELEASEGROUPID'] = rg['id'] - ac = release_detail.get('artist-credit', []) - if ac and isinstance(ac[0], dict): - aa = ac[0].get('artist', {}) - if aa.get('id'): id_tags['MUSICBRAINZ_ALBUMARTISTID'] = aa['id'] - if rg.get('primary-type'): id_tags['RELEASETYPE'] = rg['primary-type'] - if rg.get('first-release-date'): - id_tags['ORIGINALDATE'] = rg['first-release-date'] - if not pp['release_year'] and len(rg['first-release-date']) >= 4: - yr = rg['first-release-date'][:4] - if yr.isdigit(): - pp['release_year'] = yr - if release_detail.get('status'): id_tags['RELEASESTATUS'] = release_detail['status'] - if release_detail.get('country'): id_tags['RELEASECOUNTRY'] = release_detail['country'] - if release_detail.get('barcode'): id_tags['BARCODE'] = release_detail['barcode'] - media_list = release_detail.get('media', []) - if media_list: - fmt = media_list[0].get('format', '') - if fmt: id_tags['MEDIA'] = fmt - id_tags['TOTALDISCS'] = str(len(media_list)) - label_info = release_detail.get('label-info', []) - if label_info and isinstance(label_info[0], dict): - cat = label_info[0].get('catalog-number', '') - if cat: id_tags['CATALOGNUMBER'] = cat - text_rep = release_detail.get('text-representation', {}) - if isinstance(text_rep, dict) and text_rep.get('script'): - id_tags['SCRIPT'] = text_rep['script'] - if release_detail.get('asin'): id_tags['ASIN'] = release_detail['asin'] - # Picard-style: pull recording MBID from the release tracklist - # instead of using the independent match_recording() result. - # This guarantees the recording ID is consistent with the release. - _trk_num = metadata.get('track_number') - _disc_num = metadata.get('disc_number') or 1 - if _trk_num and media_list: - try: - _trk_num_int, _disc_num_int = int(_trk_num), int(_disc_num) - for medium in media_list: - if medium.get('position', 1) == _disc_num_int: - for mtrack in (medium.get('tracks') or medium.get('track-list', [])): - if mtrack.get('position') == _trk_num_int: - if mtrack.get('id'): - id_tags['MUSICBRAINZ_RELEASETRACKID'] = mtrack['id'] - # Override recording MBID with the one from this release's tracklist - _release_recording = mtrack.get('recording', {}) - if _release_recording.get('id'): - pp['recording_mbid'] = _release_recording['id'] - id_tags['MUSICBRAINZ_RECORDING_ID'] = _release_recording['id'] - logger.info(f"MusicBrainz recording from release tracklist: {_release_recording['id']}") - break - break - except (ValueError, TypeError): - pass - except Exception as e: - logger.error(f"MusicBrainz lookup failed (non-fatal): {e}") - - -def _pp_lookup_deezer(pp, _names_match): - """Deezer: BPM, ISRC fallback, track/artist IDs.""" - if not config_manager.get('deezer.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - dz_client = deezer_worker.client if deezer_worker else None - if not dz_client: - logger.info("Deezer worker not available, skipping Deezer lookup") - return - dz_result = dz_client.search_track(artist_name, track_title) - if dz_result and _names_match(dz_result.get('title', ''), track_title) and \ - _names_match(dz_result.get('artist', {}).get('name', ''), artist_name): - dz_track_id = dz_result['id'] - id_tags['DEEZER_TRACK_ID'] = str(dz_track_id) - dz_artist_id = dz_result.get('artist', {}).get('id') - if dz_artist_id: - id_tags['DEEZER_ARTIST_ID'] = str(dz_artist_id) - logger.info(f"Deezer track matched: {dz_track_id}") - dz_details = dz_client.get_track_details(dz_track_id) - if dz_details: - bpm_val = dz_details.get('bpm') - if bpm_val and bpm_val > 0: - pp['deezer_bpm'] = bpm_val - dz_isrc = dz_details.get('isrc') - if dz_isrc: - pp['deezer_isrc'] = dz_isrc - # Release year from Deezer album - if not pp['release_year']: - dz_album = dz_result.get('album', {}) - dz_release = (dz_album.get('release_date', '') if isinstance(dz_album, dict) else '') or '' - if len(dz_release) >= 4 and dz_release[:4].isdigit(): - pp['release_year'] = dz_release[:4] - except Exception as e: - logger.error(f"Deezer lookup failed (non-fatal): {e}") - - -def _pp_lookup_audiodb(pp, _names_match): - """AudioDB: mood, style, genre, track ID, MusicBrainz ID fallbacks.""" - if not config_manager.get('audiodb.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - adb_client = audiodb_worker.client if audiodb_worker else None - if not adb_client: - logger.info("AudioDB worker not available, skipping AudioDB lookup") - return - adb_result = adb_client.search_track(artist_name, track_title) - if adb_result and _names_match(adb_result.get('strTrack', ''), track_title) and \ - _names_match(adb_result.get('strArtist', ''), artist_name): - adb_track_id = adb_result.get('idTrack') - if adb_track_id: - id_tags['AUDIODB_TRACK_ID'] = str(adb_track_id) - logger.info(f"AudioDB track matched: {adb_track_id}") - adb_mb_track = adb_result.get('strMusicBrainzID') - if adb_mb_track and 'MUSICBRAINZ_RECORDING_ID' not in id_tags: - id_tags['MUSICBRAINZ_RECORDING_ID'] = adb_mb_track - pp['recording_mbid'] = adb_mb_track - logger.warning(f"MusicBrainz recording ID from AudioDB fallback: {adb_mb_track}") - adb_mb_artist = adb_result.get('strMusicBrainzArtistID') - if adb_mb_artist and 'MUSICBRAINZ_ARTIST_ID' not in id_tags: - id_tags['MUSICBRAINZ_ARTIST_ID'] = adb_mb_artist - pp['artist_mbid'] = adb_mb_artist - logger.warning(f"MusicBrainz artist ID from AudioDB fallback: {adb_mb_artist}") - pp['audiodb_mood'] = adb_result.get('strMood') or None - pp['audiodb_style'] = adb_result.get('strStyle') or None - pp['audiodb_genre'] = adb_result.get('strGenre') or None - except Exception as e: - logger.error(f"AudioDB lookup failed (non-fatal): {e}") - - -def _pp_lookup_tidal(pp, _names_match): - """Tidal: ISRC fallback, copyright, track/artist IDs.""" - if not config_manager.get('tidal.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - if not (tidal_client and tidal_client.is_authenticated()): - return - td_result = tidal_client.search_track(artist_name, track_title) - if td_result and _names_match(td_result.get('title', ''), track_title): - td_track_id = td_result.get('id') - if td_track_id: - id_tags['TIDAL_TRACK_ID'] = str(td_track_id) - logger.info(f"Tidal track matched: {td_track_id}") - td_artist = td_result.get('artist', {}) - if isinstance(td_artist, dict) and td_artist.get('id'): - id_tags['TIDAL_ARTIST_ID'] = str(td_artist['id']) - if td_track_id: - td_details = tidal_client.get_track(str(td_track_id)) - if td_details: - td_isrc = td_details.get('isrc') - if td_isrc: - pp['tidal_isrc'] = td_isrc - td_copyright = td_details.get('copyright') - if isinstance(td_copyright, dict): - td_copyright = td_copyright.get('text', td_copyright.get('name', '')) - if td_copyright: - pp['tidal_copyright'] = td_copyright - # Release year from Tidal album - if not pp['release_year']: - td_album = td_result.get('album', {}) - td_release = '' - if isinstance(td_album, dict): - td_release = str(td_album.get('release_date', '') or td_album.get('releaseDate', '') or '') - if len(td_release) >= 4 and td_release[:4].isdigit(): - pp['release_year'] = td_release[:4] - except Exception as e: - logger.error(f"Tidal lookup failed (non-fatal): {e}") - - -def _pp_lookup_qobuz(pp, _names_match): - """Qobuz: ISRC fallback, copyright, label, track/artist IDs.""" - if not config_manager.get('qobuz.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - qz_client = qobuz_enrichment_worker.client if qobuz_enrichment_worker else None - if not (qz_client and qz_client.is_authenticated()): - return - qz_result = qz_client.search_track(artist_name, track_title) - if qz_result: - qz_performer = (qz_result.get('performer') or {}) - if not isinstance(qz_performer, dict): - qz_performer = {} - qz_artist_name = qz_performer.get('name', '') - if _names_match(qz_result.get('title', ''), track_title) and \ - _names_match(qz_artist_name, artist_name): - qz_track_id = qz_result.get('id') - if qz_track_id: - id_tags['QOBUZ_TRACK_ID'] = str(qz_track_id) - logger.info(f"Qobuz track matched: {qz_track_id}") - if isinstance(qz_performer, dict) and qz_performer.get('id'): - id_tags['QOBUZ_ARTIST_ID'] = str(qz_performer['id']) - qz_isrc = qz_result.get('isrc') - if isinstance(qz_isrc, dict): - qz_isrc = qz_isrc.get('value', qz_isrc.get('id', '')) - if qz_isrc: - pp['qobuz_isrc'] = qz_isrc - qz_copyright = qz_result.get('copyright') - if isinstance(qz_copyright, dict): - qz_copyright = qz_copyright.get('text', qz_copyright.get('name', '')) - if qz_copyright and isinstance(qz_copyright, str): - pp['qobuz_copyright'] = qz_copyright - qz_album = qz_result.get('album', {}) - if isinstance(qz_album, dict): - qz_label_info = qz_album.get('label', {}) - if isinstance(qz_label_info, dict) and qz_label_info.get('name'): - pp['qobuz_label'] = qz_label_info['name'] - # Release year from Qobuz album (prefer date string over Unix timestamp) - if not pp['release_year']: - qz_release = str(qz_album.get('release_date_original', '') or '') - if not qz_release: - # released_at is a Unix timestamp — convert to year - qz_ts = qz_album.get('released_at') - if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: - import datetime as _dt - qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) - if len(qz_release) >= 4 and qz_release[:4].isdigit(): - pp['release_year'] = qz_release[:4] - except Exception as e: - logger.error(f"Qobuz lookup failed (non-fatal): {e}") - - -def _pp_lookup_lastfm(pp, _names_match): - """Last.fm: genre tags, track URL.""" - if not config_manager.get('lastfm.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - if not track_title or not artist_name: - return - try: - lf_client = lastfm_worker.client if lastfm_worker else None - if not lf_client: - return - lf_result = lf_client.get_track_info(artist_name, track_title) - if lf_result: - lf_url = lf_result.get('url') - if lf_url: - pp['lastfm_url'] = lf_url - lf_toptags = lf_result.get('toptags', {}) - if isinstance(lf_toptags, dict): - tag_list = lf_toptags.get('tag', []) - if isinstance(tag_list, list): - pp['lastfm_tags'] = [t.get('name', '') for t in tag_list if isinstance(t, dict) and t.get('name')] - elif isinstance(tag_list, dict) and tag_list.get('name'): - pp['lastfm_tags'] = [tag_list['name']] - logger.info(f"Last.fm track info found: {len(pp['lastfm_tags'])} tags") - except Exception as e: - logger.error(f"Last.fm lookup failed (non-fatal): {e}") - - -def _pp_lookup_genius(pp, _names_match): - """Genius: track ID, URL.""" - if not config_manager.get('genius.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - import core.genius_client as _genius_module - if time.time() < _genius_module._rate_limit_until: - logger.info("Genius rate-limited, skipping (non-blocking)") - return - g_client = genius_worker.client if genius_worker else None - if not g_client: - return - g_result = g_client.search_song(artist_name, track_title) - if g_result: - g_id = g_result.get('id') - if g_id: - id_tags['GENIUS_TRACK_ID'] = str(g_id) - logger.info(f"Genius song matched: {g_id}") - g_url = g_result.get('url') - if g_url: - pp['genius_url'] = g_url - except Exception as e: - logger.error(f"Genius lookup failed (non-fatal): {e}") - - def _post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id): """ NEW VERIFICATION WORKFLOW: Enhanced post-processing with file verification. Only sets task status to 'completed' after successful file verification and move operation. """ - _pp = pp_logger - try: - # Call the existing post-processing logic (but skip its completion callback) - # We'll handle the completion callback ourselves after verification - original_task_id = context.pop('task_id', None) # Temporarily remove to prevent double callback - original_batch_id = context.pop('batch_id', None) - _post_process_matched_download(context_key, context, file_path) - # Restore the IDs for our own callback - if original_task_id: - context['task_id'] = original_task_id - if original_batch_id: - context['batch_id'] = original_batch_id + from core.import_pipeline import post_process_matched_download_with_verification + return post_process_matched_download_with_verification( + context_key, + context, + file_path, + task_id, + batch_id, + _build_import_pipeline_runtime(), + ) - # Check if race guard detected the source file was already gone - if context.get('_race_guard_failed'): - _pp.info(f"Race guard: source file gone for task {task_id} — marking as failed") - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = 'Source file was already processed or removed by another task' - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - _on_download_completed(batch_id, task_id, success=False) - return - - # Check if AcoustID quarantined the file — no further processing needed - if context.get('_acoustid_quarantined'): - failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed') - _pp.info(f"File was quarantined by AcoustID verification (task={task_id}): {failure_msg}") - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {failure_msg}" - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - _on_download_completed(batch_id, task_id, success=False) - return - - # Check if simple download handler already completed everything - if context.get('_simple_download_completed'): - expected_final_path = context.get('_final_path') - - if expected_final_path and os.path.exists(expected_final_path): - with tasks_lock: - if task_id in download_tasks: - _mark_task_completed(task_id, context.get('track_info')) - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=True) - return - else: - _pp.info(f"FAILED simple download file not found at: {expected_final_path} (task={task_id}, context={context_key})") - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f'Downloaded file not found at expected location: {os.path.basename(expected_final_path)}' - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=False) - return - - # VERIFICATION: Use the actual path that _post_process_matched_download computed and - # moved the file to, instead of recomputing independently. Independent recomputation - # can produce path mismatches (e.g., track_number extraction/validation differences) - # causing false "file verification failed" errors on successfully processed files. - expected_final_path = context.get('_final_processed_path') - if not expected_final_path: - _pp.info(f"No _final_processed_path in context for task {task_id} — cannot verify, assuming success") - with tasks_lock: - if task_id in download_tasks: - _mark_task_completed(task_id, context.get('track_info')) - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - _on_download_completed(batch_id, task_id, success=True) - return - - # VERIFICATION: Check if file exists at the path processing actually used - if os.path.exists(expected_final_path): - # Mark task as completed only after successful verification - redownload_ctx = None - with tasks_lock: - if task_id in download_tasks: - _mark_task_completed(task_id, context.get('track_info')) - download_tasks[task_id]['metadata_enhanced'] = True - redownload_ctx = download_tasks[task_id].get('_redownload_context') - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - # Redownload hook: delete old file and update DB path (outside locks) - if redownload_ctx: - try: - old_path = redownload_ctx.get('old_file_path') - lib_track_id = redownload_ctx.get('library_track_id') - if redownload_ctx.get('delete_old_file') and old_path and os.path.exists(old_path): - if os.path.normpath(old_path) != os.path.normpath(expected_final_path): - os.remove(old_path) - logger.info(f"[Redownload] Deleted old file: {old_path}") - if lib_track_id and expected_final_path: - _rd_db = get_database() - _rd_conn = _rd_db._get_connection() - _rd_cursor = _rd_conn.cursor() - _rd_cursor.execute(""" - UPDATE tracks SET file_path = ?, updated_at = CURRENT_TIMESTAMP - WHERE id = ? - """, (expected_final_path, lib_track_id)) - _rd_conn.commit() - _rd_conn.close() - logger.info(f"[Redownload] Updated DB path for track {lib_track_id}") - except Exception as e: - logger.error(f"[Redownload] Post-processing hook error: {e}") - - _on_download_completed(batch_id, task_id, success=True) - else: - # Log failure details for diagnosis - track_name = context.get('original_search_result', {}).get('spotify_clean_title', context_key) - _pp.info(f"FAILED verification for '{track_name}' (task={task_id})") - _pp.info(f" expected_final_path: {expected_final_path}") - _pp.info(f" file_path (source): {file_path}, exists={os.path.exists(file_path)}") - _pp.info(f" is_album={context.get('is_album_download', False)}, has_clean_data={context.get('has_clean_spotify_data', False)}") - expected_dir = os.path.dirname(expected_final_path) - if os.path.exists(expected_dir): - dir_contents = os.listdir(expected_dir) - _pp.info(f" directory contains {len(dir_contents)} files: {dir_contents[:20]}") - else: - _pp.info(f" directory does not exist: {expected_dir}") - - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f'File verification failed: expected file at {os.path.basename(expected_final_path)} but it was not found after processing' - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=False) - - except Exception as e: - import traceback - _pp.info(f"EXCEPTION in post-processing for '{context_key}' (task={task_id}): {e}") - _pp.info(traceback.format_exc()) - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"Post-processing verification failed: {str(e)}" - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=False) - - -def _check_flac_bit_depth(file_path, context, context_key): - """ - Check if a FLAC file matches the user's preferred bit depth. - Returns True if the file was rejected (caller should return), False if OK. - With fallback enabled (default), accepts any FLAC bit depth rather than quarantining. - """ - if not context.get('_audio_quality', '').startswith('FLAC'): - return False - - from database.music_database import MusicDatabase - _qp_db = MusicDatabase() - _quality_profile = _qp_db.get_quality_profile() - _flac_config = _quality_profile.get('qualities', {}).get('flac', {}) - _flac_pref = _flac_config.get('bit_depth', 'any') - - if _flac_pref == 'any': - return False - - # Parse actual bit depth from quality string like "FLAC 16bit" - _actual_bits = context['_audio_quality'].replace('FLAC ', '').replace('bit', '') - if _actual_bits == _flac_pref: - return False - - # Bit depth doesn't match preference — check if fallback or downsample is enabled - _flac_fallback = _flac_config.get('bit_depth_fallback', True) - _downsample_enabled = config_manager.get('lossy_copy.downsample_hires', False) - - if _flac_fallback or _downsample_enabled: - # Accept the file — it will be downsampled or a FLAC at any bit depth is better than a failed download - track_info = context.get('track_info', {}) - track_name = track_info.get('name', os.path.basename(file_path)) - if _downsample_enabled: - logger.info(f"[FLAC Downsample] Accepted {_actual_bits}-bit FLAC (will be downsampled to {_flac_pref}-bit): {track_name}") - else: - logger.warning(f"[FLAC Fallback] Accepted {_actual_bits}-bit FLAC (preferred {_flac_pref}-bit): {track_name}") - return False - - # Strict mode — reject and quarantine - rejection_msg = f"FLAC bit depth mismatch: file is {_actual_bits}-bit, preference is {_flac_pref}-bit" - try: - quarantine_path = _move_to_quarantine(file_path, context, rejection_msg) - logger.info(f"File quarantined due to bit depth filter: {quarantine_path}") - except Exception as quarantine_error: - logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") - try: - os.remove(file_path) - except Exception: - pass - - context['_bitdepth_rejected'] = True - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id: - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_msg}" - if task_id and batch_id: - _on_download_completed(batch_id, task_id, success=False) - return True - - -def _move_to_quarantine(file_path: str, context: dict, reason: str) -> str: - """ - Move a file to quarantine folder when AcoustID verification fails. - Creates a JSON sidecar file with metadata about why the file was quarantined. - - Args: - file_path: Original file path - context: Download context with track info - reason: Reason for quarantine - - Returns: - Path to quarantined file - """ - import json - from pathlib import Path - from datetime import datetime - - # Get quarantine directory (inside download folder — always writable, even in Docker) - download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - quarantine_dir = Path(download_dir) / "ss_quarantine" - quarantine_dir.mkdir(parents=True, exist_ok=True) - - # Create quarantine entry with timestamp - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - original_name = Path(file_path).stem - file_ext = Path(file_path).suffix - - # Build quarantine filename: TIMESTAMP_originalname.ext.quarantined - # The .quarantined extension prevents audio file searches and media servers - # from picking up known-wrong files sitting in the downloads folder. - quarantine_filename = f"{timestamp}_{original_name}{file_ext}.quarantined" - quarantine_path = quarantine_dir / quarantine_filename - - # Move file to quarantine - _safe_move_file(file_path, str(quarantine_path)) - - # Write metadata sidecar file - metadata_path = quarantine_dir / f"{timestamp}_{original_name}.json" - - # Extract track info from context - track_info = context.get('track_info', {}) - original_search = context.get('original_search_result', {}) - spotify_artist = context.get('spotify_artist', {}) - - metadata = { - 'original_filename': Path(file_path).name, - 'quarantine_reason': reason, - 'timestamp': datetime.now().isoformat(), - 'expected_track': ( - original_search.get('spotify_clean_title') or - track_info.get('name') or - original_search.get('title', 'Unknown') - ), - 'expected_artist': spotify_artist.get('name', 'Unknown'), - 'context_key': context.get('context_key', 'unknown') - } - - try: - with open(metadata_path, 'w', encoding='utf-8') as f: - json.dump(metadata, f, indent=2, ensure_ascii=False) - except Exception as e: - logger.warning(f"Failed to write quarantine metadata: {e}") - - logger.warning(f"File quarantined: {quarantine_path} - Reason: {reason}") - - try: - if automation_engine: - ti = context.get('track_info', {}) - artists = ti.get('artists', []) - artist_name = '' - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - automation_engine.emit('download_quarantined', { - 'artist': artist_name, - 'title': ti.get('name', ''), - 'reason': reason or 'Unknown', - }) - except Exception: - pass - - return str(quarantine_path) +def _build_import_pipeline_runtime(): + """Collect live controller dependencies for the shared import pipeline.""" + return types.SimpleNamespace( + automation_engine=automation_engine, + on_download_completed=_on_download_completed, + web_scan_manager=web_scan_manager, + repair_worker=repair_worker, + ) def _safe_move_file(src, dst): @@ -21362,872 +18320,8 @@ def _post_process_matched_download(context_key, context, file_path): Also handles simple downloads (from search page "Download" button) which just move files to /Transfer without metadata enhancement. """ - # --- PER-FILE LOCK --- - # Acquire a per-context-key lock so only one thread processes a given file at a time. - # The Stream Processor and Verification Worker both call this function for the same file. - # Without serialization, they race to move the source file and the loser gets FileNotFoundError. - # With the lock, the second thread waits, then the existing protection checks detect - # "source gone + destination exists" and return early. - with _post_process_locks_lock: - if context_key not in _post_process_locks: - _post_process_locks[context_key] = threading.Lock() - file_lock = _post_process_locks[context_key] - - file_lock.acquire() - try: - import os - import shutil - import time - from pathlib import Path - - # --- RACE CONDITION GUARD --- - # If source file is already gone, another thread (stream processor or - # verification worker) already processed it. Check immediately after - # acquiring the lock to avoid wasting time on stability checks / acoustid. - if not os.path.exists(file_path): - existing_final = context.get('_final_processed_path') - if existing_final and os.path.exists(existing_final): - logger.info(f"[Race Guard] Source gone but destination exists — already processed by another thread: {os.path.basename(existing_final)}") - return - logger.error(f"[Race Guard] Source file gone and no known destination — marking as failed: {os.path.basename(file_path)}") - context['_race_guard_failed'] = True - return - # --- END RACE CONDITION GUARD --- - - # --- FILE STABILITY CHECK --- - # Wait for the file to stop growing before processing. slskd may still be - # flushing write buffers when it reports "Completed". We poll the file size - # a few times; once it stabilises we know the write is finished. - _basename = os.path.basename(file_path) - _prev_size = -1 - for _stability_check in range(5): - try: - _cur_size = os.path.getsize(file_path) - except OSError: - _cur_size = -1 - if _cur_size == _prev_size and _cur_size > 0: - # Size unchanged — file is stable - break - _prev_size = _cur_size - if _stability_check == 0: - logger.info(f"Waiting for file to stabilise: {_basename} ({_cur_size} bytes)") - time.sleep(1.5) - else: - logger.info(f"File may still be writing after stability checks: {_basename} ({_prev_size} bytes)") - # --- END FILE STABILITY CHECK --- - - # --- ACOUSTID VERIFICATION --- - # Optional verification that downloaded audio matches expected track. - # Only runs if enabled and configured. Fails gracefully (skips on any error). - # Runs for ALL download sources — streaming APIs can return wrong versions - # (live, remix, cover) despite downloading by track ID. - _skip_acoustid = False - - try: - from core.acoustid_verification import AcoustIDVerification, VerificationResult - - verifier = AcoustIDVerification() - available, available_reason = verifier.quick_check_available() - - if available and not _skip_acoustid: - # Extract expected track info from context - track_info = context.get('track_info', {}) - original_search = context.get('original_search_result', {}) - spotify_artist = context.get('spotify_artist', {}) - - expected_track = ( - original_search.get('spotify_clean_title') or - track_info.get('name') or - original_search.get('title', '') - ) - - # Use track-level artist for verification, NOT album artist. - # For compilations, spotify_artist is "Various Artists" which - # will never match AcoustID's actual track artist. - expected_artist = '' - track_artists = track_info.get('artists', []) - if track_artists: - first = track_artists[0] - if isinstance(first, dict): - expected_artist = first.get('name', '') - elif isinstance(first, str): - expected_artist = first - # Fallback to album artist if no track artists available - if not expected_artist: - expected_artist = spotify_artist.get('name', '') - - if expected_track and expected_artist: - logger.info(f"Running AcoustID verification for: '{expected_track}' by '{expected_artist}'") - verification_result, verification_msg = verifier.verify_audio_file( - file_path, - expected_track, - expected_artist, - context - ) - logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}") - context['_acoustid_result'] = verification_result.value - - if verification_result == VerificationResult.FAIL: - # Move to quarantine instead of Transfer - try: - quarantine_path = _move_to_quarantine(file_path, context, verification_msg) - logger.error(f"File quarantined due to verification failure: {quarantine_path}") - except Exception as quarantine_error: - # Quarantine failed — delete the known-wrong file instead - # NEVER save a file we've confirmed is wrong - logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}") - logger.error(f"Quarantine failed, deleting wrong file: {file_path}") - try: - os.remove(file_path) - except Exception as del_error: - logger.error(f"Could not delete wrong file either: {del_error}") - - # These always execute for FAIL — whether quarantine succeeded or not - context['_acoustid_quarantined'] = True - context['_acoustid_failure_msg'] = verification_msg - - # Clean up context - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - # Mark as failed in download tasks if we have task info - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id: - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {verification_msg}" - - # Call completion callback with failure - if task_id and batch_id: - _on_download_completed(batch_id, task_id, success=False) - - return # NEVER continue processing a known-wrong file - else: - logger.warning("AcoustID verification skipped: missing track/artist info") - context['_acoustid_result'] = 'skip' - else: - logger.info(f"ℹ️ AcoustID verification not available: {available_reason}") - context['_acoustid_result'] = 'disabled' - except Exception as verify_error: - # Any verification error should NOT block the download - fail open - logger.error(f"AcoustID verification error (continuing normally): {verify_error}") - context['_acoustid_result'] = 'error' - # --- END ACOUSTID VERIFICATION --- - - # --- SIMPLE DOWNLOAD HANDLING --- - # Check if this is a simple download (search page "Download ⬇" button only) - search_result = context.get('search_result', {}) - is_simple_download = search_result.get('is_simple_download', False) - - if is_simple_download: - # Simple transfer: move to Transfer folder, no metadata enhancement - logger.info(f"Processing simple download (no metadata enhancement): {file_path}") - - transfer_path = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) - - # Check if this download has album info (from path or search result) - album_name = None - original_filename = search_result.get('filename', '') - - if '/' in original_filename or '\\' in original_filename: - # Get parent directory as album name - path_parts = original_filename.replace('\\', '/').split('/') - if len(path_parts) >= 2: - album_name = path_parts[-2] # Parent directory - - # If no album from path, check search result - if not album_name: - album_name = search_result.get('album') - - # Determine destination - filename = Path(file_path).name - - if album_name and album_name.lower() not in ['unknown', 'unknown album', '']: - # Has album info - create album folder - import re - album_name = re.sub(r'[<>:"/\\|?*]', '_', album_name).strip() - album_folder = Path(transfer_path) / album_name - album_folder.mkdir(parents=True, exist_ok=True) - destination = album_folder / filename - logger.info(f"Moving to album folder: {album_name}") - else: - # No album info - move directly to Transfer root (singles) - Path(transfer_path).mkdir(parents=True, exist_ok=True) - destination = Path(transfer_path) / filename - logger.info("Moving to Transfer root (single track)") - - _safe_move_file(file_path, destination) - logger.info(f"Moved simple download to: {destination}") - - # Clean up context - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - # Trigger library scan (using correct method name) - if web_scan_manager: - threading.Thread( - target=lambda: web_scan_manager.request_scan("Simple download completed"), - daemon=True - ).start() - - add_activity_item("", "Download Complete", f"{album_name}/{filename}", "Now") - logger.info(f"Simple download post-processing complete: {album_name}/{filename}") - - # Set flag in context so verification function knows this was fully handled - context['_simple_download_completed'] = True - context['_final_path'] = str(destination) - _emit_track_downloaded(context) - _record_library_history_download(context) - _record_download_provenance(context) - return - # --- END SIMPLE DOWNLOAD HANDLING --- - - logger.info(f"Starting robust post-processing for: {context_key}") - - spotify_artist = context.get("spotify_artist") - if not spotify_artist: - logger.error("Post-processing failed: Missing spotify_artist context.") - return - - # ── UNKNOWN ARTIST GUARD ── - # If artist name is junk, attempt to resolve from track metadata before proceeding. - # This prevents files from landing in "Unknown Artist/" folders. - _junk_artist_names = {'', 'unknown', 'unknown artist', 'various artists', 'none', 'null'} - _artist_name = (spotify_artist.get('name', '') if isinstance(spotify_artist, dict) else '').strip() - if _artist_name.lower() in _junk_artist_names: - logger.info(f"[Unknown Artist Guard] Artist name is '{_artist_name}' — attempting to resolve") - _resolved = False - track_info_guard = context.get("track_info", {}) or {} - original_search_guard = context.get("original_search_result", {}) or {} - - # Try 1: Pull artist from track_info.artists - _ti_artists = track_info_guard.get('artists', []) - if isinstance(_ti_artists, list) and _ti_artists: - _first = _ti_artists[0] - _name = _first.get('name', '') if isinstance(_first, dict) else str(_first) - if _name and _name.strip().lower() not in _junk_artist_names: - spotify_artist['name'] = _name.strip() - logger.info(f"[Unknown Artist Guard] Resolved from track_info.artists: '{_name}'") - _resolved = True - - # Try 2: Pull from original_search_result - if not _resolved: - _os_artist = original_search_guard.get('artist') or original_search_guard.get('artist_name') or '' - if isinstance(_os_artist, str) and _os_artist.strip().lower() not in _junk_artist_names: - spotify_artist['name'] = _os_artist.strip() - logger.info(f"[Unknown Artist Guard] Resolved from original_search_result: '{_os_artist}'") - _resolved = True - - # Try 3: Re-fetch from metadata source using track ID - if not _resolved: - _track_id = track_info_guard.get('id') or track_info_guard.get('track_id') or '' - if _track_id: - try: - _fb_client = _get_metadata_fallback_client() - if hasattr(_fb_client, 'get_track_details'): - _details = _fb_client.get_track_details(str(_track_id)) - if _details and isinstance(_details, dict): - _d_artists = _details.get('artists', []) - if isinstance(_d_artists, list) and _d_artists: - _d_first = _d_artists[0] - _d_name = _d_first.get('name', '') if isinstance(_d_first, dict) else str(_d_first) - if _d_name and _d_name.strip().lower() not in _junk_artist_names: - spotify_artist['name'] = _d_name.strip() - logger.info(f"[Unknown Artist Guard] Resolved from metadata API: '{_d_name}'") - _resolved = True - except Exception as _guard_err: - logger.error(f"[Unknown Artist Guard] Metadata re-fetch failed: {_guard_err}") - - if not _resolved: - logger.error(f"[Unknown Artist Guard] Could not resolve artist — proceeding with '{_artist_name}'") - - context['spotify_artist'] = spotify_artist - # ── END UNKNOWN ARTIST GUARD ── - - # Check if playlist folder mode is enabled (sync page playlists only) - track_info = context.get("track_info", {}) - playlist_folder_mode = track_info.get("_playlist_folder_mode", False) - - logger.debug(f"[Debug] Post-processing - track_info type: {type(track_info)}, is None: {track_info is None}, is empty: {not track_info}") - logger.debug(f"[Debug] Post-processing - playlist_folder_mode: {playlist_folder_mode}") - if track_info: - logger.debug(f"[Debug] Post-processing - track_info keys: {list(track_info.keys())}") - - if playlist_folder_mode: - # Use shared path builder for playlist mode - playlist_name = track_info.get("_playlist_name", "Unknown Playlist") - logger.info(f"[Playlist Folder Mode] Organizing in playlist folder: {playlist_name}") - - file_ext = os.path.splitext(file_path)[1] - - # Build final path FIRST so we can check for already-processed files - final_path, _ = _build_final_path_for_track(context, spotify_artist, None, file_ext) - logger.info(f"Playlist mode final path: '{final_path}'") - - # RACE CONDITION GUARD: If source file is gone but destination exists, - # another thread (stream processor or verification worker) already moved it. - # Return early to avoid deleting the successfully processed file. - if not os.path.exists(file_path): - if os.path.exists(final_path): - logger.info(f"[Playlist Folder Mode] Source gone but destination exists — already processed by another thread: {os.path.basename(final_path)}") - context['_final_processed_path'] = final_path - return - else: - pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: Source file not found and destination does not exist: {file_path}") - raise FileNotFoundError(f"Source file not found and destination does not exist: {file_path}") - - context['_audio_quality'] = _get_audio_quality_string(file_path) - if context['_audio_quality']: - logger.info(f"Audio quality detected: {context['_audio_quality']}") - - # FLAC bit depth filter - if _check_flac_bit_depth(file_path, context, context_key): - return - - # Enhance metadata before moving - try: - logger.warning(f"[Metadata Input] Playlist mode - artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})") - _enhance_file_metadata(file_path, context, spotify_artist, None) - except Exception as meta_err: - import traceback - pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") - _wipe_source_tags(file_path) - - # Move file to playlist folder - logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") - _safe_move_file(file_path, final_path) - - # Store final path for verification wrapper (before conversions may override) - context['_final_processed_path'] = final_path - - # ReplayGain analysis — write track-level gain tags if enabled - if config_manager.get('post_processing.replaygain_enabled', False): - try: - from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF - if _rg_ffmpeg_ok(): - lufs, peak_dbfs = _rg_analyze(final_path) - gain_db = _RG_REF - lufs - _rg_write(final_path, gain_db, peak_dbfs) - pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB — {os.path.basename(final_path)}") - except Exception as rg_err: - pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") - - # Downsample hi-res FLAC to CD quality if enabled (must run before lossy copy) - downsampled_path = _downsample_hires_flac(final_path, context) - if downsampled_path: - final_path = downsampled_path - context['_final_processed_path'] = final_path - - # Lossy copy: create MP3 version if enabled - blasphemy_path = _create_lossy_copy(final_path) - if blasphemy_path: - context['_final_processed_path'] = blasphemy_path - - # Clean up empty directories in downloads folder - downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - _cleanup_empty_directories(downloads_path, file_path) - - logger.info(f"[Playlist Folder Mode] Post-processing complete: {final_path}") - - # WISHLIST REMOVAL: Check if this track should be removed from wishlist - try: - _check_and_remove_from_wishlist(context) - except Exception as wishlist_error: - logger.error(f"[Playlist Folder] Error checking wishlist removal: {wishlist_error}") - - _emit_track_downloaded(context) - _record_library_history_download(context) - _record_download_provenance(context) - - # Mark as stream processed so the verification worker doesn't search - # for the file by its original Soulseek name (which no longer exists after rename) - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id and batch_id: - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['stream_processed'] = True - download_tasks[task_id]['status'] = 'completed' - logger.info(f"[Playlist Folder Mode] Marked task {task_id} as completed") - _on_download_completed(batch_id, task_id, success=True) - - return # Skip normal album/artist folder structure processing - - is_album_download = context.get("is_album_download", False) - has_clean_spotify_data = context.get("has_clean_spotify_data", False) - - if is_album_download and has_clean_spotify_data: - # Build album_info directly from clean Spotify metadata (GUI PARITY) - logger.info("Album context with clean Spotify data found - using direct album info") - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album", {}) - - # Use clean Spotify metadata (matches GUI's SpotifyBasedSearchResult approach) - clean_track_name = original_search.get('spotify_clean_title', 'Unknown Track') - clean_album_name = original_search.get('spotify_clean_album', 'Unknown Album') - - logger.debug( - "Path 1 - Clean Spotify data path: keys=%s has_track_number=%s track_number=%s", - list(original_search.keys()), - 'track_number' in original_search, - original_search.get('track_number', 'NOT_FOUND'), - ) - - album_info = { - 'is_album': True, - 'album_name': clean_album_name, # Use clean Spotify album name - 'track_number': original_search.get('track_number', 1), - 'disc_number': original_search.get('disc_number', 1), - 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album.get('image_url'), - 'confidence': 1.0, # High confidence since we have clean Spotify data - 'source': 'clean_spotify_metadata' - } - - logger.info(f"Using clean Spotify album: '{clean_album_name}' for track: '{clean_track_name}'") - elif is_album_download: - # CRITICAL FIX: Album context without clean Spotify data - still force album treatment - logger.warning("Album context found but no clean Spotify data - using enhanced fallback") - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album", {}) - clean_track_name = original_search.get('spotify_clean_title') or original_search.get('title', 'Unknown Track') - - logger.debug( - "Path 2 - Enhanced fallback album context path: keys=%s has_track_number=%s track_number=%s spotify_album=%s", - list(original_search.keys()), - 'track_number' in original_search, - original_search.get('track_number', 'NOT_FOUND'), - spotify_album.get('name', 'NOT_FOUND'), - ) - - # ENHANCEMENT: Use spotify_clean_album if available for consistency - album_name = (original_search.get('spotify_clean_album') or - spotify_album.get('name') or - 'Unknown Album') - - album_info = { - 'is_album': True, # FORCE TRUE - user explicitly selected album for download - 'album_name': album_name, - 'track_number': original_search.get('track_number', 1), - 'disc_number': original_search.get('disc_number', 1), - 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album.get('image_url'), - 'confidence': 0.9, # Higher confidence - user explicitly chose album - 'source': 'enhanced_fallback_album_context' - } - logger.info(f"[FORCED ALBUM] Using album: '{album_name}' for track: '{clean_track_name}'") - else: - # For singles, we still need to detect if they belong to an album. - logger.info("Single track download - attempting album detection") - album_info = _detect_album_info_web(context, spotify_artist) - - # --- Album grouping resolution --- - # Only run smart grouping for singles/auto-detected albums. - # Explicit album downloads already have the correct Spotify album name — - # re-grouping would mangle names like "(Reworked and Remastered)" into "(Deluxe Edition)". - if album_info and album_info['is_album'] and not is_album_download: - logger.info( - "SMART ALBUM GROUPING for track=%r original_album=%r", - album_info.get('clean_track_name', 'Unknown'), - album_info.get('album_name', 'None'), - ) - - # Get original album name from context if available - original_album = None - if context.get("original_search_result", {}).get("album"): - original_album = context["original_search_result"]["album"] - - # Use the GUI's smart album grouping algorithm - consistent_album_name = _resolve_album_group(spotify_artist, album_info, original_album) - album_info['album_name'] = consistent_album_name - - logger.info("Album grouping complete: final_album=%r", consistent_album_name) - elif album_info and album_info['is_album'] and is_album_download: - logger.info( - "EXPLICIT ALBUM DOWNLOAD - preserving Spotify album name=%r; skipping smart grouping", - album_info.get('album_name', 'None'), - ) - - # 1. Get transfer path (directory creation handled by _build_final_path_for_track) - file_ext = os.path.splitext(file_path)[1] - context['_audio_quality'] = _get_audio_quality_string(file_path) - if context['_audio_quality']: - logger.info(f"Audio quality detected: {context['_audio_quality']}") - - # FLAC bit depth filter - if _check_flac_bit_depth(file_path, context, context_key): - return - - # 2. Build the final path using GUI-style track naming with multiple fallback sources - if album_info and album_info['is_album']: - album_name_sanitized = _sanitize_filename(album_info['album_name']) - - # --- GUI PARITY: Use multiple sources for clean track name --- - original_search = context.get("original_search_result", {}) - clean_track_name = album_info['clean_track_name'] - - # Priority 1: Spotify clean title from context - if original_search.get('spotify_clean_title'): - clean_track_name = original_search['spotify_clean_title'] - logger.info(f"Using Spotify clean title: '{clean_track_name}'") - # Priority 2: Album info clean name - elif album_info.get('clean_track_name'): - clean_track_name = album_info['clean_track_name'] - logger.info(f"Using album info clean name: '{clean_track_name}'") - # Priority 3: Original title as fallback - else: - clean_track_name = original_search.get('title', 'Unknown Track') - logger.warning(f"Using original title as fallback: '{clean_track_name}'") - - final_track_name_sanitized = _sanitize_filename(clean_track_name) - track_number = album_info['track_number'] - - logger.debug( - "Final track_number processing: source=%s album_info_track_number=%s track_number=%s", - album_info.get('source', 'unknown'), - album_info.get('track_number', 'NOT_FOUND'), - track_number, - ) - - # Fix: Handle None track_number - if track_number is None: - track_number = _extract_track_number_from_filename(file_path) - logger.info( - "Track number was None; extracted from filename=%r -> %s", - os.path.basename(file_path), - track_number, - ) - - # Ensure track_number is valid - if not isinstance(track_number, int) or track_number < 1: - logger.error(f"Invalid track number ({track_number}), defaulting to 1") - track_number = 1 - - logger.debug(f"FINAL track_number used for filename: {track_number}") - - # CRITICAL FIX: Update album_info with corrected track_number for metadata enhancement - album_info['track_number'] = track_number - album_info['clean_track_name'] = clean_track_name # Ensure clean name is in album_info - logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata") - - # Use shared path builder for album mode - final_path, _ = _build_final_path_for_track(context, spotify_artist, album_info, file_ext) - logger.info(f"Album path: '{final_path}'") - else: - # Single track structure: Transfer/ARTIST/ARTIST - SINGLE/SINGLE.ext - # --- GUI PARITY: Use multiple sources for clean track name --- - original_search = context.get("original_search_result", {}) - clean_track_name = album_info['clean_track_name'] - - # Priority 1: Spotify clean title from context - if original_search.get('spotify_clean_title'): - clean_track_name = original_search['spotify_clean_title'] - logger.info(f"Using Spotify clean title: '{clean_track_name}'") - # Priority 2: Album info clean name - elif album_info and album_info.get('clean_track_name'): - clean_track_name = album_info['clean_track_name'] - logger.info(f"Using album info clean name: '{clean_track_name}'") - # Priority 3: Original title as fallback - else: - clean_track_name = original_search.get('title', 'Unknown Track') - logger.warning(f"Using original title as fallback: '{clean_track_name}'") - - # Ensure clean name is in album_info for path builder - if album_info: - album_info['clean_track_name'] = clean_track_name - - # Use shared path builder for single mode - final_path, _ = _build_final_path_for_track(context, spotify_artist, album_info, file_ext) - logger.info(f"Single path: '{final_path}'") - - # Store the actual computed path so verification uses this exact path - # instead of recomputing independently (which can produce mismatches) - context['_final_processed_path'] = final_path - - # 3. Enhance metadata, move file, download art, and cleanup - try: - logger.warning(f"[Metadata Input] artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})") - if album_info: - logger.warning(f"[Metadata Input] album: '{album_info.get('album_name', 'MISSING')}', track#: {album_info.get('track_number', 'MISSING')}, disc#: {album_info.get('disc_number', 'MISSING')}, source: {album_info.get('source', 'unknown')}") - else: - logger.info("[Metadata Input] album_info: None (single track)") - _enhance_file_metadata(file_path, context, spotify_artist, album_info) - except Exception as meta_err: - import traceback - pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") - # Wipe source tags even though enhancement failed — prevents Soulseek - # uploader's MusicBrainz IDs from causing album splits in Navidrome - _wipe_source_tags(file_path) - - # Detect enhance mode from track context - _enhance_source_info = context.get('track_info', {}).get('source_info') or {} - if isinstance(_enhance_source_info, str): - try: - _enhance_source_info = json.loads(_enhance_source_info) - except (json.JSONDecodeError, TypeError): - _enhance_source_info = {} - is_enhance_download = _enhance_source_info.get('enhance', False) - - logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") - if os.path.exists(final_path): - # PROTECTION: If destination already exists, check before overwriting - # If the source file is gone, another thread already handled this - don't delete the destination - if not os.path.exists(file_path): - logger.info(f"[Protection] Destination exists and source already gone - file already transferred: {os.path.basename(final_path)}") - return - try: - from mutagen import File as MutagenFile - existing_file = MutagenFile(final_path) - has_metadata = existing_file is not None and len(existing_file.tags or {}) > 2 # More than basic tags - - if has_metadata and not is_enhance_download: - # Check if incoming file is higher quality and setting is enabled - _replace_lower = config_manager.get('import.replace_lower_quality', False) - if _replace_lower: - _existing_tier = _get_quality_tier_from_extension(final_path) - _incoming_tier = _get_quality_tier_from_extension(file_path) - if _incoming_tier[1] < _existing_tier[1]: - # Incoming is higher quality (lower tier number) — replace - logger.info(f"[Quality Replace] Replacing {_existing_tier[0]} with {_incoming_tier[0]}: {os.path.basename(final_path)}") - try: - os.remove(final_path) - except Exception as e: - logger.error(f"[Quality Replace] Could not remove existing file: {e}") - else: - logger.info(f"[Protection] Existing file is same or better quality ({_existing_tier[0]} vs {_incoming_tier[0]}) - skipping: {os.path.basename(final_path)}") - try: - os.remove(file_path) - except FileNotFoundError: - pass - except Exception as e: - logger.error(f"[Protection] Error removing redundant file: {e}") - return - else: - logger.info(f"[Protection] Existing file already has metadata enhancement - skipping overwrite: {os.path.basename(final_path)}") - logger.info(f"[Protection] Removing redundant download file: {os.path.basename(file_path)}") - try: - os.remove(file_path) - except FileNotFoundError: - logger.error(f"[Protection] Could not remove redundant file (already gone): {file_path}") - except Exception as e: - logger.error(f"[Protection] Error removing redundant file: {e}") - return # Don't overwrite the good file - elif is_enhance_download: - # ENHANCE BYPASS: Allow overwrite — backup original, then remove to allow move - logger.info(f"[Enhance] Quality enhance mode — replacing existing file: {os.path.basename(final_path)}") - try: - os.remove(final_path) - except Exception as e: - logger.error(f"[Enhance] Could not remove existing file for replacement: {e}") - else: - logger.info(f"[Protection] Existing file lacks metadata - safe to overwrite: {os.path.basename(final_path)}") - try: - os.remove(final_path) - except FileNotFoundError: - pass # It was just there, but now gone? - except Exception as check_error: - logger.error(f"[Protection] Error checking existing file metadata, proceeding with overwrite: {check_error}") - try: - if os.path.exists(final_path): - os.remove(final_path) - except Exception as e: - logger.error(f"[Protection] Failed to remove existing file for overwrite: {e}") - - # --- PRE-MOVE SOURCE CHECK --- - # Right before moving, verify the source file still exists. - # Another thread (Stream Processor or Verification Worker) may have - # already moved this file during the sleep + metadata enhancement window. - if not os.path.exists(file_path): - if os.path.exists(final_path): - logger.info(f"[Pre-Move] Source already gone and destination exists - another thread completed transfer: {os.path.basename(final_path)}") - # Still do cover art + lyrics since the other thread might not have finished those - _download_cover_art(album_info, os.path.dirname(final_path), context) - _generate_lrc_file(final_path, context, spotify_artist, album_info) - return - else: - # Source gone, exact destination not found — check if stream processor already - # moved it with a quality tag (e.g. "track [FLAC 24bit].flac" vs "track.flac") - expected_dir = os.path.dirname(final_path) - expected_stem = os.path.splitext(os.path.basename(final_path))[0] - expected_ext = os.path.splitext(final_path)[1] - found_variant = None - # Also check for lossy output if Blasphemy Mode may have deleted the .flac - check_exts = {expected_ext} - if expected_ext == '.flac' and config_manager.get('lossy_copy.enabled', False) and config_manager.get('lossy_copy.delete_original', False): - _lossy_ext_map = {'mp3': '.mp3', 'opus': '.opus', 'aac': '.m4a'} - _lossy_codec = config_manager.get('lossy_copy.codec', 'mp3') - check_exts.add(_lossy_ext_map.get(_lossy_codec, '.mp3')) - if os.path.exists(expected_dir): - for f in os.listdir(expected_dir): - # Match files that start with the expected stem and have a matching extension - # This catches "01 - track [FLAC 24bit].flac" when expecting "01 - track.flac" - # and "01 - track [MP3-320].mp3" when Blasphemy Mode deleted the FLAC - f_ext = os.path.splitext(f)[1].lower() - if f_ext in check_exts and os.path.splitext(f)[0].startswith(expected_stem): - found_variant = os.path.join(expected_dir, f) - break - if found_variant: - logger.debug(f"[Pre-Move] Source gone but found variant in destination (stream processor handled it): {os.path.basename(found_variant)}") - context['_final_processed_path'] = found_variant - _download_cover_art(album_info, expected_dir, context) - _generate_lrc_file(found_variant, context, spotify_artist, album_info) - return - else: - logger.warning(f"[Pre-Move] Source file gone and no matching file in destination: {os.path.basename(file_path)}") - raise FileNotFoundError(f"Source file vanished before move and destination does not exist: {file_path}") - - _safe_move_file(file_path, final_path) - - # ENHANCE CLEANUP: If format changed (e.g. MP3→FLAC), remove old-format file - if is_enhance_download and _enhance_source_info.get('original_file_path'): - original_enhance_path = _enhance_source_info['original_file_path'] - if os.path.normpath(original_enhance_path) != os.path.normpath(final_path) and os.path.exists(original_enhance_path): - try: - os.remove(original_enhance_path) - old_fmt = os.path.splitext(original_enhance_path)[1] - new_fmt = os.path.splitext(final_path)[1] - logger.info(f"[Enhance] Upgraded {old_fmt} → {new_fmt}: {os.path.basename(final_path)}") - except Exception as e: - logger.error(f"[Enhance] Could not remove old-format file: {e}") - elif is_enhance_download: - old_fmt = _enhance_source_info.get('original_format', 'unknown') - new_fmt = os.path.splitext(final_path)[1] - logger.info(f"[Enhance] Replaced in-place ({old_fmt} → {new_fmt}): {os.path.basename(final_path)}") - - _download_cover_art(album_info, os.path.dirname(final_path), context) - - # 4. Generate LRC lyrics file at final location (elegant addition) - _generate_lrc_file(final_path, context, spotify_artist, album_info) - - # 5. ReplayGain analysis — write track-level gain tags if enabled - if config_manager.get('post_processing.replaygain_enabled', False): - try: - from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF - if _rg_ffmpeg_ok(): - lufs, peak_dbfs = _rg_analyze(final_path) - gain_db = _RG_REF - lufs - _rg_write(final_path, gain_db, peak_dbfs) - pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB, peak {peak_dbfs:.2f} dBFS — {os.path.basename(final_path)}") - except Exception as rg_err: - pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") - - # Downsample hi-res FLAC to CD quality if enabled (must run before lossy copy) - downsampled_path = _downsample_hires_flac(final_path, context) - if downsampled_path: - final_path = downsampled_path - context['_final_processed_path'] = final_path - - # Lossy copy: create MP3 version if enabled - blasphemy_path = _create_lossy_copy(final_path) - if blasphemy_path: - context['_final_processed_path'] = blasphemy_path - - downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - _cleanup_empty_directories(downloads_path, file_path) - - logger.info(f"Post-processing complete for: {context.get('_final_processed_path', final_path)}") - - _emit_track_downloaded(context) - _record_library_history_download(context) - _record_download_provenance(context) - _record_soulsync_library_entry(context, spotify_artist, album_info) - - # RETAG DATA CAPTURE: Record completed album/single downloads for retag tool - try: - if not playlist_folder_mode: - completed_path = context.get('_final_processed_path', final_path) - _record_retag_download(context, spotify_artist, album_info, completed_path) - except Exception as retag_err: - logger.error(f"[Post-Process] Retag data capture failed (non-fatal): {retag_err}") - - # REPAIR: Register album folder for repair scanning when batch completes - try: - completed_path = context.get('_final_processed_path', final_path) - batch_id_for_repair = context.get('batch_id') - if completed_path and batch_id_for_repair and repair_worker: - album_folder = os.path.dirname(str(completed_path)) - if album_folder: - repair_worker.register_folder(batch_id_for_repair, album_folder) - except Exception as repair_err: - logger.error(f"[Post-Process] Repair folder registration failed: {repair_err}") - - # ALBUM CONSISTENCY: Register completed file for post-batch MB tag reconciliation - try: - completed_path = context.get('_final_processed_path', final_path) - batch_id_for_consistency = context.get('batch_id') - if completed_path and batch_id_for_consistency and album_info and album_info.get('is_album'): - _original_search = context.get('original_search_result', {}) - _file_info = { - 'path': str(completed_path), - 'track_number': album_info.get('track_number', 1), - 'disc_number': album_info.get('disc_number', 1), - 'title': _original_search.get('spotify_clean_title', '') or album_info.get('clean_track_name', ''), - } - with tasks_lock: - if batch_id_for_consistency in download_batches: - download_batches[batch_id_for_consistency].setdefault('_consistency_files', []).append(_file_info) - except Exception as cons_err: - logger.error(f"[Post-Process] Album consistency registration failed: {cons_err}") - - # WISHLIST REMOVAL: Check if this track should be removed from wishlist after successful download - try: - _check_and_remove_from_wishlist(context) - except Exception as wishlist_error: - logger.error(f"[Post-Process] Error checking wishlist removal: {wishlist_error}") - - # Call completion callback for missing downloads tasks to start next batch - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id and batch_id: - logger.info(f"[Post-Process] Calling completion callback for task {task_id} in batch {batch_id}") - - # Mark task as stream processed and set terminal status so - # _validate_worker_counts won't count this task as active - # (prevents active_count inflation race). - # NOTE: Only set status here — don't call _mark_task_completed() because - # _run_post_processing_worker will call it later with the session counter - # increment. Calling it here too would double-count the download. - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['stream_processed'] = True - download_tasks[task_id]['status'] = 'completed' - logger.info(f"[Post-Process] Marked task {task_id} as completed") - - _on_download_completed(batch_id, task_id, success=True) - - except Exception as e: - import traceback - pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: {e}") - pp_logger.info(traceback.format_exc()) - logger.error(f"\nCRITICAL ERROR in post-processing for {context_key}: {e}") - traceback.print_exc() - - # Only retry if the source file still exists - otherwise retrying is pointless - # and creates an infinite loop of failures - import os as _os - source_exists = _os.path.exists(file_path) if file_path else False - if source_exists: - # Remove from processed set so it can be retried - if context_key in _processed_download_ids: - _processed_download_ids.remove(context_key) - logger.warning(f"Removed {context_key} from processed set - will retry on next check") - - # Re-add to matched context for retry - with matched_context_lock: - if context_key not in matched_downloads_context: - matched_downloads_context[context_key] = context - logger.warning(f"Re-added {context_key} to context for retry") - else: - logger.warning(f"Source file gone, not retrying: {context_key}") - finally: - file_lock.release() - # Clean up the lock entry to prevent unbounded memory growth - with _post_process_locks_lock: - _post_process_locks.pop(context_key, None) + from core.import_pipeline import post_process_matched_download + return post_process_matched_download(context_key, context, file_path, _build_import_pipeline_runtime()) # Keep track of processed downloads to avoid re-processing _processed_download_ids = set() @@ -22236,12 +18330,6 @@ _processed_download_ids = set() # so we only log the warning once per key instead of spamming every poll cycle _stale_transfer_keys = set() -# Per-context-key locks to prevent two threads from processing the same file simultaneously. -# Without this, the Stream Processor and Verification Worker race to move the same source file, -# and the loser gets a FileNotFoundError because the winner already moved it. -_post_process_locks = {} # {context_key: threading.Lock()} -_post_process_locks_lock = threading.Lock() # protects the dict itself - # --- File Discovery Retry System --- # Prevents race condition where slskd reports completion before file is written to disk # Tracks retry attempts per download to give files time to finish writing @@ -22249,83 +18337,6 @@ _download_retry_attempts = {} # {context_key: {'count': N, 'first_attempt': tim _download_retry_max = 10 # Max retries before giving up (10 seconds with 1s poll interval) _download_retry_lock = threading.Lock() -def _record_retag_download(context, spotify_artist, album_info, final_path): - """Record a completed download in the retag tables for later re-tagging.""" - from database.music_database import get_database - db = get_database() - - # Extract artist name - if isinstance(spotify_artist, dict): - artist_name = spotify_artist.get('name', 'Unknown Artist') - else: - artist_name = getattr(spotify_artist, 'name', 'Unknown Artist') - - spotify_album = context.get('spotify_album', {}) - original_search = context.get('original_search_result', {}) - track_info = context.get('track_info', {}) - - is_album = album_info and album_info.get('is_album', False) - group_type = 'album' if is_album else 'single' - album_name = album_info.get('album_name', '') if album_info else ( - original_search.get('spotify_clean_title', 'Unknown')) - - # Determine album IDs (Spotify vs iTunes) — skip beatport_ prefixed IDs - spotify_album_id = None - itunes_album_id = None - if spotify_album: - album_id_raw = str(spotify_album.get('id', '')) - if album_id_raw and album_id_raw.isdigit(): - itunes_album_id = album_id_raw - elif album_id_raw and not album_id_raw.startswith('beatport_'): - spotify_album_id = album_id_raw - - image_url = album_info.get('album_image_url') if album_info else None - total_tracks = spotify_album.get('total_tracks', 1) if spotify_album else 1 - release_date = spotify_album.get('release_date', '') if spotify_album else '' - - # Find or create group (avoid duplicating for multi-track albums) - group_id = db.find_retag_group(artist_name, album_name) - if group_id is None: - group_id = db.add_retag_group( - group_type=group_type, artist_name=artist_name, album_name=album_name, - image_url=image_url, spotify_album_id=spotify_album_id, - itunes_album_id=itunes_album_id, total_tracks=total_tracks, - release_date=release_date - ) - if group_id is None: - return - - # Track details - track_number = album_info.get('track_number', 1) if album_info else 1 - disc_number = original_search.get('disc_number') or ( - album_info.get('disc_number', 1) if album_info else 1) - title = original_search.get('spotify_clean_title') or ( - album_info.get('clean_track_name', 'Unknown Track') if album_info else 'Unknown Track') - file_format = os.path.splitext(str(final_path))[1].lstrip('.').lower() - - # Track IDs (Spotify vs iTunes) — skip beatport_ prefixed IDs - spotify_track_id = None - itunes_track_id = None - if track_info and track_info.get('id'): - tid = str(track_info['id']) - if tid.isdigit(): - itunes_track_id = tid - elif not tid.startswith('beatport_'): - spotify_track_id = tid - - # Avoid duplicate track entries - if not db.retag_track_exists(group_id, str(final_path)): - db.add_retag_track( - group_id=group_id, track_number=track_number, disc_number=disc_number, - title=title, file_path=str(final_path), file_format=file_format, - spotify_track_id=spotify_track_id, itunes_track_id=itunes_track_id - ) - logger.info(f"[Retag] Recorded track for retag: '{title}' in '{album_name}'") - - # Cap retag groups at 100, remove oldest - db.trim_retag_groups(100) - - def _execute_retag(group_id, album_id): """Execute a retag operation: re-tag files in a group with metadata from a new album match.""" global retag_state @@ -27449,7 +23460,7 @@ def _on_download_completed(batch_id, task_id, success=True): artist_name=_cons_artist_name, mb_service=_cons_mb_svc, total_discs=_cons_album.get('total_discs', 1), - file_lock_fn=_get_file_lock, + file_lock_fn=get_file_lock, ) if _cons_result.get('success'): logger.info(f"[Album Consistency] {_cons_result['tags_written']}/{_cons_result['total_files']} files " @@ -27533,7 +23544,7 @@ def _run_full_missing_tracks_process(batch_id, playlist_id, tracks_json): if release and release.get('id'): release_mbid = release['id'] _artist_key = artist_name_pf.lower().strip() - _rc_key_norm = (_normalize_album_cache_key(album_name_pf), _artist_key) + _rc_key_norm = (normalize_album_cache_key(album_name_pf), _artist_key) _rc_key_exact = (album_name_pf.lower().strip(), _artist_key) with _mb_release_cache_lock: _mb_release_cache[_rc_key_norm] = release_mbid @@ -28123,19 +24134,19 @@ def _run_post_processing_worker(task_id, batch_id): original_search = context.get("original_search_result", {}) logger.info(f"[Post-Processing] original_search keys: {list(original_search.keys())}") - spotify_clean_title = original_search.get('spotify_clean_title') + clean_title = get_import_clean_title(context, default=original_search.get('title', '')) track_number = original_search.get('track_number') - logger.info(f"[Post-Processing] spotify_clean_title: '{spotify_clean_title}', track_number: {track_number}") + logger.info(f"[Post-Processing] clean_title: '{clean_title}', track_number: {track_number}") - if spotify_clean_title and track_number: + if clean_title and track_number: # Generate expected final filename that stream processor would create # Pattern: f"{track_number:02d} - {clean_title}.flac" - sanitized_title = spotify_clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') + sanitized_title = clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') expected_final_filename = f"{track_number:02d} - {sanitized_title}.flac" logger.info(f"[Post-Processing] Generated expected final filename: {expected_final_filename}") else: - logger.warning(f"[Post-Processing] Missing required data - spotify_clean_title: {bool(spotify_clean_title)}, track_number: {bool(track_number)}") + logger.warning(f"[Post-Processing] Missing required data - clean_title: {bool(clean_title)}, track_number: {bool(track_number)}") except Exception as e: logger.error(f"[Post-Processing] Error generating expected filename: {e}") import traceback @@ -28160,19 +24171,19 @@ def _run_post_processing_worker(task_id, batch_id): original_search = context.get("original_search_result", {}) logger.info(f"[Post-Processing] fuzzy context original_search keys: {list(original_search.keys())}") - spotify_clean_title = original_search.get('spotify_clean_title') + clean_title = get_import_clean_title(context, default=original_search.get('title', '')) track_number = original_search.get('track_number') - logger.info(f"[Post-Processing] fuzzy context spotify_clean_title: '{spotify_clean_title}', track_number: {track_number}") + logger.info(f"[Post-Processing] fuzzy context clean_title: '{clean_title}', track_number: {track_number}") - if spotify_clean_title and track_number: + if clean_title and track_number: # Generate expected final filename that stream processor would create # Pattern: f"{track_number:02d} - {clean_title}.flac" - sanitized_title = spotify_clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') + sanitized_title = clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') expected_final_filename = f"{track_number:02d} - {sanitized_title}.flac" logger.info(f"[Post-Processing] Generated expected final filename from fuzzy match: {expected_final_filename}") else: - logger.warning(f"[Post-Processing] Missing required data from fuzzy match - spotify_clean_title: {bool(spotify_clean_title)}, track_number: {bool(track_number)}") + logger.warning(f"[Post-Processing] Missing required data from fuzzy match - clean_title: {bool(clean_title)}, track_number: {bool(track_number)}") except Exception as e: logger.error(f"[Post-Processing] Error generating expected filename from fuzzy match: {e}") import traceback @@ -28312,22 +24323,22 @@ def _run_post_processing_worker(task_id, batch_id): # Attempt to complete metadata enhancement using context if context and expected_final_filename: try: + context = normalize_import_context(context) # Extract required data from context - original_search = context.get("original_search_result", {}) - spotify_artist = context.get("spotify_artist") - spotify_album = context.get("spotify_album") + original_search = get_import_original_search(context) + artist_context = get_import_context_artist(context) + album_context = get_import_context_album(context) - if spotify_artist and spotify_album: + if artist_context and album_context: # CRITICAL FIX: Create album_info dict with proper structure for metadata enhancement # This must match the format used in main stream processor to ensure consistency # Extract track number from context (should be available from fuzzy match) - original_search = context.get("original_search_result", {}) track_number = original_search.get('track_number', 1) # If no track number in context, extract from filename if track_number == 1 and found_file: - track_number = _extract_track_number_from_filename(found_file) + track_number = extract_track_number_from_filename(found_file) logger.warning( "[Verification] missing track_number; extracted from filename=%r -> %s", os.path.basename(found_file), @@ -28339,17 +24350,22 @@ def _run_post_processing_worker(task_id, batch_id): logger.error(f"[Verification] Invalid track number ({track_number}), defaulting to 1") track_number = 1 - # Get clean track name - clean_track_name = (original_search.get('spotify_clean_title') or - original_search.get('title', 'Unknown Track')) + # Get clean track name + clean_track_name = get_import_clean_title(context, default=original_search.get('title', 'Unknown Track')) + album_name = get_import_clean_album(context, default=album_context.get('name', 'Unknown Album')) + album_image_url = album_context.get('image_url') + if not album_image_url and album_context.get('images'): + album_images = album_context.get('images', []) + if album_images and isinstance(album_images[0], dict): + album_image_url = album_images[0].get('url') album_info = { 'is_album': True, # CRITICAL: Mark as album track - 'album_name': spotify_album.get('name', 'Unknown Album'), # CORRECT KEY + 'album_name': album_name, 'track_number': track_number, # CORRECTED TRACK NUMBER 'disc_number': original_search.get('disc_number', 1), 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album.get('images', [{}])[0].get('url') if spotify_album.get('images') else None, + 'album_image_url': album_image_url, 'confidence': 0.9, 'source': 'verification_worker_corrected' } @@ -28366,19 +24382,19 @@ def _run_post_processing_worker(task_id, batch_id): original_album_ctx = raw_album_ctx['name'] else: original_album_ctx = None - consistent_album_name = _resolve_album_group(spotify_artist, album_info, original_album_ctx) + consistent_album_name = _resolve_album_group(artist_context, album_info, original_album_ctx) album_info['album_name'] = consistent_album_name except Exception as group_err: logger.error(f"[Verification] Album grouping failed, using raw name: {group_err}") else: - logger.info(f"[Verification] Explicit album download - preserving Spotify album name: '{album_info['album_name']}'") + logger.info(f"[Verification] Explicit album download - preserving album name: '{album_info['album_name']}'") logger.info(f"[Verification] Created proper album_info - track_number: {track_number}, album: {album_info['album_name']}") logger.info(f"[Post-Processing] Attempting metadata enhancement for: {found_file}") - logger.warning(f"[Metadata Input] Verification worker - artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})") + logger.warning(f"[Metadata Input] Verification worker - artist: '{artist_context.get('name', 'MISSING')}' (id: {artist_context.get('id', 'MISSING')})") logger.warning(f"[Metadata Input] Verification worker - album: '{album_info.get('album_name', 'MISSING')}', track#: {album_info.get('track_number', 'MISSING')}, source: {album_info.get('source', 'unknown')}") - enhancement_success = _enhance_file_metadata(found_file, context, spotify_artist, album_info) + enhancement_success = _enhance_file_metadata(found_file, context, artist_context, album_info) if enhancement_success: with tasks_lock: @@ -28388,8 +24404,8 @@ def _run_post_processing_worker(task_id, batch_id): else: logger.info(f"[Post-Processing] Metadata enhancement returned False for: {os.path.basename(found_file)}") else: - logger.warning("[Post-Processing] Missing spotify_artist or spotify_album in context") - logger.info(f"[Post-Processing] spotify_artist: {spotify_artist is not None}, spotify_album: {spotify_album is not None}") + logger.warning("[Post-Processing] Missing artist or album in context") + logger.info(f"[Post-Processing] artist_context: {artist_context is not None}, album_context: {album_context is not None}") # Wipe source tags even without full enhancement — prevents # Soulseek uploader's MusicBrainz IDs from causing album splits if found_file and os.path.exists(found_file): @@ -28466,7 +24482,6 @@ def _run_post_processing_worker(task_id, batch_id): download_tasks[task_id]['error_message'] = f"Critical post-processing error: {str(e)}" _on_download_completed(batch_id, task_id, success=False) - def _download_track_worker(task_id, batch_id=None): """ Enhanced download worker that matches the GUI's exact retry logic. @@ -29126,7 +25141,7 @@ def _get_staging_file_cache(batch_id): if batch_id in _staging_cache: return _staging_cache[batch_id] - staging_path = _get_staging_path() + staging_path = get_staging_path() if not os.path.isdir(staging_path): with _staging_cache_lock: _staging_cache[batch_id] = [] @@ -30654,7 +26669,7 @@ def _check_batch_completion_v2(batch_id): artist_name=_cons_artist_name, mb_service=_cons_mb_svc, total_discs=_cons_album.get('total_discs', 1), - file_lock_fn=_get_file_lock, + file_lock_fn=get_file_lock, ) if _cons_result.get('success'): logger.info(f"[Album Consistency V2] {_cons_result['tags_written']}/{_cons_result['total_files']} files " @@ -45808,142 +41823,6 @@ def cancel_listenbrainz_sync(playlist_mbid): logger.error(f"Error cancelling ListenBrainz sync: {e}") return jsonify({"error": str(e)}), 500 - -# OLD ENDPOINT - REMOVE ALL THE CODE BELOW FOR THE OLD IMPLEMENTATION -def _old_get_listenbrainz_playlist_tracks_DEPRECATED(playlist_mbid): - """DEPRECATED - Old implementation that fetches from API""" - try: - from core.listenbrainz_client import ListenBrainzClient - - client = ListenBrainzClient() - - playlist = client.get_playlist_details(playlist_mbid, fetch_metadata=True) - - if not playlist: - return jsonify({ - "success": False, - "error": "Playlist not found or not accessible" - }), 404 - - # Extract tracks from JSPF format - jspf_tracks = playlist.get('track', []) - - # Convert to our standard format - prepare tracks first without cover art - tracks = [] - logger.info(f"Processing {len(jspf_tracks)} tracks from playlist") - - # First pass: extract all track data without cover art - track_data_list = [] - for idx, track in enumerate(jspf_tracks): - # Get recording MBID from identifier - recording_mbid = None - identifiers = track.get('identifier', []) - for identifier in identifiers: - if 'musicbrainz.org/recording/' in identifier: - recording_mbid = identifier.split('/')[-1] - break - - # Get extension data (has MusicBrainz metadata) - extension = track.get('extension', {}) - mb_data = extension.get('https://musicbrainz.org/doc/jspf#track', {}) - - if idx == 0: - logger.debug(f"Sample track extension data: {extension}") - logger.debug(f"Sample mb_data keys: {mb_data.keys() if mb_data else 'None'}") - - # Extract release MBID for cover art - release_mbid = None - if mb_data: - # Check in additional_metadata first - additional_metadata = mb_data.get('additional_metadata', {}) - if 'caa_release_mbid' in additional_metadata: - release_mbid = additional_metadata['caa_release_mbid'] - # Fallback to top-level release_mbid - elif 'release_mbid' in mb_data: - release_mbid = mb_data['release_mbid'] - - if idx == 0: - logger.debug(f"🆔 First track release_mbid: {release_mbid}") - - track_data = { - 'track_name': track.get('title', 'Unknown Track'), - 'artist_name': track.get('creator', 'Unknown Artist'), - 'album_name': track.get('album', 'Unknown Album'), - 'duration_ms': track.get('duration', 0), - 'mbid': recording_mbid, - 'release_mbid': release_mbid, - 'album_cover_url': None, # Will be fetched in parallel - 'additional_metadata': mb_data - } - - track_data_list.append(track_data) - - # Second pass: fetch cover art in parallel using threading (much faster) - from concurrent.futures import ThreadPoolExecutor, as_completed - import time - - def fetch_cover_art(track_data): - """Fetch cover art for a single track""" - release_mbid = track_data.get('release_mbid') - if not release_mbid: - return None - - try: - cover_art_url = f"https://coverartarchive.org/release/{release_mbid}" - cover_response = requests.get(cover_art_url, timeout=3) - - if cover_response.status_code == 200: - cover_data = cover_response.json() - images = cover_data.get('images', []) - - # Get front cover - for img in images: - if img.get('front'): - return img.get('thumbnails', {}).get('small') or img.get('image') - - # Fallback to first image - if images: - return images[0].get('thumbnails', {}).get('small') or images[0].get('image') - except: - pass - - return None - - logger.info(f"Fetching cover art for {len(track_data_list)} tracks in parallel...") - start_time = time.time() - - # Fetch up to 10 covers at a time - with ThreadPoolExecutor(max_workers=10) as executor: - future_to_track = {executor.submit(fetch_cover_art, track): idx - for idx, track in enumerate(track_data_list)} - - for future in as_completed(future_to_track): - idx = future_to_track[future] - try: - cover_url = future.result() - if cover_url: - track_data_list[idx]['album_cover_url'] = cover_url - except Exception as e: - pass - - elapsed = time.time() - start_time - covers_found = sum(1 for t in track_data_list if t.get('album_cover_url')) - logger.info(f"Fetched {covers_found}/{len(track_data_list)} covers in {elapsed:.2f}s") - - tracks = track_data_list - - return jsonify({ - "success": True, - "tracks": tracks, - "track_count": len(tracks) - }) - - except Exception as e: - logger.error(f"Error getting ListenBrainz playlist tracks: {e}") - import traceback - traceback.print_exc() - return jsonify({"success": False, "error": str(e)}), 500 - @app.route('/api/metadata/start', methods=['POST']) def start_metadata_update(): """Start the metadata update process - EXACT copy of dashboard.py logic""" @@ -52006,17 +47885,11 @@ def repair_job_progress(): AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif', '.ape'} -def _get_staging_path(): - """Get the resolved staging folder path.""" - raw = config_manager.get('import.staging_path', './Staging') - return docker_resolve_path(raw) - - @app.route('/api/import/staging/files', methods=['GET']) def import_staging_files(): """Scan the staging folder and return audio files with tag metadata.""" try: - staging_path = _get_staging_path() + staging_path = get_staging_path() os.makedirs(staging_path, exist_ok=True) files = [] @@ -52028,7 +47901,7 @@ def import_staging_files(): full_path = os.path.join(root, fname) rel_path = os.path.relpath(full_path, staging_path) - meta = _read_staging_file_metadata(full_path, rel_path) + meta = read_staging_file_metadata(full_path, rel_path) files.append({ 'filename': fname, @@ -52058,7 +47931,7 @@ def import_staging_groups(): the same album+artist combo. Returns groups sorted by file count descending. """ try: - staging_path = _get_staging_path() + staging_path = get_staging_path() if not os.path.isdir(staging_path): return jsonify({'success': True, 'groups': []}) @@ -52072,7 +47945,7 @@ def import_staging_groups(): full_path = os.path.join(root, fname) rel_path = os.path.relpath(full_path, staging_path) - meta = _read_staging_file_metadata(full_path, rel_path) + meta = read_staging_file_metadata(full_path, rel_path) album = meta['album'] artist = meta['albumartist'] or meta['artist'] if not album or not artist: @@ -52118,7 +47991,7 @@ def import_staging_groups(): def import_staging_hints(): """Extract album search hints from staging folder (tags + folder names). Fast — no Spotify calls.""" try: - staging_path = _get_staging_path() + staging_path = get_staging_path() if not os.path.isdir(staging_path): return jsonify({'success': True, 'hints': []}) @@ -52182,34 +48055,20 @@ def import_staging_hints(): @app.route('/api/import/search/albums', methods=['GET']) def import_search_albums(): - """Search for albums via Spotify for import matching.""" + """Search for albums using the active metadata provider.""" try: query = request.args.get('q', '').strip() if not query: return jsonify({'success': False, 'error': 'Missing query parameter'}), 400 limit = min(int(request.args.get('limit', 12)), 50) + from core.metadata_service import get_primary_source - if _is_hydrabase_active(): - albums = hydrabase_client.search_albums(query, limit=limit) - else: - if hydrabase_worker and dev_mode_enabled: - hydrabase_worker.enqueue(query, 'albums') - albums = spotify_client.search_albums(query, limit=limit) + if get_primary_source() == 'hydrabase' and hydrabase_worker and dev_mode_enabled: + hydrabase_worker.enqueue(query, 'albums') - results = [] - for a in albums: - results.append({ - 'id': a.id, - 'name': a.name, - 'artist': ', '.join(a.artists) if a.artists else 'Unknown Artist', - 'release_date': a.release_date or '', - 'total_tracks': a.total_tracks, - 'image_url': a.image_url, - 'album_type': a.album_type or 'album' - }) - - return jsonify({'success': True, 'albums': results}) + albums = search_import_albums(query, limit=limit) + return jsonify({'success': True, 'albums': albums}) except Exception as e: logger.error(f"Error searching albums for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @@ -52219,204 +48078,24 @@ def import_search_albums(): def import_album_match(): """Match staging files to an album's tracklist.""" try: - data = request.get_json() + data = request.get_json() or {} album_id = data.get('album_id') album_name = data.get('album_name', '') album_artist = data.get('album_artist', '') + source = str(data.get('source') or '').strip().lower() # Optional: only match specific files (from auto-group selection) filter_file_paths = set(data.get('file_paths', [])) if not album_id: return jsonify({'success': False, 'error': 'Missing album_id'}), 400 - spotify_tracks = None - album_info = None - - # Try Hydrabase first when active — look up by album soul_id - if _is_hydrabase_active(): - try: - hydra_tracks = hydrabase_client.get_album_tracks(album_id, limit=50) - if hydra_tracks: - spotify_tracks = [] - for t in hydra_tracks: - artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else [] - spotify_tracks.append({ - 'name': t.name, - 'track_number': t.track_number or 0, - 'disc_number': t.disc_number or 1, - 'duration_ms': t.duration_ms, - 'id': t.id, - 'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list], - 'uri': '' - }) - album_info = { - 'id': album_id, - 'name': album_name or hydra_tracks[0].album or '', - 'artist': album_artist, - 'artists': [album_artist] if album_artist else [], - 'release_date': '', - 'total_tracks': len(spotify_tracks), - 'image_url': None, - 'genres': [] - } - logger.info(f"Hydrabase album_tracks returned {len(spotify_tracks)} tracks for album_id '{album_id}'") - except Exception as e: - logger.warning(f"Hydrabase album_tracks failed, falling back to Spotify: {e}") - spotify_tracks = None - - # Fall back to Spotify - if spotify_tracks is None: - album_data = spotify_client.get_album(album_id) - if not album_data: - return jsonify({'success': False, 'error': 'Album not found'}), 404 - - tracks_data = spotify_client.get_album_tracks(album_id) - if not tracks_data or 'items' not in tracks_data: - return jsonify({'success': False, 'error': 'Could not get album tracks'}), 500 - - spotify_tracks = tracks_data['items'] - - # Build album summary - album_artists = [a['name'] for a in album_data.get('artists', [])] - album_info = { - 'id': album_id, - 'name': album_data.get('name', 'Unknown Album'), - 'artist': ', '.join(album_artists), - 'artists': album_artists, - 'release_date': album_data.get('release_date', ''), - 'total_tracks': album_data.get('total_tracks', len(spotify_tracks)), - 'image_url': (album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None), - 'genres': album_data.get('genres', []) - } - - # Get artist info for context building later - if album_data.get('artists'): - primary_artist = album_data['artists'][0] - album_info['artist_id'] = primary_artist.get('id', '') - - # Scan staging files - staging_path = _get_staging_path() - staging_files = [] - for root, _dirs, filenames in os.walk(staging_path): - for fname in filenames: - ext = os.path.splitext(fname)[1].lower() - if ext not in AUDIO_EXTENSIONS: - continue - full_path = os.path.join(root, fname) - - meta = _read_staging_file_metadata(full_path, fname) - - staging_files.append({ - 'filename': fname, - 'full_path': full_path, - 'title': meta['title'], - 'artist': meta['albumartist'] or meta['artist'], - 'album': meta['album'], - 'track_number': meta['track_number'], - 'disc_number': meta['disc_number'], - }) - - # Filter to specific files if requested (auto-group selection) - if filter_file_paths: - staging_files = [sf for sf in staging_files if sf['full_path'] in filter_file_paths] - - # Match each Spotify track to the best staging file - matches = [] - used_files = set() - album_name_for_match = album_info.get('name', '') - - for sp_track in spotify_tracks: - sp_name = sp_track.get('name', '') - sp_number = sp_track.get('track_number', 0) - sp_disc = sp_track.get('disc_number', 1) - sp_artists = sp_track.get('artists', []) - sp_artist_name = sp_artists[0]['name'] if sp_artists and isinstance(sp_artists[0], dict) else (sp_artists[0] if sp_artists else '') - - best_match = None - best_score = 0.0 - - for i, sf in enumerate(staging_files): - if i in used_files: - continue - - score = 0.0 - - # Title similarity (weight 0.45) - title_sim = matching_engine.similarity_score( - matching_engine.normalize_string(sp_name), - matching_engine.normalize_string(sf['title'] or '') - ) - score += title_sim * 0.45 - - # Artist similarity (weight 0.15) - sf_artist = sf.get('artist') or '' - if sf_artist and sp_artist_name: - artist_sim = matching_engine.similarity_score( - matching_engine.normalize_string(sp_artist_name), - matching_engine.normalize_string(sf_artist) - ) - score += artist_sim * 0.15 - else: - score += 0.075 # neutral when no artist data - - # Track number match (weight 0.30) - if sf['track_number'] and sp_number: - if sf['track_number'] == sp_number: - score += 0.30 - elif abs(sf['track_number'] - sp_number) <= 1: - score += 0.12 - - # Album tag bonus (weight 0.10) — reward files whose album tag matches - sf_album = sf.get('album') or '' - if sf_album and album_name_for_match: - album_sim = matching_engine.similarity_score( - matching_engine.normalize_string(sf_album), - matching_engine.normalize_string(album_name_for_match) - ) - score += album_sim * 0.10 - - if score > best_score and score >= 0.4: - best_score = score - best_match = i - - if best_match is not None: - used_files.add(best_match) - matches.append({ - 'spotify_track': { - 'name': sp_name, - 'track_number': sp_number, - 'disc_number': sp_disc, - 'duration_ms': sp_track.get('duration_ms', 0), - 'id': sp_track.get('id', ''), - 'artists': [a['name'] for a in sp_track.get('artists', [])], - 'uri': sp_track.get('uri', '') - }, - 'staging_file': staging_files[best_match], - 'confidence': round(best_score, 2) - }) - else: - matches.append({ - 'spotify_track': { - 'name': sp_name, - 'track_number': sp_number, - 'disc_number': sp_disc, - 'duration_ms': sp_track.get('duration_ms', 0), - 'id': sp_track.get('id', ''), - 'artists': [a['name'] for a in sp_track.get('artists', [])], - 'uri': sp_track.get('uri', '') - }, - 'staging_file': None, - 'confidence': 0 - }) - - # Unmatched staging files - unmatched_files = [sf for i, sf in enumerate(staging_files) if i not in used_files] - - return jsonify({ - 'success': True, - 'album': album_info, - 'matches': matches, - 'unmatched_files': unmatched_files - }) + payload = build_album_import_match_payload( + album_id, + album_name=album_name, + album_artist=album_artist, + file_paths=filter_file_paths, + source=source or None, + ) + return jsonify(payload) except Exception as e: logger.error(f"Error matching album for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @@ -52426,7 +48105,7 @@ def import_album_match(): def import_album_process(): """Process matched album files through the post-processing pipeline.""" try: - data = request.get_json() + data = request.get_json() or {} album = data.get('album', {}) matches = data.get('matches', []) @@ -52435,34 +48114,25 @@ def import_album_process(): processed = 0 errors = [] - album_name = album.get('name', 'Unknown Album') - artist_name = album.get('artist', 'Unknown Artist') - artist_id = album.get('artist_id', '') - album_id = album.get('id', '') + album_name = album.get('name', album.get('album_name', 'Unknown Album')) + artist_name = album.get('artist', album.get('artist_name', 'Unknown Artist')) + album_id = album.get('id', album.get('album_id', '')) + source = str(album.get('source') or data.get('source') or '').strip().lower() - # Get artist genres from Spotify if possible - artist_genres = album.get('genres', []) - if not artist_genres and artist_id and not _spotify_rate_limited(): - try: - from core.api_call_tracker import api_call_tracker - if hasattr(spotify_client, 'sp') and spotify_client.sp: - api_call_tracker.record_call('spotify', endpoint='artist') - sp_artist = spotify_client.sp.artist(artist_id) if hasattr(spotify_client, 'sp') and spotify_client.sp else None - if sp_artist: - artist_genres = sp_artist.get('genres', []) - except Exception: - pass - - # Compute total_discs across all matched tracks for multi-disc subfolder support total_discs = max( - (m['spotify_track'].get('disc_number', 1) for m in matches if m.get('spotify_track')), - default=1 + ( + match.get('track', {}).get('disc_number', 1) + for match in matches + if match.get('track') + ), + default=1, ) + artist_context = resolve_album_artist_context(album, source=source) for match in matches: staging_file = match.get('staging_file') - spotify_track = match.get('spotify_track') - if not staging_file or not spotify_track: + track = match.get('track') or {} + if not staging_file or not track: continue file_path = staging_file.get('full_path', '') @@ -52470,49 +48140,18 @@ def import_album_process(): errors.append(f"File not found: {staging_file.get('filename', '?')}") continue - track_name = spotify_track.get('name', 'Unknown Track') - track_number = spotify_track.get('track_number', 1) - disc_number = spotify_track.get('disc_number', 1) - track_artists = spotify_track.get('artists', [artist_name]) + track_name = track.get('name', 'Unknown Track') + track_number = track.get('track_number', 1) + disc_number = track.get('disc_number', 1) context_key = f"import_album_{album_id}_{track_number}_{uuid.uuid4().hex[:8]}" - context = { - 'spotify_artist': { - 'name': artist_name, - 'id': artist_id, - 'genres': artist_genres - }, - 'spotify_album': { - 'id': album_id, - 'name': album_name, - 'release_date': album.get('release_date', ''), - 'total_tracks': album.get('total_tracks', len(matches)), - 'total_discs': total_discs, - 'image_url': album.get('image_url', '') - }, - 'track_info': { - 'name': track_name, - 'id': spotify_track.get('id', ''), - 'track_number': track_number, - 'disc_number': disc_number, - 'duration_ms': spotify_track.get('duration_ms', 0), - 'artists': [{'name': a} if isinstance(a, str) else a for a in track_artists], - 'uri': spotify_track.get('uri', '') - }, - 'original_search_result': { - 'title': track_name, - 'artist': artist_name, - 'album': album_name, - 'track_number': track_number, - 'disc_number': disc_number, - 'spotify_clean_title': track_name, - 'spotify_clean_album': album_name, - 'artists': [{'name': a} if isinstance(a, str) else a for a in track_artists] - }, - 'is_album_download': True, - 'has_clean_spotify_data': True, - 'has_full_spotify_metadata': True - } + context = build_album_import_context( + album, + track, + artist_context=artist_context, + total_discs=total_discs, + source=source, + ) try: _post_process_matched_download(context_key, context, file_path) @@ -52561,35 +48200,18 @@ def import_album_process(): @app.route('/api/import/search/tracks', methods=['GET']) def import_search_tracks(): - """Search Spotify for individual tracks (used for manual singles identification).""" + """Search tracks using the configured metadata provider priority order.""" try: query = request.args.get('q', '').strip() if not query: return jsonify({'success': False, 'error': 'Missing query parameter'}), 400 limit = min(int(request.args.get('limit', 10)), 30) + if get_primary_source() == 'hydrabase' and hydrabase_worker and dev_mode_enabled: + hydrabase_worker.enqueue(query, 'tracks') - if _is_hydrabase_active(): - tracks = hydrabase_client.search_tracks(query, limit=limit) - else: - if hydrabase_worker and dev_mode_enabled: - hydrabase_worker.enqueue(query, 'tracks') - tracks = spotify_client.search_tracks(query, limit=limit) - - results = [] - for t in tracks: - results.append({ - 'id': t.id, - 'name': t.name, - 'artist': ', '.join(t.artists) if hasattr(t, 'artists') and t.artists else 'Unknown Artist', - 'album': t.album if hasattr(t, 'album') else '', - 'album_id': t.album_id if hasattr(t, 'album_id') else '', - 'duration_ms': t.duration_ms if hasattr(t, 'duration_ms') else 0, - 'image_url': t.image_url if hasattr(t, 'image_url') else '', - 'track_number': t.track_number if hasattr(t, 'track_number') else 1, - }) - - return jsonify({'success': True, 'tracks': results}) + tracks = search_import_tracks(query, limit=limit) + return jsonify({'success': True, 'tracks': tracks}) except Exception as e: logger.error(f"Error searching tracks for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @@ -52616,180 +48238,51 @@ def import_singles_process(): title = file_info.get('title', '') artist = file_info.get('artist', '') - spotify_override = file_info.get('spotify_override', None) + manual_match = file_info.get('manual_match') + if manual_match is not None and not isinstance(manual_match, dict): + manual_match = None + + manual_match_source = '' + manual_match_id = None + if manual_match: + manual_match_source = str(manual_match.get('source') or '').strip().lower() + manual_match_id = str(manual_match.get('id') or '').strip() + if not manual_match_id or not manual_match_source: + errors.append(f"Malformed manual match for file: {file_info.get('filename', '?')}") + continue # Fallback to filename parsing if no metadata - if not title and not spotify_override: - parsed = _parse_filename_metadata(file_info.get('filename', '')) - title = parsed.get('title', os.path.splitext(file_info.get('filename', 'Unknown'))[0]) + if not title and not manual_match: + parsed = parse_filename_metadata(file_info.get('filename', '')) + title = parsed.get('title') or os.path.splitext(file_info.get('filename', 'Unknown'))[0] if not artist: artist = parsed.get('artist', '') - # Search Spotify for rich metadata - spotify_track_data = None - spotify_artist_data = None - spotify_album_data = None + from core.metadata_service import get_single_track_import_context - # If a manual spotify_override is provided, look up that specific track - if spotify_override and spotify_override.get('id'): - try: - override_id = spotify_override['id'] - sp_track = None - if hasattr(spotify_client, 'sp') and spotify_client.sp and not _spotify_rate_limited(): - from core.api_call_tracker import api_call_tracker - api_call_tracker.record_call('spotify', endpoint='track') - sp_track = spotify_client.sp.track(override_id) - if sp_track: - sp_track_artists = sp_track.get('artists', []) - spotify_track_data = { - 'name': sp_track.get('name', ''), - 'id': override_id, - 'track_number': sp_track.get('track_number', 1), - 'disc_number': sp_track.get('disc_number', 1), - 'duration_ms': sp_track.get('duration_ms', 0), - 'artists': [{'name': a.get('name', '')} for a in sp_track_artists], - 'uri': sp_track.get('uri', f"spotify:track:{override_id}") - } - title = sp_track.get('name', title) - artist = sp_track_artists[0].get('name', artist) if sp_track_artists else artist - # Get album info - sp_album_info = sp_track.get('album', {}) - if sp_album_info: - spotify_album_data = { - 'id': sp_album_info.get('id', ''), - 'name': sp_album_info.get('name', ''), - 'release_date': sp_album_info.get('release_date', ''), - 'total_tracks': sp_album_info.get('total_tracks', 1), - 'image_url': (sp_album_info.get('images', [{}])[0].get('url') if sp_album_info.get('images') else ''), - 'album_type': sp_album_info.get('album_type', 'album'), - } - album_artists = sp_album_info.get('artists', []) - if album_artists: - spotify_artist_data = { - 'name': album_artists[0].get('name', artist), - 'id': album_artists[0].get('id', ''), - 'genres': [] - } - try: - sp_a = None - if hasattr(spotify_client, 'sp') and spotify_client.sp and not _spotify_rate_limited(): - from core.api_call_tracker import api_call_tracker - api_call_tracker.record_call('spotify', endpoint='artist') - sp_a = spotify_client.sp.artist(album_artists[0]['id']) - if sp_a: - spotify_artist_data['genres'] = sp_a.get('genres', []) - except Exception: - pass - except Exception as override_err: - logger.warning(f"Spotify override lookup failed for track {spotify_override.get('id')}: {override_err}") - - if not spotify_track_data and title: - try: - search_q = f"{title} {artist}" if artist else title - tracks = spotify_client.search_tracks(search_q, limit=1) - if tracks: - t = tracks[0] - spotify_track_data = { - 'name': t.name, - 'id': t.id, - 'track_number': t.track_number if hasattr(t, 'track_number') else 1, - 'disc_number': 1, - 'duration_ms': t.duration_ms if hasattr(t, 'duration_ms') else 0, - 'artists': [{'name': a} for a in (t.artists if hasattr(t, 'artists') else [artist])], - 'uri': f"spotify:track:{t.id}" - } - # Get album info from the track's album - if hasattr(t, 'album_id') and t.album_id: - sp_album = spotify_client.get_album(t.album_id) - if sp_album: - spotify_album_data = { - 'id': t.album_id, - 'name': sp_album.get('name', ''), - 'release_date': sp_album.get('release_date', ''), - 'total_tracks': sp_album.get('total_tracks', 1), - 'image_url': (sp_album.get('images', [{}])[0].get('url') if sp_album.get('images') else ''), - 'album_type': sp_album.get('album_type', 'album'), - } - # Get artist genres - sp_artists = sp_album.get('artists', []) - if sp_artists: - spotify_artist_data = { - 'name': sp_artists[0].get('name', artist), - 'id': sp_artists[0].get('id', ''), - 'genres': [] - } - try: - sp_a = None - if hasattr(spotify_client, 'sp') and spotify_client.sp and not _spotify_rate_limited(): - from core.api_call_tracker import api_call_tracker - api_call_tracker.record_call('spotify', endpoint='artist') - sp_a = spotify_client.sp.artist(sp_artists[0]['id']) - if sp_a: - spotify_artist_data['genres'] = sp_a.get('genres', []) - except Exception: - pass - - # Fallback artist data from track - if not spotify_artist_data: - track_artists = t.artists if hasattr(t, 'artists') else [artist] - spotify_artist_data = { - 'name': track_artists[0] if track_artists else artist, - 'id': '', - 'genres': [] - } - - # Fallback album data - if not spotify_album_data: - spotify_album_data = { - 'id': '', - 'name': t.album if hasattr(t, 'album') else '', - 'release_date': '', - 'total_tracks': 1, - 'image_url': t.image_url if hasattr(t, 'image_url') else '', - 'album_type': t.album_type if hasattr(t, 'album_type') else 'album', - } - except Exception as sp_err: - logger.warning(f"Spotify lookup failed for '{title}': {sp_err}") - - # Build context — use Spotify data if found, else use file metadata - if not spotify_artist_data: - spotify_artist_data = {'name': artist or 'Unknown Artist', 'id': '', 'genres': []} - if not spotify_album_data: - spotify_album_data = {'id': '', 'name': '', 'release_date': '', 'total_tracks': 1, 'image_url': '', 'album_type': 'album'} - if not spotify_track_data: - spotify_track_data = { - 'name': title, 'id': '', 'track_number': 1, 'disc_number': 1, - 'duration_ms': 0, 'artists': [{'name': artist or 'Unknown Artist'}], 'uri': '' - } - - final_title = spotify_track_data.get('name', title) - final_artist = spotify_artist_data.get('name', artist) - final_album = spotify_album_data.get('name', '') + resolved = get_single_track_import_context( + title, + artist, + override_id=manual_match_id, + override_source=manual_match_source, + ) + context = normalize_import_context(resolved['context']) + artist_data = get_import_context_artist(context) + track_data = get_import_track_info(context) + final_title = track_data.get('name', title) + final_artist = artist_data.get('name', artist) context_key = f"import_single_{uuid.uuid4().hex[:8]}" - context = { - 'spotify_artist': spotify_artist_data, - 'spotify_album': spotify_album_data, - 'track_info': spotify_track_data, - 'original_search_result': { - 'title': final_title, - 'artist': final_artist, - 'album': final_album, - 'track_number': spotify_track_data.get('track_number', 1), - 'disc_number': 1, - 'spotify_clean_title': final_title, - 'spotify_clean_album': final_album, - 'artists': spotify_track_data.get('artists', [{'name': final_artist}]) - }, - 'is_album_download': False, - 'has_clean_spotify_data': bool(spotify_track_data.get('id')), - 'has_full_spotify_metadata': bool(spotify_track_data.get('id')) - } try: _post_process_matched_download(context_key, context, file_path) processed += 1 - logger.info(f"Import single processed: {final_title} by {final_artist}") + logger.info( + "Import single processed: %s by %s (source=%s)", + final_title, + final_artist, + resolved.get('source') or 'local', + ) except Exception as proc_err: err_msg = f"{title}: {str(proc_err)}" errors.append(err_msg) @@ -52830,126 +48323,6 @@ def import_singles_process(): logger.error(f"Error processing singles import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 -# --- Import Suggestion Cache (server-side background builder) --- - -_import_suggestions_cache = { - 'suggestions': [], - 'building': False, - 'built': False, -} - -def _build_import_suggestions_background(): - """Background thread: extract hints from staging folder, search Spotify, cache results.""" - cache = _import_suggestions_cache - if cache['building']: - return - cache['building'] = True - - try: - staging_path = _get_staging_path() - if not os.path.isdir(staging_path): - cache['suggestions'] = [] - cache['built'] = True - cache['building'] = False - return - - # Reuse the hint extraction logic - tag_albums = {} - folder_hints = {} - - for root, _dirs, filenames in os.walk(staging_path): - audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS] - if not audio_files: - continue - rel_dir = os.path.relpath(root, staging_path) - if rel_dir != '.': - top_folder = rel_dir.split(os.sep)[0] - folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files) - for fname in audio_files: - full_path = os.path.join(root, fname) - try: - from mutagen import File as MutagenFile - tags = MutagenFile(full_path, easy=True) - if tags: - album = (tags.get('album') or [None])[0] - artist = (tags.get('artist') or (tags.get('albumartist') or [None]))[0] - if album: - key = (album.strip(), (artist or '').strip()) - tag_albums[key] = tag_albums.get(key, 0) + 1 - except Exception: - pass - - queries = [] - seen_lower = set() - for (album, artist), _ in sorted(tag_albums.items(), key=lambda x: -x[1]): - q = f"{album} {artist}".strip() if artist else album - if q.lower() not in seen_lower: - seen_lower.add(q.lower()) - queries.append(q) - for folder, _ in sorted(folder_hints.items(), key=lambda x: -x[1]): - q = folder.replace('_', ' ') - if q.lower() not in seen_lower: - seen_lower.add(q.lower()) - queries.append(q) - queries = queries[:5] - - if not queries: - cache['suggestions'] = [] - cache['built'] = True - cache['building'] = False - return - - suggestions = [] - seen_ids = set() - for q in queries: - try: - if not spotify_client: - break - albums = spotify_client.search_albums(q, limit=2) - for a in albums: - if a.id not in seen_ids: - seen_ids.add(a.id) - suggestions.append({ - 'id': a.id, - 'name': a.name, - 'artist': ', '.join(a.artists) if a.artists else 'Unknown Artist', - 'release_date': a.release_date or '', - 'total_tracks': a.total_tracks, - 'image_url': a.image_url, - 'album_type': a.album_type or 'album', - }) - except Exception as e: - logger.warning(f"Import suggestion search failed for '{q}': {e}") - - cache['suggestions'] = suggestions[:8] - cache['built'] = True - logger.info(f"Import suggestions cache built: {len(cache['suggestions'])} suggestions from {len(queries)} hints") - except Exception as e: - logger.error(f"Error building import suggestions cache: {e}") - cache['suggestions'] = [] - cache['built'] = True - finally: - cache['building'] = False - - -def start_import_suggestions_cache(): - """Start building the import suggestions cache in a background thread (called on server startup).""" - threading.Thread( - target=_build_import_suggestions_background, - daemon=True, - name='import-suggestions-cache' - ).start() - - -def refresh_import_suggestions_cache(): - """Invalidate and rebuild the suggestions cache (called after imports change staging contents).""" - _import_suggestions_cache['built'] = False - threading.Thread( - target=_build_import_suggestions_background, - daemon=True, - name='import-suggestions-cache' - ).start() - # ── Auto-Import Worker ── auto_import_worker = None @@ -53089,7 +48462,7 @@ def auto_import_clear_completed(): @app.route('/api/import/staging/suggestions', methods=['GET']) def import_staging_suggestions(): """Return cached import suggestions. If cache isn't built yet, returns partial/empty with a flag.""" - cache = _import_suggestions_cache + cache = get_import_suggestions_cache() return jsonify({ 'success': True, 'suggestions': cache['suggestions'], From 6872e5080df1d07c332ef2ba1ab707fc1c72a15c Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Fri, 24 Apr 2026 16:55:46 +0300 Subject: [PATCH 02/19] Refine import module boundaries - Move filename and staging helpers into their canonical modules - Extract album naming and grouping from path handling - Update import and test call sites to the new layout --- core/import_album.py | 41 +------- core/import_album_naming.py | 184 ++++++++++++++++++++++++++++++++++ core/import_file_ops.py | 106 +------------------- core/import_filename.py | 19 ++++ core/import_paths.py | 174 +------------------------------- core/import_pipeline.py | 4 +- core/import_staging.py | 104 ++++++++++++++++++- tests/test_import_album.py | 23 +++-- tests/test_import_file_ops.py | 4 +- tests/test_import_paths.py | 4 +- 10 files changed, 337 insertions(+), 326 deletions(-) create mode 100644 core/import_album_naming.py diff --git a/core/import_album.py b/core/import_album.py index 953e091e..1f439f1b 100644 --- a/core/import_album.py +++ b/core/import_album.py @@ -2,12 +2,10 @@ from __future__ import annotations -import os from typing import Any, Dict, Iterable, List, Optional, Set from core.import_context import normalize_import_context -from core.import_file_ops import read_staging_file_metadata -from core.import_staging import AUDIO_EXTENSIONS, get_staging_path +from core.import_staging import collect_staging_files from utils.logging_config import get_logger @@ -138,41 +136,6 @@ def _coerce_track_int(value: Any, default: int = 1) -> int: return default -def _collect_staging_files(file_paths: Optional[Iterable[str]] = None) -> List[Dict[str, Any]]: - staging_path = get_staging_path() - file_filter: Optional[Set[str]] = set(file_paths) if file_paths else None - staging_files: List[Dict[str, Any]] = [] - - if not os.path.isdir(staging_path): - return staging_files - - for root, _dirs, filenames in os.walk(staging_path): - for filename in filenames: - ext = os.path.splitext(filename)[1].lower() - if ext not in AUDIO_EXTENSIONS: - continue - - full_path = os.path.join(root, filename) - if file_filter is not None and full_path not in file_filter: - continue - - meta = read_staging_file_metadata(full_path, filename) - staging_files.append( - { - "filename": filename, - "full_path": full_path, - "title": meta.get("title", ""), - "artist": meta.get("albumartist") or meta.get("artist") or "", - "album": meta.get("album", ""), - "albumartist": meta.get("albumartist") or meta.get("artist") or "", - "track_number": meta.get("track_number", 1), - "disc_number": meta.get("disc_number", 1), - } - ) - - return staging_files - - def _normalize_match_track(track: Dict[str, Any], source: str, album: Dict[str, Any]) -> Dict[str, Any]: track_album = track.get("album") if isinstance(track.get("album"), dict) else album if isinstance(track_album, dict): @@ -472,7 +435,7 @@ def build_album_import_match_payload( "resolved_album_id": album_response.get("resolved_album_id") or album_id, } - staging_files = _collect_staging_files(file_paths) + staging_files = collect_staging_files(file_paths) album_name_for_match = album.get("name") or album_name or "" matches: List[Dict[str, Any]] = [] used_files: Set[int] = set() diff --git a/core/import_album_naming.py b/core/import_album_naming.py new file mode 100644 index 00000000..338f326d --- /dev/null +++ b/core/import_album_naming.py @@ -0,0 +1,184 @@ +"""Album naming and grouping helpers used by import flows.""" + +from __future__ import annotations + +import re +import threading +from typing import Any, Dict + +from core.import_context import extract_artist_name +from utils.logging_config import get_logger + + +logger = get_logger("import_album_naming") + +_album_cache_lock = threading.Lock() +_album_editions: dict[str, str] = {} +_album_name_cache: dict[str, str] = {} + + +def clear_album_grouping_cache() -> None: + """Clear cached album grouping decisions. + + Useful for tests and for any future config reload flows. + """ + with _album_cache_lock: + _album_editions.clear() + _album_name_cache.clear() + + +def get_base_album_name(album_name: str) -> str: + """Extract the base album name without edition indicators.""" + base_name = album_name or "" + base_name = re.sub( + r"\s*[\[\(][^)\]]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^)\]]*[\]\)]\s*$", + "", + base_name, + flags=re.IGNORECASE, + ) + base_name = re.sub(r"\s*[\[\(][^)\]]*\bedition\b[^)\]]*[\]\)]\s*$", "", base_name, flags=re.IGNORECASE) + base_name = re.sub( + r"\s+(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", + "", + base_name, + flags=re.IGNORECASE, + ) + return base_name.strip() + + +def detect_deluxe_edition(album_name: str) -> bool: + """Detect if an album name indicates a deluxe/special edition.""" + if not album_name: + return False + + album_lower = album_name.lower() + deluxe_indicators = [ + "deluxe", + "deluxe edition", + "special edition", + "expanded edition", + "extended edition", + "bonus", + "remastered", + "anniversary", + "collectors edition", + "limited edition", + "silver edition", + "gold edition", + "platinum edition", + ] + for indicator in deluxe_indicators: + if indicator in album_lower: + logger.info("Detected deluxe edition: %r contains %r", album_name, indicator) + return True + return False + + +def normalize_base_album_name(base_album: str, artist_name: str) -> str: + """Normalize the base album name to handle case variations and known corrections.""" + normalized_lower = (base_album or "").lower().strip() + known_corrections = { + # Add specific album name corrections here as needed. + } + + for variant, correction in known_corrections.items(): + if normalized_lower == variant.lower(): + logger.info("Album correction applied: %r -> %r", base_album, correction) + return correction + + normalized = base_album or "" + normalized = re.sub(r"\s*&\s*", " & ", normalized) + normalized = re.sub(r"\s+", " ", normalized) + normalized = normalized.strip() + logger.info("Album variant normalization: %r -> %r", base_album, normalized) + return normalized + + +def clean_album_title(album_title: str, artist_name: str) -> str: + """Clean up album title by removing common prefixes, suffixes, and artist redundancy.""" + original = (album_title or "").strip() + cleaned = original + logger.info("Album Title Cleaning: %r (artist: %r)", original, artist_name) + + cleaned = re.sub(r"^Album\s*-\s*", "", cleaned, flags=re.IGNORECASE) + artist_pattern = re.escape(artist_name or "") + r"\s*-\s*" + cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*[\[\(]\d{4}[\]\)]\s*", " ", cleaned) + + quality_patterns = [ + r"\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*", + r"\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*", + r"\s*[\[\(].*?flac.*?[\]\)]\s*", + r"\s*[\[\(].*?mp3.*?[\]\)]\s*", + r"\s*[\[\(].*?itunes.*?[\]\)]\s*", + r"\s*[\[\(].*?web.*?[\]\)]\s*", + r"\s*[\[\(].*?cd.*?[\]\)]\s*", + ] + for pattern in quality_patterns: + cleaned = re.sub(pattern, " ", cleaned, flags=re.IGNORECASE) + + cleaned = re.sub(r"\s*[\[\(][^\]\)]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*[\[\(][^\]\)]*\bedition\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"^[-\s\.]+", "", cleaned) + cleaned = re.sub(r"[-\s\.]+$", "", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned).strip() + return cleaned if cleaned else original + + +def resolve_album_group(artist_context: dict, album_info: dict, original_album: str = None) -> str: + """Smart album grouping: upgrade to deluxe if any track is deluxe.""" + try: + with _album_cache_lock: + artist_name = extract_artist_name(artist_context) + detected_album = (album_info or {}).get("album_name", "") + + if detected_album: + base_album = get_base_album_name(detected_album) + elif original_album: + cleaned_original = clean_album_title(original_album, artist_name) + base_album = get_base_album_name(cleaned_original) + else: + base_album = get_base_album_name(detected_album) + + base_album = normalize_base_album_name(base_album, artist_name) + album_key = f"{artist_name}::{base_album}" + is_deluxe_track = False + if detected_album: + is_deluxe_track = detect_deluxe_edition(detected_album) + elif original_album: + is_deluxe_track = detect_deluxe_edition(original_album) + + if album_key in _album_name_cache: + cached_name = _album_name_cache[album_key] + current_edition = _album_editions.get(album_key, "standard") + if is_deluxe_track and current_edition == "standard": + final_album_name = f"{base_album} (Deluxe Edition)" + _album_editions[album_key] = "deluxe" + _album_name_cache[album_key] = final_album_name + logger.info("Album cache upgrade: %r -> %r", album_key, final_album_name) + return final_album_name + logger.info("Using cached album name for %r: %r", album_key, cached_name) + return cached_name + + logger.info("Album grouping - Key: %r, Detected: %r", album_key, detected_album) + + current_edition = _album_editions.get(album_key, "standard") + if is_deluxe_track and current_edition == "standard": + logger.info("UPGRADE: Album %r upgraded from standard to deluxe!", base_album) + _album_editions[album_key] = "deluxe" + current_edition = "deluxe" + + if current_edition == "deluxe": + final_album_name = f"{base_album} (Deluxe Edition)" + else: + final_album_name = base_album + + _album_name_cache[album_key] = final_album_name + + logger.info("Album resolution: %r -> %r (edition: %s)", detected_album, final_album_name, current_edition) + return final_album_name + except Exception as e: + logger.error("Error resolving album group: %s", e) + album_name = (album_info or {}).get("album_name", "Unknown Album") + return album_name diff --git a/core/import_file_ops.py b/core/import_file_ops.py index 261687a8..21cf9318 100644 --- a/core/import_file_ops.py +++ b/core/import_file_ops.py @@ -4,13 +4,17 @@ from __future__ import annotations import logging import os -import re import shutil import subprocess import time from pathlib import Path from typing import Any, Dict, Optional +# Backward-compatible re-exports; canonical homes are core.import_filename +# and core.import_staging. +from core.import_filename import extract_track_number_from_filename +from core.import_staging import read_staging_file_metadata + logger = logging.getLogger("import_file_ops") @@ -85,106 +89,6 @@ def safe_move_file(src, dst): raise -def extract_track_number_from_filename(filename: str, title: str = None) -> int: - """Extract track number from a filename. Returns 1 if not found.""" - basename = os.path.splitext(os.path.basename(filename))[0].strip() - - match = re.match(r"^\d[\-\.](\d{1,2})\s*[\-\.]\s*", basename) - if match: - num = int(match.group(1)) - if 1 <= num <= 99: - return num - - match = re.match(r"^\(?(\d{1,3})\)?\s*[\-\.)\]]\s*", basename) - if match: - num = int(match.group(1)) - if 1 <= num <= 999: - return num - - return 1 - - -def _coerce_tag_number(value: Any, default: int = 1) -> int: - if value in (None, ""): - return default - - if isinstance(value, (list, tuple)): - value = value[0] if value else None - - if value in (None, ""): - return default - - text = str(value).strip() - if not text: - return default - - match = re.match(r"^(\d+)", text) - if match: - try: - return int(match.group(1)) - except ValueError: - return default - - try: - return int(text) - except (TypeError, ValueError): - return default - - -def read_staging_file_metadata(file_path: str, filename: Optional[str] = None) -> Dict[str, Any]: - """Read common audio tag metadata from a staging file.""" - try: - from mutagen import File as MutagenFile - - tags = MutagenFile(file_path, easy=True) - except Exception: - tags = None - - filename = filename or os.path.basename(file_path) - stem = os.path.splitext(os.path.basename(filename))[0] - - def _first_tag(*keys: str) -> str: - if not tags: - return "" - for key in keys: - try: - value = tags.get(key) # type: ignore[attr-defined] - except Exception: - value = None - if value: - if isinstance(value, (list, tuple)): - value = value[0] if value else "" - text = str(value).strip() - if text: - return text - return "" - - title = _first_tag("title") - artist = _first_tag("artist") - albumartist = _first_tag("albumartist") - album = _first_tag("album") - - if not title: - title = stem - if not albumartist: - albumartist = artist - - track_number = _coerce_tag_number(_first_tag("tracknumber", "track_number"), default=0) - if not track_number: - track_number = extract_track_number_from_filename(filename or file_path) - - disc_number = _coerce_tag_number(_first_tag("discnumber", "disc_number"), default=1) - - return { - "title": title, - "artist": artist, - "albumartist": albumartist, - "album": album, - "track_number": track_number, - "disc_number": disc_number, - } - - def cleanup_empty_directories(download_path, moved_file_path): """Remove empty directories after a move, ignoring hidden files.""" try: diff --git a/core/import_filename.py b/core/import_filename.py index 9298750d..92ff026a 100644 --- a/core/import_filename.py +++ b/core/import_filename.py @@ -14,6 +14,25 @@ _TRACK_PATTERNS = ( ) +def extract_track_number_from_filename(filename: str, title: str = None) -> int: + """Extract track number from a filename. Returns 1 if not found.""" + basename = os.path.splitext(os.path.basename(filename))[0].strip() + + match = re.match(r"^\d[\-\.](\d{1,2})\s*[\-\.]\s*", basename) + if match: + num = int(match.group(1)) + if 1 <= num <= 99: + return num + + match = re.match(r"^\(?(\d{1,3})\)?\s*[\-\.)\]]\s*", basename) + if match: + num = int(match.group(1)) + if 1 <= num <= 999: + return num + + return 1 + + def parse_filename_metadata(filename: str) -> Dict[str, Any]: """Extract artist/title/album hints from a loose filename.""" raw_path = str(filename or "") diff --git a/core/import_paths.py b/core/import_paths.py index 7848425c..39e280c8 100644 --- a/core/import_paths.py +++ b/core/import_paths.py @@ -6,11 +6,14 @@ import json import logging import os import re -import threading from pathlib import Path from typing import Any +# Album grouping lives in core.import_album_naming; this module keeps the +# imported helper because the path builder still needs it. +from core.import_album_naming import resolve_album_group from core.import_context import ( + extract_artist_name, get_import_clean_title, get_import_context_album, get_import_original_search, @@ -21,10 +24,6 @@ from core.import_context import ( logger = logging.getLogger("import_paths") -_album_cache_lock = threading.Lock() -_album_editions: dict[str, str] = {} -_album_name_cache: dict[str, str] = {} - def _get_config_manager(): try: @@ -55,14 +54,6 @@ def _get_album_tracks_for_source(source: str, album_id: str): return None -def _extract_artist_name(artist_context: Any) -> str: - if not artist_context: - return "" - if isinstance(artist_context, dict): - return str(artist_context.get("name", "") or "").strip() - return str(artist_context).strip() - - def docker_resolve_path(path_str: str) -> str: """Resolve Docker-hosted Windows paths into container paths.""" if os.path.exists("/.dockerenv") and len(path_str) >= 3 and path_str[1] == ":" and path_str[0].isalpha(): @@ -147,161 +138,6 @@ def clean_track_title(track_title: str, artist_name: str) -> str: return cleaned if cleaned else original -def get_base_album_name(album_name: str) -> str: - """Extract the base album name without edition indicators.""" - base_name = album_name or "" - base_name = re.sub( - r"\s*[\[\(][^)\]]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^)\]]*[\]\)]\s*$", - "", - base_name, - flags=re.IGNORECASE, - ) - base_name = re.sub(r"\s*[\[\(][^)\]]*\bedition\b[^)\]]*[\]\)]\s*$", "", base_name, flags=re.IGNORECASE) - base_name = re.sub( - r"\s+(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", - "", - base_name, - flags=re.IGNORECASE, - ) - return base_name.strip() - - -def detect_deluxe_edition(album_name: str) -> bool: - """Detect if an album name indicates a deluxe/special edition.""" - if not album_name: - return False - - album_lower = album_name.lower() - deluxe_indicators = [ - "deluxe", - "deluxe edition", - "special edition", - "expanded edition", - "extended edition", - "bonus", - "remastered", - "anniversary", - "collectors edition", - "limited edition", - "silver edition", - "gold edition", - "platinum edition", - ] - for indicator in deluxe_indicators: - if indicator in album_lower: - logger.info("Detected deluxe edition: %r contains %r", album_name, indicator) - return True - return False - - -def normalize_base_album_name(base_album: str, artist_name: str) -> str: - """Normalize the base album name to handle case variations and known corrections.""" - normalized_lower = (base_album or "").lower().strip() - known_corrections = { - # Add specific album name corrections here as needed. - } - - for variant, correction in known_corrections.items(): - if normalized_lower == variant.lower(): - logger.info("Album correction applied: %r -> %r", base_album, correction) - return correction - - normalized = base_album or "" - normalized = re.sub(r"\s*&\s*", " & ", normalized) - normalized = re.sub(r"\s+", " ", normalized) - normalized = normalized.strip() - logger.info("Album variant normalization: %r -> %r", base_album, normalized) - return normalized - - -def clean_album_title(album_title: str, artist_name: str) -> str: - """Clean up album title by removing common prefixes, suffixes, and artist redundancy.""" - original = (album_title or "").strip() - cleaned = original - logger.info("Album Title Cleaning: %r (artist: %r)", original, artist_name) - - cleaned = re.sub(r"^Album\s*-\s*", "", cleaned, flags=re.IGNORECASE) - artist_pattern = re.escape(artist_name or "") + r"\s*-\s*" - cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE) - cleaned = re.sub(r"\s*[\[\(]\d{4}[\]\)]\s*", " ", cleaned) - - quality_patterns = [ - r"\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*", - r"\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*", - r"\s*[\[\(].*?flac.*?[\]\)]\s*", - r"\s*[\[\(].*?mp3.*?[\]\)]\s*", - r"\s*[\[\(].*?itunes.*?[\]\)]\s*", - r"\s*[\[\(].*?web.*?[\]\)]\s*", - r"\s*[\[\(].*?cd.*?[\]\)]\s*", - ] - for pattern in quality_patterns: - cleaned = re.sub(pattern, " ", cleaned, flags=re.IGNORECASE) - - cleaned = re.sub(r"\s*[\[\(][^\]\)]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE) - cleaned = re.sub(r"\s*[\[\(][^\]\)]*\bedition\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE) - cleaned = re.sub(r"\s*(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", "", cleaned, flags=re.IGNORECASE) - cleaned = re.sub(r"^[-\s\.]+", "", cleaned) - cleaned = re.sub(r"[-\s\.]+$", "", cleaned) - cleaned = re.sub(r"\s+", " ", cleaned).strip() - return cleaned if cleaned else original - - -def resolve_album_group(artist_context: dict, album_info: dict, original_album: str = None) -> str: - """Smart album grouping: upgrade to deluxe if any track is deluxe.""" - try: - with _album_cache_lock: - artist_name = _extract_artist_name(artist_context) - detected_album = (album_info or {}).get("album_name", "") - - if detected_album: - base_album = get_base_album_name(detected_album) - elif original_album: - cleaned_original = clean_album_title(original_album, artist_name) - base_album = get_base_album_name(cleaned_original) - else: - base_album = get_base_album_name(detected_album) - - base_album = normalize_base_album_name(base_album, artist_name) - album_key = f"{artist_name}::{base_album}" - is_deluxe_track = False - if detected_album: - is_deluxe_track = detect_deluxe_edition(detected_album) - elif original_album: - is_deluxe_track = detect_deluxe_edition(original_album) - - if album_key in _album_name_cache: - cached_name = _album_name_cache[album_key] - current_edition = _album_editions.get(album_key, "standard") - if is_deluxe_track and current_edition == "standard": - final_album_name = f"{base_album} (Deluxe Edition)" - _album_editions[album_key] = "deluxe" - _album_name_cache[album_key] = final_album_name - logger.info("Album cache upgrade: %r -> %r", album_key, final_album_name) - return final_album_name - logger.info("Using cached album name for %r: %r", album_key, cached_name) - return cached_name - - logger.info("Album grouping - Key: %r, Detected: %r", album_key, detected_album) - - current_edition = _album_editions.get(album_key, "standard") - if is_deluxe_track and current_edition == "standard": - logger.info("UPGRADE: Album %r upgraded from standard to deluxe!", base_album) - _album_editions[album_key] = "deluxe" - current_edition = "deluxe" - - if current_edition == "deluxe": - final_album_name = f"{base_album} (Deluxe Edition)" - else: - final_album_name = base_album - - _album_name_cache[album_key] = final_album_name - - logger.info("Album resolution: %r -> %r (edition: %s)", detected_album, final_album_name, current_edition) - return final_album_name - except Exception as e: - logger.error("Error resolving album group: %s", e) - album_name = (album_info or {}).get("album_name", "Unknown Album") - return album_name def get_album_type_display(raw_type, track_count) -> str: @@ -549,7 +385,7 @@ def build_final_path_for_track(context, artist_context, album_info, file_ext): album_context = get_import_context_album(context) source = get_import_source(context) playlist_folder_mode = track_info.get("_playlist_folder_mode", False) - artist_name = _extract_artist_name(artist_context) + artist_name = extract_artist_name(artist_context) source_info = track_info.get("source_info") or {} if isinstance(source_info, str): diff --git a/core/import_pipeline.py b/core/import_pipeline.py index ede3ae18..1574fe69 100644 --- a/core/import_pipeline.py +++ b/core/import_pipeline.py @@ -12,7 +12,6 @@ from core.import_file_ops import ( cleanup_empty_directories, create_lossy_copy, downsample_hires_flac, - extract_track_number_from_filename, get_audio_quality_string, get_quality_tier_from_extension, safe_move_file, @@ -29,6 +28,7 @@ from core.import_context import ( get_import_track_info, normalize_import_context, ) +from core.import_filename import extract_track_number_from_filename from core.import_guards import check_flac_bit_depth, move_to_quarantine from core.import_side_effects import ( check_and_remove_from_wishlist, @@ -61,8 +61,8 @@ from core.import_paths import ( build_final_path_for_track, build_simple_download_destination, docker_resolve_path, - resolve_album_group, ) +from core.import_album_naming import resolve_album_group from database.music_database import get_database from utils.logging_config import get_logger diff --git a/core/import_staging.py b/core/import_staging.py index da93580a..8f8fc6f0 100644 --- a/core/import_staging.py +++ b/core/import_staging.py @@ -4,9 +4,10 @@ from __future__ import annotations import os import threading -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple from core.import_paths import docker_resolve_path +from core.import_filename import extract_track_number_from_filename from utils.logging_config import get_logger logger = get_logger("import_staging") @@ -64,6 +65,71 @@ def get_client_for_source(source: str): return _get_client_for_source(source) +def read_staging_file_metadata(file_path: str, filename: Optional[str] = None) -> Dict[str, Any]: + """Read common audio tag metadata from a staging file.""" + try: + from mutagen import File as MutagenFile + + tags = MutagenFile(file_path, easy=True) + except Exception: + tags = None + + filename = filename or os.path.basename(file_path) + stem = os.path.splitext(os.path.basename(filename))[0] + + def _first_tag(*keys: str) -> str: + if not tags: + return "" + for key in keys: + try: + value = tags.get(key) # type: ignore[attr-defined] + except Exception: + value = None + if value: + if isinstance(value, (list, tuple)): + value = value[0] if value else "" + text = str(value).strip() + if text: + return text + return "" + + title = _first_tag("title") + artist = _first_tag("artist") + albumartist = _first_tag("albumartist") + album = _first_tag("album") + + if not title: + title = stem + if not albumartist: + albumartist = artist + + track_number = extract_track_number_from_filename(filename or file_path) + try: + # Preserve tag-based numbers when present, but still fall back to the filename parser. + tag_track_number = _first_tag("tracknumber", "track_number") + if tag_track_number: + track_number = int(str(tag_track_number).split("/")[0].strip() or track_number) + except (TypeError, ValueError): + pass + + disc_number = 1 + try: + tag_disc_number = _first_tag("discnumber", "disc_number") + if tag_disc_number: + disc_number = int(str(tag_disc_number).split("/")[0].strip() or 1) + except (TypeError, ValueError): + pass + + return { + "title": title, + "artist": artist, + "albumartist": albumartist, + "album": album, + "track_number": track_number, + "disc_number": disc_number, + } + + def _search_albums_for_source(source: str, client: Any, query: str, limit: int = 5): from core.metadata_service import _search_albums_for_source as _metadata_search_albums_for_source @@ -452,3 +518,39 @@ def refresh_import_suggestions_cache(): with _import_suggestions_cache_lock: _import_suggestions_cache["built"] = False start_import_suggestions_cache() + + +def collect_staging_files(file_paths: Optional[Iterable[str]] = None) -> List[Dict[str, Any]]: + """Collect audio files from the staging area with normalized metadata.""" + staging_path = get_staging_path() + file_filter: Optional[set[str]] = set(file_paths) if file_paths else None + staging_files: List[Dict[str, Any]] = [] + + if not os.path.isdir(staging_path): + return staging_files + + for root, _dirs, filenames in os.walk(staging_path): + for filename in filenames: + ext = os.path.splitext(filename)[1].lower() + if ext not in AUDIO_EXTENSIONS: + continue + + full_path = os.path.join(root, filename) + if file_filter is not None and full_path not in file_filter: + continue + + meta = read_staging_file_metadata(full_path, filename) + staging_files.append( + { + "filename": filename, + "full_path": full_path, + "title": meta.get("title", ""), + "artist": meta.get("albumartist") or meta.get("artist") or "", + "album": meta.get("album", ""), + "albumartist": meta.get("albumartist") or meta.get("artist") or "", + "track_number": meta.get("track_number", 1), + "disc_number": meta.get("disc_number", 1), + } + ) + + return staging_files diff --git a/tests/test_import_album.py b/tests/test_import_album.py index 81ab6fb6..45706da3 100644 --- a/tests/test_import_album.py +++ b/tests/test_import_album.py @@ -87,19 +87,22 @@ def test_build_album_import_match_payload_uses_generic_track_keys(monkeypatch, t staging_root.mkdir() (staging_root / "Song One.flac").write_text("fake") - monkeypatch.setattr(import_album, "get_staging_path", lambda: str(staging_root)) monkeypatch.setattr(import_album, "_get_matching_engine", lambda: _FakeEngine()) monkeypatch.setattr( import_album, - "read_staging_file_metadata", - lambda file_path, filename=None: { - "title": "Song One", - "artist": "Artist One", - "albumartist": "Artist One", - "album": "Album One", - "track_number": 1, - "disc_number": 1, - }, + "collect_staging_files", + lambda file_paths=None: [ + { + "filename": "Song One.flac", + "full_path": str(staging_root / "Song One.flac"), + "title": "Song One", + "artist": "Artist One", + "album": "Album One", + "albumartist": "Artist One", + "track_number": 1, + "disc_number": 1, + } + ], ) monkeypatch.setattr( import_album, diff --git a/tests/test_import_file_ops.py b/tests/test_import_file_ops.py index e9d72901..056c95bf 100644 --- a/tests/test_import_file_ops.py +++ b/tests/test_import_file_ops.py @@ -3,10 +3,10 @@ import types from core.import_file_ops import ( cleanup_empty_directories, - extract_track_number_from_filename, safe_move_file, - read_staging_file_metadata, ) +from core.import_filename import extract_track_number_from_filename +from core.import_staging import read_staging_file_metadata def test_extract_track_number_from_filename_handles_common_patterns(): diff --git a/tests/test_import_paths.py b/tests/test_import_paths.py index f9a39fab..a3e766f9 100644 --- a/tests/test_import_paths.py +++ b/tests/test_import_paths.py @@ -1,3 +1,4 @@ +import core.import_album_naming as album_naming import core.import_paths as import_paths @@ -73,8 +74,7 @@ def test_get_file_path_from_template_raw_handles_quality_and_disc_placeholders(m def test_resolve_album_group_upgrades_standard_to_deluxe(): - import_paths._album_name_cache.clear() - import_paths._album_editions.clear() + album_naming.clear_album_grouping_cache() artist_context = {"name": "Cache Artist"} standard_album = {"album_name": "Cache Album"} From edd9048f867364777f56485401d25a9cda3afc30 Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Fri, 24 Apr 2026 21:30:35 +0300 Subject: [PATCH 03/19] Checkpoint metadata runtime cleanup - remove runtime from metadata helper APIs where it only carried config, logger, mutagen, and database access - keep runtime only for the source-ID enrichment path that still needs live worker handles - add the new metadata helper modules and update the tests to match the slimmer interfaces --- core/import_resolution.py | 412 +++++++ core/metadata_artwork.py | 151 +++ core/metadata_common.py | 276 +++++ core/metadata_enrichment.py | 1374 +----------------------- core/metadata_lyrics.py | 63 ++ core/metadata_service.py | 384 +------ core/metadata_source.py | 896 ++++++++++++++++ tests/test_metadata_enrichment.py | 105 +- web_server.py | 1656 ++++++++++++++++++++++++++++- 9 files changed, 3535 insertions(+), 1782 deletions(-) create mode 100644 core/import_resolution.py create mode 100644 core/metadata_artwork.py create mode 100644 core/metadata_common.py create mode 100644 core/metadata_lyrics.py create mode 100644 core/metadata_source.py diff --git a/core/import_resolution.py b/core/import_resolution.py new file mode 100644 index 00000000..8f9c712c --- /dev/null +++ b/core/import_resolution.py @@ -0,0 +1,412 @@ +"""Single-track import lookup and context-building helpers.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from utils.logging_config import get_logger + + +logger = get_logger("import_resolution") + + +def _get_metadata_service(): + from core import metadata_service + + return metadata_service + + +def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any: + if value is None: + return default + + for name in names: + if isinstance(value, dict): + if name in value and value[name] is not None: + return value[name] + else: + candidate = getattr(value, name, None) + if candidate is not None: + return candidate + return default + + +def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]: + if not artists: + return [] + + if isinstance(artists, (str, bytes)): + artists = [artists] + elif isinstance(artists, dict): + artists = [artists] + else: + try: + artists = list(artists) + except TypeError: + artists = [artists] + + normalized: List[Dict[str, Any]] = [] + for artist in artists: + if isinstance(artist, dict): + name = _extract_lookup_value(artist, 'name', 'artist_name', 'title', default='') or '' + artist_id = _extract_lookup_value(artist, 'id', 'artist_id', default='') or '' + entry: Dict[str, Any] = {} + if name: + entry['name'] = str(name) + if artist_id: + entry['id'] = str(artist_id) + genres = _extract_lookup_value(artist, 'genres', default=None) + if genres is not None: + entry['genres'] = genres + if entry: + normalized.append(entry) + continue + + name = str(artist).strip() + if name: + normalized.append({'name': name}) + + return normalized + + +def _get_source_chain_for_lookup( + source_override: Optional[str] = None, + allow_fallback: bool = True, +) -> List[str]: + metadata_service = _get_metadata_service() + primary_source = metadata_service.get_primary_source() + source_chain = list(metadata_service.get_source_priority(primary_source)) + override = (source_override or '').strip().lower() + + if override: + source_chain = [override] + [source for source in source_chain if source != override] + + if not allow_fallback: + source_chain = source_chain[:1] + + return source_chain + + +def _build_track_search_query(source: str, title: str, artist: str) -> str: + base_query = " ".join(part for part in (title, artist) if part).strip() + if source == 'deezer' and title: + if artist: + return f'artist:"{artist}" track:"{title}"' + return f'track:"{title}"' + return base_query or title or artist + + +def _pick_best_track_match(search_results: List[Any], title: str, artist: str = '') -> Optional[Any]: + if not search_results: + return None + + target_title = str(title or '').strip().lower() + target_artist = str(artist or '').strip().lower() + + for candidate in search_results: + candidate_title = str(_extract_lookup_value(candidate, 'name', 'title', 'track_name', default='') or '').strip().lower() + if candidate_title != target_title: + continue + + if not target_artist: + return candidate + + candidate_artists = _normalize_context_artists(_extract_lookup_value(candidate, 'artists', default=[])) + candidate_artist_name = candidate_artists[0]['name'].strip().lower() if candidate_artists else '' + if candidate_artist_name == target_artist: + return candidate + + return search_results[0] + + +def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]: + if not client or not hasattr(client, 'search_tracks'): + return [] + + try: + kwargs = {'limit': limit} + if source == 'spotify': + kwargs['allow_fallback'] = False + return client.search_tracks(query, **kwargs) or [] + except Exception as exc: + logger.debug("Could not search %s for %s: %s", source, query, exc) + return [] + + +def _build_single_import_context_payload( + track_data: Any, + source: Optional[str], + source_priority: List[str], + requested_title: str = '', + requested_artist: str = '', +) -> Dict[str, Any]: + album_data = _extract_lookup_value(track_data, 'album', default=None) + + track_id = str(_extract_lookup_value(track_data, 'id', 'track_id', 'trackId', default='') or '') + track_name = _extract_lookup_value(track_data, 'name', 'title', 'trackName', default='') or requested_title or 'Unknown Track' + track_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'artists', default=[])) + if not track_artists and requested_artist: + track_artists = [{'name': requested_artist}] + + primary_track_artist = track_artists[0] if track_artists else {} + primary_artist_name = primary_track_artist.get('name') or requested_artist or 'Unknown Artist' + primary_artist_id = str(primary_track_artist.get('id', '') or _extract_lookup_value(track_data, 'artist_id', 'artistId', default='') or '') + + album_name = _extract_lookup_value(track_data, 'album_name', 'collectionName', default='') or '' + album_id = str(_extract_lookup_value(track_data, 'album_id', 'collectionId', 'albumId', default='') or '') + release_date = str(_extract_lookup_value(track_data, 'release_date', default='') or '') + album_type = str(_extract_lookup_value(track_data, 'album_type', default='album') or 'album') + total_tracks = int(_extract_lookup_value(track_data, 'total_tracks', 'track_count', default=0) or 0) + album_images: List[Dict[str, Any]] = [] + album_image_url = str(_extract_lookup_value(track_data, 'image_url', 'thumb_url', default='') or '') + album_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'album_artists', 'artists', default=[])) + + if isinstance(album_data, dict): + album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name + album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id) + release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date) + album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type) + total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks) + album_images = _extract_lookup_value(album_data, 'images', default=[]) or [] + if not album_image_url: + album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '') + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) + elif album_data: + album_name = album_name or str(album_data) + + if not album_artists and primary_artist_name: + album_artists = [{'name': primary_artist_name}] + + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + + track_info = { + 'id': track_id, + 'name': track_name, + 'track_number': int(_extract_lookup_value(track_data, 'track_number', 'trackNumber', default=1) or 1), + 'disc_number': int(_extract_lookup_value(track_data, 'disc_number', 'discNumber', default=1) or 1), + 'duration_ms': int(_extract_lookup_value(track_data, 'duration_ms', 'duration', 'trackTimeMillis', default=0) or 0), + 'artists': track_artists or [{'name': primary_artist_name}], + 'uri': str(_extract_lookup_value(track_data, 'uri', default='') or ''), + 'album': album_name, + 'album_id': album_id, + 'album_type': album_type, + 'release_date': release_date, + '_source': source or '', + } + + album_payload = { + 'id': album_id, + 'name': album_name, + 'release_date': release_date, + 'total_tracks': total_tracks or 1, + 'album_type': album_type, + 'image_url': album_image_url, + 'images': album_images, + 'artists': album_artists, + '_source': source or '', + } + + artist_payload = { + 'id': primary_artist_id, + 'name': primary_artist_name, + 'genres': [], + '_source': source or '', + } + + original_search = { + 'title': track_name, + 'artist': primary_artist_name, + 'album': album_name, + 'track_number': track_info['track_number'], + 'disc_number': track_info['disc_number'], + 'clean_title': track_name, + 'clean_album': album_name, + 'clean_artist': primary_artist_name, + 'artists': track_info['artists'], + 'duration_ms': track_info['duration_ms'], + 'id': track_id, + '_source': source or '', + } + + return { + 'success': bool(track_id or track_name != requested_title or album_name), + 'source': source, + 'source_priority': source_priority, + 'context': { + 'artist': artist_payload, + 'album': album_payload, + 'track_info': track_info, + 'original_search_result': original_search, + 'is_album_download': False, + 'has_clean_metadata': bool(track_id), + 'has_full_metadata': bool(track_id), + 'source': source, + 'source_priority': source_priority, + }, + } + + +def _build_single_import_fallback_context( + requested_title: str, + requested_artist: str, + source_priority: List[str], +) -> Dict[str, Any]: + artist_name = requested_artist or 'Unknown Artist' + title = requested_title or 'Unknown Track' + return { + 'success': False, + 'source': None, + 'source_priority': source_priority, + 'context': { + 'artist': { + 'id': '', + 'name': artist_name, + 'genres': [], + '_source': '', + }, + 'album': { + 'id': '', + 'name': '', + 'release_date': '', + 'total_tracks': 1, + 'album_type': 'album', + 'image_url': '', + 'images': [], + 'artists': [], + '_source': '', + }, + 'track_info': { + 'id': '', + 'name': title, + 'track_number': 1, + 'disc_number': 1, + 'duration_ms': 0, + 'artists': [{'name': artist_name}], + 'uri': '', + 'album': '', + 'album_id': '', + 'album_type': 'album', + 'release_date': '', + '_source': '', + }, + 'original_search_result': { + 'title': title, + 'artist': artist_name, + 'album': '', + 'track_number': 1, + 'disc_number': 1, + 'clean_title': title, + 'clean_album': '', + 'clean_artist': artist_name, + 'artists': [{'name': artist_name}], + 'duration_ms': 0, + 'id': '', + '_source': '', + }, + 'is_album_download': False, + 'has_clean_metadata': False, + 'has_full_metadata': False, + 'source': None, + 'source_priority': source_priority, + }, + } + + +def get_single_track_import_context( + title: str, + artist: str = '', + override_id: Optional[str] = None, + override_source: str = 'spotify', + source_override: Optional[str] = None, +) -> Dict[str, Any]: + """Build an import context for singles using source-priority metadata lookup.""" + metadata_service = _get_metadata_service() + source_priority = _get_source_chain_for_lookup(source_override=source_override, allow_fallback=True) + title = (title or '').strip() + artist = (artist or '').strip() + + if override_id: + chosen_source = (override_source or 'spotify').strip().lower() or 'spotify' + client = metadata_service.get_client_for_source(chosen_source) + if client and hasattr(client, 'get_track_details'): + try: + track_data = client.get_track_details(str(override_id)) + if track_data: + payload = _build_single_import_context_payload( + track_data, + chosen_source, + source_priority, + requested_title=title, + requested_artist=artist, + ) + if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): + try: + artist_details = client.get_artist(payload['context']['artist']['id']) + if artist_details: + payload['context']['artist']['genres'] = _extract_lookup_value( + artist_details, + 'genres', + default=[], + ) or [] + except Exception: + pass + return payload + except Exception as exc: + logger.debug("Override track lookup failed on %s for %s: %s", chosen_source, override_id, exc) + + for source in source_priority: + client = metadata_service.get_client_for_source(source) + if not client: + continue + + search_query = _build_track_search_query(source, title, artist) + if not search_query: + continue + + search_results = _search_tracks_for_source(source, client, search_query, limit=5) + if not search_results and search_query != title: + search_results = _search_tracks_for_source(source, client, title, limit=5) + if not search_results and artist and search_query != artist: + search_results = _search_tracks_for_source(source, client, artist, limit=5) + + if not search_results: + continue + + best_match = _pick_best_track_match(search_results, title or search_query, artist) + if not best_match: + continue + + resolved_track_id = str(_extract_lookup_value(best_match, 'id', 'track_id', 'trackId', default='') or '') + resolved_data = best_match + if resolved_track_id and hasattr(client, 'get_track_details'): + try: + detailed = client.get_track_details(resolved_track_id) + if detailed: + resolved_data = detailed + except Exception as exc: + logger.debug("Track detail lookup failed on %s for %s: %s", source, resolved_track_id, exc) + + payload = _build_single_import_context_payload( + resolved_data, + source, + source_priority, + requested_title=title, + requested_artist=artist, + ) + if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): + try: + artist_details = client.get_artist(payload['context']['artist']['id']) + if artist_details: + payload['context']['artist']['genres'] = _extract_lookup_value( + artist_details, + 'genres', + default=[], + ) or [] + except Exception: + pass + return payload + + return _build_single_import_fallback_context(title, artist, source_priority) diff --git a/core/metadata_artwork.py b/core/metadata_artwork.py new file mode 100644 index 00000000..4c83f325 --- /dev/null +++ b/core/metadata_artwork.py @@ -0,0 +1,151 @@ +"""Album artwork helpers for metadata enrichment.""" + +from __future__ import annotations + +import os +import re +import urllib.request + +from core.import_context import get_import_context_album +from core.metadata_common import ( + _get_config_manager, + _get_image_dimensions, + _get_logger, + _get_mutagen_symbols, +) + + +def embed_album_art_metadata(audio_file, metadata: dict): + cfg = _get_config_manager() + logger_ = _get_logger() + symbols = _get_mutagen_symbols() + if not symbols: + return + + try: + image_data = None + mime_type = None + + release_mbid = metadata.get("musicbrainz_release_id") + if release_mbid and cfg.get("metadata_enhancement.prefer_caa_art", False): + try: + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) + with urllib.request.urlopen(req, timeout=10) as response: + image_data = response.read() + mime_type = response.info().get_content_type() or "image/jpeg" + if not image_data or len(image_data) <= 1000: + image_data = None + except Exception: + image_data = None + + if not image_data: + art_url = metadata.get("album_art_url") + if not art_url: + logger_.warning("No album art URL available for embedding.") + return + with urllib.request.urlopen(art_url, timeout=10) as response: + image_data = response.read() + mime_type = response.info().get_content_type() or "image/jpeg" + + if not image_data: + logger_.error("Failed to download album art data.") + return + + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.APIC(encoding=3, mime=mime_type, type=3, desc="Cover", data=image_data)) + elif isinstance(audio_file, symbols.FLAC): + picture = symbols.Picture() + picture.data = image_data + picture.type = 3 + picture.mime = mime_type + width, height = _get_image_dimensions(image_data) + picture.width = width or 640 + picture.height = height or 640 + picture.depth = 24 + audio_file.add_picture(picture) + elif isinstance(audio_file, symbols.MP4): + fmt = symbols.MP4Cover.FORMAT_JPEG if "jpeg" in mime_type else symbols.MP4Cover.FORMAT_PNG + audio_file["covr"] = [symbols.MP4Cover(image_data, imageformat=fmt)] + + logger_.info("Album art successfully embedded.") + except Exception as exc: + logger_.error("Error embedding album art: %s", exc) + + +def download_cover_art(album_info: dict, target_dir: str, context: dict = None): + cfg = _get_config_manager() + logger_ = _get_logger() + if cfg.get("metadata_enhancement.cover_art_download", True) is False: + return + + try: + cover_path = os.path.join(target_dir, "cover.jpg") + album_info = album_info or {} + release_mbid = album_info.get("musicbrainz_release_id") + prefer_caa = cfg.get("metadata_enhancement.prefer_caa_art", False) + + if os.path.exists(cover_path): + if release_mbid and prefer_caa: + try: + existing_size = os.path.getsize(cover_path) + if existing_size > 200_000: + return + is_upgrade = True + except Exception: + return + else: + return + else: + is_upgrade = False + + image_data = None + if release_mbid and prefer_caa: + try: + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) + with urllib.request.urlopen(req, timeout=10) as response: + image_data = response.read() + if not image_data or len(image_data) <= 1000: + image_data = None + except Exception: + image_data = None + + if is_upgrade and not image_data: + logger_.error("CAA upgrade failed - keeping existing cover.jpg") + return + + if not image_data: + art_url = album_info.get("album_image_url") + if not art_url and context: + album_ctx = get_import_context_album(context) + art_url = album_ctx.get("image_url") + if not art_url and album_ctx.get("images"): + images = album_ctx.get("images", []) + if images and isinstance(images[0], dict): + art_url = images[0].get("url", "") + if art_url: + logger_.info("Using cover art URL from album context") + if art_url and "i.scdn.co" in art_url: + try: + from core.spotify_client import _upgrade_spotify_image_url + + art_url = _upgrade_spotify_image_url(art_url) + except Exception: + pass + elif art_url and "mzstatic.com" in art_url: + art_url = re.sub(r"\d+x\d+bb", "3000x3000bb", art_url) + if not art_url: + logger_.warning("No cover art URL available for download.") + return + with urllib.request.urlopen(art_url, timeout=10) as response: + image_data = response.read() + + if not image_data: + return + + with open(cover_path, "wb") as handle: + handle.write(image_data) + logger_.info("Cover art downloaded to: %s", cover_path) + except Exception as exc: + logger_.error("Error downloading cover.jpg: %s", exc) diff --git a/core/metadata_common.py b/core/metadata_common.py new file mode 100644 index 00000000..975a8e36 --- /dev/null +++ b/core/metadata_common.py @@ -0,0 +1,276 @@ +"""Shared low-level helpers for metadata enrichment.""" + +from __future__ import annotations + +import os +import threading +from types import SimpleNamespace +from typing import Any, Dict + +from utils.logging_config import get_logger + + +logger = get_logger("metadata_enrichment") + +_FILE_LOCKS: Dict[str, threading.Lock] = {} +_FILE_LOCKS_LOCK = threading.Lock() + + +class _NullConfigManager: + def get(self, _key: str, default: Any = None) -> Any: + return default + + +def _get_logger(): + return logger + + +def _get_config_manager(): + try: + from config.settings import config_manager as settings_config_manager + + return settings_config_manager + except Exception: + return _NullConfigManager() + + +def _get_database(): + try: + from database.music_database import get_database + + return get_database() + except Exception: + return None + + +def _get_itunes_client(): + try: + from core.metadata_service import get_itunes_client + + return get_itunes_client() + except Exception: + return None + + +def _extract_artist_name(artist: Any) -> str: + if isinstance(artist, dict): + return str(artist.get("name", "") or "") + if hasattr(artist, "name"): + return str(getattr(artist, "name") or "") + return str(artist) if artist else "" + + +def _get_mutagen_symbols(): + """Lazy mutagen import so tests can monkeypatch this without the package installed.""" + try: + from mutagen import File as MutagenFile + from mutagen.apev2 import APEv2, APENoHeaderError + from mutagen.flac import FLAC, Picture + from mutagen.id3 import ( + APIC, + ID3, + TBPM, + TCOP, + TDOR, + TDRC, + TCON, + TIT2, + TALB, + TPE1, + TPE2, + TPOS, + TPUB, + TRCK, + TSRC, + TXXX, + UFID, + TMED, + ) + from mutagen.mp4 import MP4, MP4Cover, MP4FreeForm + from mutagen.oggvorbis import OggVorbis + try: + from mutagen.oggopus import OggOpus + except Exception: + OggOpus = None + except Exception as exc: + logger.debug("Mutagen unavailable for metadata enrichment: %s", exc) + return None + + return SimpleNamespace( + File=MutagenFile, + APEv2=APEv2, + APENoHeaderError=APENoHeaderError, + FLAC=FLAC, + Picture=Picture, + ID3=ID3, + APIC=APIC, + TBPM=TBPM, + TCOP=TCOP, + TDOR=TDOR, + TDRC=TDRC, + TCON=TCON, + TIT2=TIT2, + TALB=TALB, + TPE1=TPE1, + TPE2=TPE2, + TPOS=TPOS, + TPUB=TPUB, + TRCK=TRCK, + TSRC=TSRC, + TXXX=TXXX, + UFID=UFID, + TMED=TMED, + MP4=MP4, + MP4Cover=MP4Cover, + MP4FreeForm=MP4FreeForm, + OggVorbis=OggVorbis, + OggOpus=OggOpus, + ) + + +def _get_file_lock(file_path: str) -> threading.Lock: + with _FILE_LOCKS_LOCK: + lock = _FILE_LOCKS.get(file_path) + if lock is None: + lock = threading.Lock() + _FILE_LOCKS[file_path] = lock + return lock + + +def _is_ogg_opus(audio_file: Any) -> bool: + return type(audio_file).__name__ == "OggOpus" + + +def _is_vorbis_like(audio_file: Any, symbols: Any) -> bool: + vorbis_classes = tuple( + cls for cls in ( + getattr(symbols, "FLAC", None), + getattr(symbols, "OggVorbis", None), + ) if cls is not None + ) + return bool(vorbis_classes) and isinstance(audio_file, vorbis_classes) or _is_ogg_opus(audio_file) + + +def _save_audio_file(audio_file: Any, symbols: Any) -> None: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.save(v1=0, v2_version=4) + elif isinstance(audio_file, symbols.FLAC): + audio_file.save(deleteid3=True) + else: + audio_file.save() + + +def _get_image_dimensions(data: bytes): + try: + if data[:8] == b"\x89PNG\r\n\x1a\n": + import struct + + w, h = struct.unpack(">II", data[16:24]) + return w, h + if data[:2] == b"\xff\xd8": + import struct + + i = 2 + while i < len(data) - 9: + if data[i] != 0xFF: + break + marker = data[i + 1] + if marker in (0xC0, 0xC2): + h, w = struct.unpack(">HH", data[i + 5 : i + 9]) + return w, h + length = struct.unpack(">H", data[i + 2 : i + 4])[0] + i += 2 + length + except Exception: + pass + return None, None + + +def _strip_all_non_audio_tags(file_path: str) -> dict: + summary = {"apev2_stripped": False, "apev2_tag_count": 0} + if os.path.splitext(file_path)[1].lower() != ".mp3": + return summary + + symbols = _get_mutagen_symbols() + if not symbols: + return summary + + try: + apev2_tags = symbols.APEv2(file_path) + tag_count = len(apev2_tags) + tag_keys = list(apev2_tags.keys()) + apev2_tags.delete(file_path) + summary["apev2_stripped"] = True + summary["apev2_tag_count"] = tag_count + logger.info("Stripped %s APEv2 tags: %s", tag_count, ", ".join(tag_keys[:10])) + except symbols.APENoHeaderError: + pass + except Exception as exc: + logger.error("Could not strip APEv2 tags (non-fatal): %s", exc) + return summary + + +def _verify_metadata_written(file_path: str) -> bool: + symbols = _get_mutagen_symbols() + if not symbols: + return False + + try: + check = symbols.File(file_path) + if check is None or check.tags is None: + logger.info("[VERIFY] Tags are None after save: %s", file_path) + return False + + title_found = False + artist_found = False + if isinstance(check.tags, symbols.ID3): + title_found = bool(check.tags.getall("TIT2")) + artist_found = bool(check.tags.getall("TPE1")) + try: + symbols.APEv2(file_path) + logger.info("[VERIFY] APEv2 tags still present after processing!") + return False + except symbols.APENoHeaderError: + pass + elif _is_vorbis_like(check, symbols): + title_found = bool(check.get("title")) + artist_found = bool(check.get("artist")) + elif isinstance(check, symbols.MP4): + title_found = bool(check.get("\xa9nam")) + artist_found = bool(check.get("\xa9ART")) + + if not title_found or not artist_found: + logger.warning("[VERIFY] Missing metadata - title:%s artist:%s", title_found, artist_found) + return False + + logger.info("[VERIFY] Metadata verified OK") + return True + except Exception as exc: + logger.error("[VERIFY] Verification error (non-fatal): %s", exc) + return False + + +def wipe_source_tags(file_path: str) -> bool: + try: + _strip_all_non_audio_tags(file_path) + symbols = _get_mutagen_symbols() + if not symbols: + return False + + audio = symbols.File(file_path) + if audio is None: + return False + if hasattr(audio, "clear_pictures"): + audio.clear_pictures() + if audio.tags is not None: + tag_count = len(audio.tags) + audio.tags.clear() + else: + audio.add_tags() + tag_count = 0 + _save_audio_file(audio, symbols) + if tag_count > 0: + logger.info("[Tag Wipe] Stripped %s source tags from: %s", tag_count, os.path.basename(file_path)) + return True + except Exception as exc: + logger.error("[Tag Wipe] Failed (non-fatal): %s", exc) + return False diff --git a/core/metadata_enrichment.py b/core/metadata_enrichment.py index 293af162..89651f1d 100644 --- a/core/metadata_enrichment.py +++ b/core/metadata_enrichment.py @@ -1,1350 +1,56 @@ -"""Source-aware metadata enrichment helpers for imported audio files.""" +"""Compatibility facade and orchestration for metadata enrichment.""" from __future__ import annotations -import json import os -import re -import threading -import time -import urllib.request -from pathlib import Path -from types import SimpleNamespace -from typing import Any, Dict, Optional -from core.import_context import ( - get_import_clean_album, - get_import_clean_artist, - get_import_clean_title, - get_import_context_album, - get_import_context_artist, - get_import_original_search, - get_import_source, - get_import_source_ids, - get_import_track_info, - get_source_tag_names, - normalize_import_context, +from core.metadata_artwork import ( + download_cover_art as _download_cover_art_impl, + embed_album_art_metadata, ) -from config.settings import config_manager -from core.metadata_service import get_itunes_client -from database.music_database import get_database -from utils.logging_config import get_logger - - -logger = get_logger("metadata_enrichment") - -_FILE_LOCKS: Dict[str, threading.Lock] = {} -_FILE_LOCKS_LOCK = threading.Lock() - -_MB_RELEASE_CACHE: Dict[tuple, str] = {} -_MB_RELEASE_CACHE_LOCK = threading.RLock() -_MB_RELEASE_DETAIL_CACHE: Dict[str, Dict[str, Any]] = {} -_MB_RELEASE_DETAIL_CACHE_LOCK = threading.RLock() - -_EDITION_PAREN_RE = re.compile( - r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' - r'limited|bonus|platinum|gold|super\s*deluxe|standard)' - r'(?:\s+(?:edition|version))?[^)\]]*[\)\]]', - re.IGNORECASE, -) -_EDITION_BARE_RE = re.compile( - r'\s+(?:-\s+)?(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' - r'limited|bonus|platinum|gold|super\s*deluxe|standard)' - r'(?:\s+(?:edition|version))?\s*$', - re.IGNORECASE, +from core.metadata_common import ( + _get_config_manager, + _get_file_lock, + _get_image_dimensions as _common_get_image_dimensions, + _get_logger, + _get_mutagen_symbols, + _is_ogg_opus as _common_is_ogg_opus, + _is_vorbis_like, + _save_audio_file, + _strip_all_non_audio_tags, + _verify_metadata_written, + wipe_source_tags as _common_wipe_source_tags, ) +from core.metadata_lyrics import generate_lrc_file as _generate_lrc_file_impl +from core.metadata_source import embed_source_ids, extract_source_metadata -class _NullConfigManager: - def get(self, _key: str, default: Any = None) -> Any: - return default - - -def _get_logger(runtime=None): - return logger - - -def _get_config_manager(runtime=None): - return config_manager - - -def _get_database(runtime=None): - try: - return get_database() - except Exception: - return None - - -def _get_itunes_client(runtime=None): - try: - return get_itunes_client() - except Exception: - worker = getattr(runtime, "itunes_enrichment_worker", None) - if worker and getattr(worker, "client", None): - return worker.client - return getattr(runtime, "itunes_client", None) - - -def _extract_artist_name(artist: Any) -> str: - if isinstance(artist, dict): - return str(artist.get("name", "") or "") - if hasattr(artist, "name"): - return str(getattr(artist, "name") or "") - return str(artist) if artist else "" - - -def _get_mutagen_symbols(runtime=None): - """Lazy mutagen import so tests can monkeypatch this without the package installed.""" - try: - from mutagen import File as MutagenFile - from mutagen.apev2 import APEv2, APENoHeaderError - from mutagen.flac import FLAC, Picture - from mutagen.id3 import ( - APIC, - ID3, - TBPM, - TCOP, - TDOR, - TDRC, - TCON, - TIT2, - TALB, - TPE1, - TPE2, - TPOS, - TPUB, - TRCK, - TSRC, - TXXX, - UFID, - TMED, - ) - from mutagen.mp4 import MP4, MP4Cover, MP4FreeForm - from mutagen.oggvorbis import OggVorbis - try: - from mutagen.oggopus import OggOpus - except Exception: - OggOpus = None - except Exception as exc: - _get_logger(runtime).debug("Mutagen unavailable for metadata enrichment: %s", exc) - return None - - return SimpleNamespace( - File=MutagenFile, - APEv2=APEv2, - APENoHeaderError=APENoHeaderError, - FLAC=FLAC, - Picture=Picture, - ID3=ID3, - APIC=APIC, - TBPM=TBPM, - TCOP=TCOP, - TDOR=TDOR, - TDRC=TDRC, - TCON=TCON, - TIT2=TIT2, - TALB=TALB, - TPE1=TPE1, - TPE2=TPE2, - TPOS=TPOS, - TPUB=TPUB, - TRCK=TRCK, - TSRC=TSRC, - TXXX=TXXX, - UFID=UFID, - TMED=TMED, - MP4=MP4, - MP4Cover=MP4Cover, - MP4FreeForm=MP4FreeForm, - OggVorbis=OggVorbis, - OggOpus=OggOpus, - ) - - -def _get_file_lock(file_path: str) -> threading.Lock: - with _FILE_LOCKS_LOCK: - lock = _FILE_LOCKS.get(file_path) - if lock is None: - lock = threading.Lock() - _FILE_LOCKS[file_path] = lock - return lock - - -def _is_ogg_opus(audio_file: Any) -> bool: - return type(audio_file).__name__ == "OggOpus" - - -def _is_vorbis_like(audio_file: Any, symbols: Any) -> bool: - vorbis_classes = tuple( - cls for cls in ( - getattr(symbols, "FLAC", None), - getattr(symbols, "OggVorbis", None), - ) if cls is not None - ) - return bool(vorbis_classes) and isinstance(audio_file, vorbis_classes) or _is_ogg_opus(audio_file) - - -def _save_audio_file(audio_file: Any, symbols: Any) -> None: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.save(v1=0, v2_version=4) - elif isinstance(audio_file, symbols.FLAC): - audio_file.save(deleteid3=True) - else: - audio_file.save() - - -def _strip_all_non_audio_tags(file_path: str, runtime=None) -> dict: - summary = {"apev2_stripped": False, "apev2_tag_count": 0} - if os.path.splitext(file_path)[1].lower() != ".mp3": - return summary - - symbols = _get_mutagen_symbols(runtime) - if not symbols: - return summary - - try: - apev2_tags = symbols.APEv2(file_path) - tag_count = len(apev2_tags) - tag_keys = list(apev2_tags.keys()) - apev2_tags.delete(file_path) - summary["apev2_stripped"] = True - summary["apev2_tag_count"] = tag_count - _get_logger(runtime).info("Stripped %s APEv2 tags: %s", tag_count, ", ".join(tag_keys[:10])) - except symbols.APENoHeaderError: - pass - except Exception as exc: - _get_logger(runtime).error("Could not strip APEv2 tags (non-fatal): %s", exc) - return summary - - -def _verify_metadata_written(file_path: str, runtime=None) -> bool: - symbols = _get_mutagen_symbols(runtime) - if not symbols: - return False - - try: - check = symbols.File(file_path) - if check is None or check.tags is None: - _get_logger(runtime).info("[VERIFY] Tags are None after save: %s", file_path) - return False - - title_found = False - artist_found = False - if isinstance(check.tags, symbols.ID3): - title_found = bool(check.tags.getall("TIT2")) - artist_found = bool(check.tags.getall("TPE1")) - try: - symbols.APEv2(file_path) - _get_logger(runtime).info("[VERIFY] APEv2 tags still present after processing!") - return False - except symbols.APENoHeaderError: - pass - elif _is_vorbis_like(check, symbols): - title_found = bool(check.get("title")) - artist_found = bool(check.get("artist")) - elif isinstance(check, symbols.MP4): - title_found = bool(check.get("\xa9nam")) - artist_found = bool(check.get("\xa9ART")) - - if not title_found or not artist_found: - _get_logger(runtime).warning("[VERIFY] Missing metadata - title:%s artist:%s", title_found, artist_found) - return False - - _get_logger(runtime).info("[VERIFY] Metadata verified OK") - return True - except Exception as exc: - _get_logger(runtime).error("[VERIFY] Verification error (non-fatal): %s", exc) - return False +logger = _get_logger() def _get_image_dimensions(data: bytes): - try: - if data[:8] == b"\x89PNG\r\n\x1a\n": - import struct + return _common_get_image_dimensions(data) - w, h = struct.unpack(">II", data[16:24]) - return w, h - if data[:2] == b"\xff\xd8": - import struct - i = 2 - while i < len(data) - 9: - if data[i] != 0xFF: - break - marker = data[i + 1] - if marker in (0xC0, 0xC2): - h, w = struct.unpack(">HH", data[i + 5 : i + 9]) - return w, h - length = struct.unpack(">H", data[i + 2 : i + 4])[0] - i += 2 + length - except Exception: - pass - return None, None +def _is_ogg_opus(audio_file): + return _common_is_ogg_opus(audio_file) -def extract_source_metadata(context: dict, artist: dict, album_info: dict, runtime=None) -> dict: - if album_info is None: - album_info = {} +def wipe_source_tags(file_path: str) -> bool: + return _common_wipe_source_tags(file_path) - cfg = _get_config_manager(runtime) - context = normalize_import_context(context) - original_search = get_import_original_search(context) - album_ctx = get_import_context_album(context) - track_info = get_import_track_info(context) - source = get_import_source(context) - source_ids = get_import_source_ids(context) - artist_dict = artist if isinstance(artist, dict) else { - "name": _extract_artist_name(artist), - "id": getattr(artist, "id", ""), - "genres": list(getattr(artist, "genres", []) or []), - } +def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: + return _generate_lrc_file_impl(file_path, context, artist, album_info) - metadata: Dict[str, Any] = { - "source": source, - "source_track_id": source_ids["track_id"], - "source_artist_id": source_ids["artist_id"], - "source_album_id": source_ids["album_id"], - } - metadata["title"] = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "")) - if original_search.get("clean_title"): - _get_logger(runtime).info("Metadata: Using clean title: '%s'", metadata["title"]) - elif album_info.get("clean_track_name"): - _get_logger(runtime).info("Metadata: Using album info clean name: '%s'", metadata["title"]) - else: - _get_logger(runtime).warning("Metadata: Using original title as fallback: '%s'", metadata["title"]) +def download_cover_art(album_info: dict, target_dir: str, context: dict = None): + return _download_cover_art_impl(album_info, target_dir, context) - artists = original_search.get("artists") - if isinstance(artists, list) and artists: - all_artists = [] - for artist_item in artists: - if isinstance(artist_item, dict) and artist_item.get("name"): - all_artists.append(artist_item["name"]) - elif isinstance(artist_item, str): - all_artists.append(artist_item) - else: - all_artists.append(str(artist_item)) - metadata["artist"] = ", ".join(all_artists) - _get_logger(runtime).info("Metadata: Using all artists: '%s'", metadata["artist"]) - else: - metadata["artist"] = artist_dict.get("name", "") or get_import_clean_artist(context) - _get_logger(runtime).info("Metadata: Using primary artist: '%s'", metadata["artist"]) - raw_album_artist = artist_dict.get("name", "") or metadata["artist"] - track_info_ctx = track_info or {} - explicit_artist = track_info_ctx.get("_explicit_artist_context") if isinstance(track_info_ctx, dict) else None - album_artists_for_collab = None - - if isinstance(explicit_artist, dict) and explicit_artist.get("name"): - raw_album_artist = explicit_artist["name"] - album_artists_for_collab = [explicit_artist] - elif isinstance(explicit_artist, str) and explicit_artist: - raw_album_artist = explicit_artist - album_artists_for_collab = [{"name": explicit_artist}] - elif album_ctx and isinstance(album_ctx, dict): - album_artists = album_ctx.get("artists", []) - if album_artists: - first_album_artist = album_artists[0] - if isinstance(first_album_artist, dict) and first_album_artist.get("name"): - raw_album_artist = first_album_artist["name"] - elif isinstance(first_album_artist, str) and first_album_artist: - raw_album_artist = first_album_artist - album_artists_for_collab = album_artists - - collab_mode = cfg.get("file_organization.collab_artist_mode", "first") - if collab_mode == "first" and raw_album_artist: - context_artists = album_artists_for_collab or original_search.get("artists") or track_info_ctx.get("artists") or [] - if len(context_artists) > 1: - first = context_artists[0] - raw_album_artist = first.get("name", first) if isinstance(first, dict) else str(first) - elif len(context_artists) == 1 and ("," in raw_album_artist or " & " in raw_album_artist): - artist_id = str(artist_dict.get("id", "")) - if source == "itunes" and artist_id.isdigit(): - try: - itunes_client = _get_itunes_client(runtime) - if itunes_client and hasattr(itunes_client, "resolve_primary_artist"): - resolved = itunes_client.resolve_primary_artist(artist_id) - if resolved and resolved != raw_album_artist: - raw_album_artist = resolved - except Exception: - pass - metadata["album_artist"] = raw_album_artist - - if album_info.get("is_album"): - metadata["album"] = album_info.get("album_name", "Unknown Album") - metadata["track_number"] = album_info.get("track_number", 1) - metadata["total_tracks"] = album_ctx.get("total_tracks", 1) if album_ctx else 1 - _get_logger(runtime).info("[METADATA] Album track - track_number: %s, album: %s", metadata["track_number"], metadata["album"]) - else: - if album_ctx and album_ctx.get("name"): - _get_logger(runtime).info("[SAFEGUARD] Using album context name instead of track title for album metadata") - metadata["album"] = album_ctx["name"] - metadata["track_number"] = album_info.get("track_number", 1) if album_info else 1 - metadata["total_tracks"] = album_ctx.get("total_tracks", 1) - else: - metadata["album"] = metadata["title"] - metadata["track_number"] = 1 - metadata["total_tracks"] = 1 - - disc_num = original_search.get("disc_number") - if disc_num is None and album_info: - disc_num = album_info.get("disc_number") - metadata["disc_number"] = disc_num if disc_num is not None else 1 - - if album_ctx and album_ctx.get("release_date"): - metadata["date"] = album_ctx["release_date"][:4] - - genres = artist_dict.get("genres") or [] - if genres: - from core.genre_filter import filter_genres - - filtered = filter_genres(list(genres[:2]), cfg) - if filtered: - metadata["genre"] = ", ".join(filtered) - - metadata["album_art_url"] = album_info.get("album_image_url") if album_info else None - if not metadata["album_art_url"] and album_ctx: - album_image = album_ctx.get("image_url") - if not album_image and album_ctx.get("images"): - first_image = album_ctx["images"][0] - album_image = first_image.get("url") if isinstance(first_image, dict) else None - metadata["album_art_url"] = album_image - - _get_logger(runtime).info( - "[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | track=%s/%s | disc=%s", - metadata.get("title"), - metadata.get("artist"), - metadata.get("album_artist"), - metadata.get("album"), - metadata.get("track_number"), - metadata.get("total_tracks"), - metadata.get("disc_number"), - ) - - return metadata - - -def embed_album_art_metadata(audio_file, metadata: dict, runtime=None): - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) - symbols = _get_mutagen_symbols(runtime) - if not symbols: - return - - try: - image_data = None - mime_type = None - - release_mbid = metadata.get("musicbrainz_release_id") - if release_mbid and cfg.get("metadata_enhancement.prefer_caa_art", False): - try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" - req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) - with urllib.request.urlopen(req, timeout=10) as response: - image_data = response.read() - mime_type = response.info().get_content_type() or "image/jpeg" - if not image_data or len(image_data) <= 1000: - image_data = None - except Exception: - image_data = None - - if not image_data: - art_url = metadata.get("album_art_url") - if not art_url: - logger_.warning("No album art URL available for embedding.") - return - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - mime_type = response.info().get_content_type() or "image/jpeg" - - if not image_data: - logger_.error("Failed to download album art data.") - return - - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.APIC(encoding=3, mime=mime_type, type=3, desc="Cover", data=image_data)) - elif isinstance(audio_file, symbols.FLAC): - picture = symbols.Picture() - picture.data = image_data - picture.type = 3 - picture.mime = mime_type - width, height = _get_image_dimensions(image_data) - picture.width = width or 640 - picture.height = height or 640 - picture.depth = 24 - audio_file.add_picture(picture) - elif isinstance(audio_file, symbols.MP4): - fmt = symbols.MP4Cover.FORMAT_JPEG if "jpeg" in mime_type else symbols.MP4Cover.FORMAT_PNG - audio_file["covr"] = [symbols.MP4Cover(image_data, imageformat=fmt)] - - logger_.info("Album art successfully embedded.") - except Exception as exc: - logger_.error("Error embedding album art: %s", exc) - - -def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=None): - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) - symbols = _get_mutagen_symbols(runtime) - if not symbols: - return - - try: - tag_config = { - "SPOTIFY_TRACK_ID": "spotify.tags.track_id", - "SPOTIFY_ARTIST_ID": "spotify.tags.artist_id", - "SPOTIFY_ALBUM_ID": "spotify.tags.album_id", - "ITUNES_TRACK_ID": "itunes.tags.track_id", - "ITUNES_ARTIST_ID": "itunes.tags.artist_id", - "ITUNES_ALBUM_ID": "itunes.tags.album_id", - "MUSICBRAINZ_RECORDING_ID": "musicbrainz.tags.recording_id", - "MUSICBRAINZ_ARTIST_ID": "musicbrainz.tags.artist_id", - "MUSICBRAINZ_RELEASE_ID": "musicbrainz.tags.release_id", - "MUSICBRAINZ_RELEASEGROUPID": "musicbrainz.tags.release_group_id", - "MUSICBRAINZ_ALBUMARTISTID": "musicbrainz.tags.album_artist_id", - "MUSICBRAINZ_RELEASETRACKID": "musicbrainz.tags.release_track_id", - "RELEASETYPE": "musicbrainz.tags.release_type", - "ORIGINALDATE": "musicbrainz.tags.original_date", - "RELEASESTATUS": "musicbrainz.tags.release_status", - "RELEASECOUNTRY": "musicbrainz.tags.release_country", - "BARCODE": "musicbrainz.tags.barcode", - "MEDIA": "musicbrainz.tags.media", - "TOTALDISCS": "musicbrainz.tags.total_discs", - "CATALOGNUMBER": "musicbrainz.tags.catalog_number", - "SCRIPT": "musicbrainz.tags.script", - "ASIN": "musicbrainz.tags.asin", - "DEEZER_TRACK_ID": "deezer.tags.track_id", - "DEEZER_ARTIST_ID": "deezer.tags.artist_id", - "AUDIODB_TRACK_ID": "audiodb.tags.track_id", - "TIDAL_TRACK_ID": "tidal.tags.track_id", - "TIDAL_ARTIST_ID": "tidal.tags.artist_id", - "QOBUZ_TRACK_ID": "qobuz.tags.track_id", - "QOBUZ_ARTIST_ID": "qobuz.tags.artist_id", - "GENIUS_TRACK_ID": "genius.tags.track_id", - } - - def _tag_enabled(path: str) -> bool: - return cfg.get(path, True) is not False - - def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: - if not a or not b: - return False - from difflib import SequenceMatcher - - norm = lambda s: re.sub(r"[^a-z0-9 ]", "", re.sub(r"\(.*?\)", "", s).lower()).strip() - return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold - - context = normalize_import_context(context) - source = (metadata.get("source") or "").strip().lower() - source_ids = {} - if source: - source_tag_names = get_source_tag_names(source) - source_track_id = metadata.get("source_track_id") - source_artist_id = metadata.get("source_artist_id") - source_album_id = metadata.get("source_album_id") - if cfg.get(f"{source}.embed_tags", True) is not False: - if source_tag_names.get("track") and source_track_id: - source_ids[source_tag_names["track"]] = source_track_id - if source_tag_names.get("artist") and source_artist_id: - source_ids[source_tag_names["artist"]] = source_artist_id - if source_tag_names.get("album") and source_album_id: - source_ids[source_tag_names["album"]] = source_album_id - - if not source_ids: - if cfg.get("spotify.embed_tags", True) is not False: - if metadata.get("spotify_track_id"): - source_ids["SPOTIFY_TRACK_ID"] = metadata["spotify_track_id"] - if metadata.get("spotify_artist_id"): - source_ids["SPOTIFY_ARTIST_ID"] = metadata["spotify_artist_id"] - if metadata.get("spotify_album_id"): - source_ids["SPOTIFY_ALBUM_ID"] = metadata["spotify_album_id"] - if cfg.get("itunes.embed_tags", True) is not False: - if metadata.get("itunes_track_id"): - source_ids["ITUNES_TRACK_ID"] = metadata["itunes_track_id"] - if metadata.get("itunes_artist_id"): - source_ids["ITUNES_ARTIST_ID"] = metadata["itunes_artist_id"] - if metadata.get("itunes_album_id"): - source_ids["ITUNES_ALBUM_ID"] = metadata["itunes_album_id"] - - track_title = metadata.get("title", "") - artist_name = metadata.get("album_artist", "") or metadata.get("artist", "") - track_info = get_import_track_info(context) - explicit_artist = (track_info or {}).get("_explicit_artist_context") if isinstance(track_info, dict) else None - batch_artist_name = None - if isinstance(explicit_artist, dict) and explicit_artist.get("name"): - batch_artist_name = explicit_artist["name"] - elif isinstance(explicit_artist, str) and explicit_artist: - batch_artist_name = explicit_artist - - pp = { - "id_tags": source_ids, - "track_title": track_title, - "artist_name": artist_name, - "batch_artist_name": batch_artist_name, - "metadata": metadata, - "recording_mbid": None, - "artist_mbid": None, - "release_mbid": "", - "mb_genres": [], - "isrc": None, - "deezer_bpm": None, - "deezer_isrc": None, - "audiodb_mood": None, - "audiodb_style": None, - "audiodb_genre": None, - "tidal_isrc": None, - "tidal_copyright": None, - "qobuz_isrc": None, - "qobuz_copyright": None, - "qobuz_label": None, - "lastfm_tags": [], - "lastfm_url": None, - "genius_url": None, - "release_year": None, - } - - source_order = cfg.get("metadata_enhancement.post_process_order", None) - if not isinstance(source_order, list) or not source_order: - source_order = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"] - - db = _get_database(runtime) - - for source_name in source_order: - if source_name == "musicbrainz": - if cfg.get("musicbrainz.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - mb_worker = getattr(runtime, "mb_worker", None) - mb_service = mb_worker.mb_service if mb_worker else None - if not mb_service: - continue - try: - result = mb_service.match_recording(track_title, artist_name) - if result and result.get("mbid"): - pp["recording_mbid"] = result["mbid"] - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"] - details = mb_service.mb_client.get_recording(pp["recording_mbid"], includes=["isrcs", "genres"]) - if details: - isrcs = details.get("isrcs", []) - if isrcs: - pp["isrc"] = isrcs[0] - pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)] - - track_artist_name = metadata.get("artist", "") or artist_name - if ", " in track_artist_name: - track_artist_name = track_artist_name.split(", ")[0] - artist_result = mb_service.match_artist(track_artist_name) - if artist_result and artist_result.get("mbid"): - pp["artist_mbid"] = artist_result["mbid"] - pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"] - - album_name_for_mb = metadata.get("album", "") - if album_name_for_mb: - artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() - rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), artist_key) - rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) - with _MB_RELEASE_CACHE_LOCK: - cached = _MB_RELEASE_CACHE.get(rc_key_norm) - if cached is None: - cached = _MB_RELEASE_CACHE.get(rc_key_exact) - if cached is not None: - pp["release_mbid"] = cached - else: - try: - rc_result = mb_service.match_release(album_name_for_mb, artist_name) - pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" - except Exception: - pp["release_mbid"] = "" - _MB_RELEASE_CACHE[rc_key_norm] = pp["release_mbid"] - _MB_RELEASE_CACHE[rc_key_exact] = pp["release_mbid"] - if pp["release_mbid"]: - pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] - - if pp["release_mbid"]: - with _MB_RELEASE_DETAIL_CACHE_LOCK: - release_detail = _MB_RELEASE_DETAIL_CACHE.get(pp["release_mbid"]) - if release_detail is None: - release_detail = mb_service.mb_client.get_release( - pp["release_mbid"], - includes=["release-groups", "labels", "media", "artist-credits", "recordings"], - ) or {} - with _MB_RELEASE_DETAIL_CACHE_LOCK: - _MB_RELEASE_DETAIL_CACHE[pp["release_mbid"]] = release_detail - if release_detail: - rg = release_detail.get("release-group", {}) - if rg.get("id"): - pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"] - ac = release_detail.get("artist-credit", []) - if ac and isinstance(ac[0], dict): - aa = ac[0].get("artist", {}) - if aa.get("id"): - pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"] - if rg.get("primary-type"): - pp["id_tags"]["RELEASETYPE"] = rg["primary-type"] - if rg.get("first-release-date"): - pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"] - if not pp["release_year"] and len(rg["first-release-date"]) >= 4: - year = rg["first-release-date"][:4] - if year.isdigit(): - pp["release_year"] = year - if release_detail.get("status"): - pp["id_tags"]["RELEASESTATUS"] = release_detail["status"] - if release_detail.get("country"): - pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"] - if release_detail.get("barcode"): - pp["id_tags"]["BARCODE"] = release_detail["barcode"] - media_list = release_detail.get("media", []) - if media_list: - fmt = media_list[0].get("format", "") - if fmt: - pp["id_tags"]["MEDIA"] = fmt - pp["id_tags"]["TOTALDISCS"] = str(len(media_list)) - label_info = release_detail.get("label-info", []) - if label_info and isinstance(label_info[0], dict): - cat = label_info[0].get("catalog-number", "") - if cat: - pp["id_tags"]["CATALOGNUMBER"] = cat - text_rep = release_detail.get("text-representation", {}) - if isinstance(text_rep, dict) and text_rep.get("script"): - pp["id_tags"]["SCRIPT"] = text_rep["script"] - if release_detail.get("asin"): - pp["id_tags"]["ASIN"] = release_detail["asin"] - track_num = metadata.get("track_number") - disc_num = metadata.get("disc_number") or 1 - if track_num and media_list: - try: - track_num_int = int(track_num) - disc_num_int = int(disc_num) - for medium in media_list: - if medium.get("position", 1) == disc_num_int: - for mtrack in (medium.get("tracks") or medium.get("track-list", [])): - if mtrack.get("position") == track_num_int: - if mtrack.get("id"): - pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"] - release_recording = mtrack.get("recording", {}) - if release_recording.get("id"): - pp["recording_mbid"] = release_recording["id"] - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"] - break - break - except (ValueError, TypeError): - pass - except Exception as exc: - logger_.error("MusicBrainz lookup failed (non-fatal): %s", exc) - continue - - if source_name == "deezer": - if cfg.get("deezer.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - deezer_worker = getattr(runtime, "deezer_worker", None) - dz_client = deezer_worker.client if deezer_worker else None - if not dz_client: - continue - dz_result = dz_client.search_track(artist_name, track_title) - if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name): - dz_track_id = dz_result["id"] - pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id) - dz_artist_id = dz_result.get("artist", {}).get("id") - if dz_artist_id: - pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id) - dz_details = dz_client.get_track_details(dz_track_id) - if dz_details: - bpm_val = dz_details.get("bpm") - if bpm_val and bpm_val > 0: - pp["deezer_bpm"] = bpm_val - dz_isrc = dz_details.get("isrc") - if dz_isrc: - pp["deezer_isrc"] = dz_isrc - if not pp["release_year"]: - dz_album = dz_result.get("album", {}) - dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or "" - if len(dz_release) >= 4 and dz_release[:4].isdigit(): - pp["release_year"] = dz_release[:4] - except Exception as exc: - logger_.error("Deezer lookup failed (non-fatal): %s", exc) - continue - - if source_name == "audiodb": - if cfg.get("audiodb.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - audiodb_worker = getattr(runtime, "audiodb_worker", None) - adb_client = audiodb_worker.client if audiodb_worker else None - if not adb_client: - continue - adb_result = adb_client.search_track(artist_name, track_title) - if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name): - adb_track_id = adb_result.get("idTrack") - if adb_track_id: - pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id) - adb_mb_track = adb_result.get("strMusicBrainzID") - if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]: - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track - pp["recording_mbid"] = adb_mb_track - adb_mb_artist = adb_result.get("strMusicBrainzArtistID") - if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]: - pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist - pp["artist_mbid"] = adb_mb_artist - pp["audiodb_mood"] = adb_result.get("strMood") or None - pp["audiodb_style"] = adb_result.get("strStyle") or None - pp["audiodb_genre"] = adb_result.get("strGenre") or None - except Exception as exc: - logger_.error("AudioDB lookup failed (non-fatal): %s", exc) - continue - - if source_name == "tidal": - if cfg.get("tidal.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - tidal_client = getattr(runtime, "tidal_client", None) - if not (tidal_client and tidal_client.is_authenticated()): - continue - td_result = tidal_client.search_track(artist_name, track_title) - if td_result and _names_match(td_result.get("title", ""), track_title): - td_track_id = td_result.get("id") - if td_track_id: - pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id) - td_artist = td_result.get("artist", {}) - if isinstance(td_artist, dict) and td_artist.get("id"): - pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"]) - if td_track_id: - td_details = tidal_client.get_track(str(td_track_id)) - if td_details: - pp["tidal_isrc"] = td_details.get("isrc") - td_copyright = td_details.get("copyright") - if isinstance(td_copyright, dict): - td_copyright = td_copyright.get("text", td_copyright.get("name", "")) - pp["tidal_copyright"] = td_copyright or None - if not pp["release_year"]: - td_album = td_result.get("album", {}) - td_release = "" - if isinstance(td_album, dict): - td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "") - if len(td_release) >= 4 and td_release[:4].isdigit(): - pp["release_year"] = td_release[:4] - except Exception as exc: - logger_.error("Tidal lookup failed (non-fatal): %s", exc) - continue - - if source_name == "qobuz": - if cfg.get("qobuz.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None) - qz_client = qobuz_worker.client if qobuz_worker else None - if not (qz_client and qz_client.is_authenticated()): - continue - qz_result = qz_client.search_track(artist_name, track_title) - if qz_result: - qz_performer = qz_result.get("performer") or {} - if not isinstance(qz_performer, dict): - qz_performer = {} - qz_artist_name = qz_performer.get("name", "") - if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name): - qz_track_id = qz_result.get("id") - if qz_track_id: - pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id) - if qz_performer.get("id"): - pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"]) - qz_isrc = qz_result.get("isrc") - if isinstance(qz_isrc, dict): - qz_isrc = qz_isrc.get("value", qz_isrc.get("id", "")) - if qz_isrc: - pp["qobuz_isrc"] = qz_isrc - qz_copyright = qz_result.get("copyright") - if isinstance(qz_copyright, dict): - qz_copyright = qz_copyright.get("text", qz_copyright.get("name", "")) - if isinstance(qz_copyright, str): - pp["qobuz_copyright"] = qz_copyright - qz_album = qz_result.get("album", {}) - if isinstance(qz_album, dict): - qz_label_info = qz_album.get("label", {}) - if isinstance(qz_label_info, dict) and qz_label_info.get("name"): - pp["qobuz_label"] = qz_label_info["name"] - if not pp["release_year"]: - qz_release = str(qz_album.get("release_date_original", "") or "") - if not qz_release: - qz_ts = qz_album.get("released_at") - if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: - import datetime as _dt - qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) - if len(qz_release) >= 4 and qz_release[:4].isdigit(): - pp["release_year"] = qz_release[:4] - except Exception as exc: - logger_.error("Qobuz lookup failed (non-fatal): %s", exc) - continue - - if source_name == "lastfm": - if cfg.get("lastfm.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - lastfm_worker = getattr(runtime, "lastfm_worker", None) - lf_client = lastfm_worker.client if lastfm_worker else None - if not lf_client: - continue - lf_result = lf_client.get_track_info(artist_name, track_title) - if lf_result: - lf_url = lf_result.get("url") - if lf_url: - pp["lastfm_url"] = lf_url - lf_toptags = lf_result.get("toptags", {}) - if isinstance(lf_toptags, dict): - tag_list = lf_toptags.get("tag", []) - if isinstance(tag_list, list): - pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")] - elif isinstance(tag_list, dict) and tag_list.get("name"): - pp["lastfm_tags"] = [tag_list["name"]] - except Exception as exc: - logger_.error("Last.fm lookup failed (non-fatal): %s", exc) - continue - - if source_name == "genius": - if cfg.get("genius.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - import core.genius_client as _genius_module - - if time.time() < _genius_module._rate_limit_until: - logger_.info("Genius rate-limited, skipping (non-blocking)") - continue - genius_worker = getattr(runtime, "genius_worker", None) - g_client = genius_worker.client if genius_worker else None - if not g_client: - continue - g_result = g_client.search_song(artist_name, track_title) - if g_result: - g_id = g_result.get("id") - if g_id: - pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id) - g_url = g_result.get("url") - if g_url: - pp["genius_url"] = g_url - except Exception as exc: - logger_.error("Genius lookup failed (non-fatal): %s", exc) - continue - - if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["audiodb_mood"] and not pp["audiodb_style"]: - return - - filtered_tags: Dict[str, str] = {} - for tag_name, value in pp["id_tags"].items(): - config_path = tag_config.get(tag_name) - if config_path and not _tag_enabled(config_path): - continue - filtered_tags[tag_name] = value - - written = [] - id3_tag_map = { - "MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"), - "MUSICBRAINZ_ARTIST_ID": ("TXXX", "MusicBrainz Artist Id"), - "MUSICBRAINZ_RELEASE_ID": ("TXXX", "MusicBrainz Album Id"), - "MUSICBRAINZ_RELEASEGROUPID": ("TXXX", "MusicBrainz Release Group Id"), - "MUSICBRAINZ_ALBUMARTISTID": ("TXXX", "MusicBrainz Album Artist Id"), - "MUSICBRAINZ_RELEASETRACKID": ("TXXX", "MusicBrainz Release Track Id"), - "RELEASETYPE": ("TXXX", "MusicBrainz Album Type"), - "RELEASESTATUS": ("TXXX", "MusicBrainz Album Status"), - "RELEASECOUNTRY": ("TXXX", "MusicBrainz Album Release Country"), - "ORIGINALDATE": ("TDOR", None), - "MEDIA": ("TMED", None), - } - vorbis_tag_map = { - "MUSICBRAINZ_RECORDING_ID": "MUSICBRAINZ_TRACKID", - "MUSICBRAINZ_ARTIST_ID": "MUSICBRAINZ_ARTISTID", - "MUSICBRAINZ_RELEASE_ID": "MUSICBRAINZ_ALBUMID", - "MUSICBRAINZ_RELEASEGROUPID": "MUSICBRAINZ_RELEASEGROUPID", - "MUSICBRAINZ_ALBUMARTISTID": "MUSICBRAINZ_ALBUMARTISTID", - "MUSICBRAINZ_RELEASETRACKID": "MUSICBRAINZ_RELEASETRACKID", - } - mp4_tag_map = { - "MUSICBRAINZ_RECORDING_ID": "MusicBrainz Track Id", - "MUSICBRAINZ_ARTIST_ID": "MusicBrainz Artist Id", - "MUSICBRAINZ_RELEASE_ID": "MusicBrainz Album Id", - "MUSICBRAINZ_RELEASEGROUPID": "MusicBrainz Release Group Id", - "MUSICBRAINZ_ALBUMARTISTID": "MusicBrainz Album Artist Id", - "MUSICBRAINZ_RELEASETRACKID": "MusicBrainz Release Track Id", - "RELEASETYPE": "MusicBrainz Album Type", - "RELEASESTATUS": "MusicBrainz Album Status", - "RELEASECOUNTRY": "MusicBrainz Album Release Country", - } - - if isinstance(audio_file.tags, symbols.ID3): - for tag_name, value in filtered_tags.items(): - spec = id3_tag_map.get(tag_name) - if spec: - frame_type, desc = spec - if frame_type == "UFID": - audio_file.tags.add(symbols.UFID(owner=desc, data=str(value).encode("ascii"))) - written.append(f"UFID:{desc}") - elif frame_type == "TDOR": - audio_file.tags.add(symbols.TDOR(encoding=3, text=[value])) - written.append("TDOR") - elif frame_type == "TMED": - audio_file.tags.add(symbols.TMED(encoding=3, text=[value])) - written.append("TMED") - else: - audio_file.tags.add(symbols.TXXX(encoding=3, desc=desc, text=[value])) - written.append(f"TXXX:{desc}") - else: - audio_file.tags.add(symbols.TXXX(encoding=3, desc=tag_name, text=[str(value)])) - written.append(f"TXXX:{tag_name}") - elif _is_vorbis_like(audio_file, symbols): - for tag_name, value in filtered_tags.items(): - audio_file[vorbis_tag_map.get(tag_name, tag_name)] = [str(value)] - written.append(vorbis_tag_map.get(tag_name, tag_name)) - elif isinstance(audio_file, symbols.MP4): - for tag_name, value in filtered_tags.items(): - key = f"----:com.apple.iTunes:{mp4_tag_map.get(tag_name, tag_name)}" - audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))] - written.append(key) - - if written: - logger_.info("Embedded IDs: %s", ", ".join(written)) - - release_year = pp["release_year"] - needs_date_tag = bool(release_year and not metadata.get("date")) - if needs_date_tag: - metadata["date"] = release_year - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["date"] = [release_year] - elif isinstance(audio_file, symbols.MP4): - audio_file["\xa9day"] = [release_year] - logger_.info("Date tag: %s", release_year) - - if _tag_enabled("deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0: - bpm_int = int(pp["deezer_bpm"]) - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["BPM"] = [str(bpm_int)] - elif isinstance(audio_file, symbols.MP4): - audio_file["tmpo"] = [bpm_int] - logger_.info("BPM: %s", bpm_int) - - if _tag_enabled("audiodb.tags.mood") and pp["audiodb_mood"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["MOOD"] = [pp["audiodb_mood"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))] - - if _tag_enabled("audiodb.tags.style") and pp["audiodb_style"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["STYLE"] = [pp["audiodb_style"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))] - - if _tag_enabled("metadata_enhancement.tags.genre_merge"): - enrichment_genres = [] - if _tag_enabled("musicbrainz.tags.genres"): - enrichment_genres += pp["mb_genres"] - if pp["audiodb_genre"] and _tag_enabled("audiodb.tags.genre"): - enrichment_genres.append(pp["audiodb_genre"]) - if _tag_enabled("lastfm.tags.genres"): - enrichment_genres += pp["lastfm_tags"] - if enrichment_genres: - from core.genre_filter import filter_genres as _filter_genres - - enrichment_genres = _filter_genres(enrichment_genres, cfg) - source_genres = [g.strip() for g in str(metadata.get("genre", "")).split(",") if g.strip()] - seen = set() - merged = [] - for genre in source_genres + enrichment_genres: - key = genre.strip().lower() - if key and key not in seen: - seen.add(key) - merged.append(genre.strip().title()) - if len(merged) >= 5: - break - if merged: - genre_string = ", ".join(merged) - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["GENRE"] = [genre_string] - elif isinstance(audio_file, symbols.MP4): - audio_file["\xa9gen"] = [genre_string] - logger_.info("Genres merged: %s", genre_string) - - isrc_candidates = [] - if pp["isrc"] and _tag_enabled("musicbrainz.tags.isrc"): - isrc_candidates.append(("MusicBrainz", pp["isrc"])) - if pp["deezer_isrc"] and _tag_enabled("deezer.tags.isrc"): - isrc_candidates.append(("Deezer", pp["deezer_isrc"])) - if pp["tidal_isrc"] and _tag_enabled("tidal.tags.isrc"): - isrc_candidates.append(("Tidal", pp["tidal_isrc"])) - if pp["qobuz_isrc"] and _tag_enabled("qobuz.tags.isrc"): - isrc_candidates.append(("Qobuz", pp["qobuz_isrc"])) - if isrc_candidates: - isrc_source, final_isrc = isrc_candidates[0] - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["ISRC"] = [final_isrc] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))] - logger_.info("ISRC (%s): %s", isrc_source, final_isrc) - - copyright_candidates = [] - if pp["tidal_copyright"] and _tag_enabled("tidal.tags.copyright"): - copyright_candidates.append(("Tidal", pp["tidal_copyright"])) - if pp["qobuz_copyright"] and _tag_enabled("qobuz.tags.copyright"): - copyright_candidates.append(("Qobuz", pp["qobuz_copyright"])) - if copyright_candidates: - copyright_source, final_copyright = copyright_candidates[0] - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["COPYRIGHT"] = [final_copyright] - elif isinstance(audio_file, symbols.MP4): - audio_file["cprt"] = [final_copyright] - logger_.info("Copyright (%s): %s", copyright_source, final_copyright[:60]) - - if _tag_enabled("qobuz.tags.label") and pp["qobuz_label"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["LABEL"] = [pp["qobuz_label"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))] - - if _tag_enabled("lastfm.tags.url") and pp["lastfm_url"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["LASTFM_URL"] = [pp["lastfm_url"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))] - - if _tag_enabled("genius.tags.url") and pp["genius_url"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]])) - elif _is_vorbis_like(audio_file, symbols): - audio_file["GENIUS_URL"] = [pp["genius_url"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))] - - release_id = pp["release_mbid"] - if release_id: - metadata["musicbrainz_release_id"] = release_id - if db is not None: - try: - album_name_for_db = metadata.get("album", "") - album_artist_for_db = metadata.get("album_artist", "") or metadata.get("artist", "") - if album_name_for_db and album_artist_for_db: - conn = db._get_connection() - try: - cursor = conn.cursor() - cursor.execute( - """ - UPDATE albums SET year = ? - WHERE (year IS NULL OR year = 0) - AND id IN ( - SELECT al.id FROM albums al - JOIN artists ar ON ar.id = al.artist_id - WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) - ) - """, - (int(release_year), album_name_for_db, album_artist_for_db), - ) - if cursor.rowcount > 0: - conn.commit() - logger_.info("Updated album year to %s in database", release_year) - else: - conn.rollback() - finally: - conn.close() - except Exception as exc: - logger_.error("Could not update album year in DB: %s", exc) - - except Exception as exc: - logger_.error("Error embedding source IDs (non-fatal): %s", exc) - - -def download_cover_art(album_info: dict, target_dir: str, context: dict = None, runtime=None): - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) - if cfg.get("metadata_enhancement.cover_art_download", True) is False: - return - - try: - cover_path = os.path.join(target_dir, "cover.jpg") - album_info = album_info or {} - release_mbid = album_info.get("musicbrainz_release_id") - prefer_caa = cfg.get("metadata_enhancement.prefer_caa_art", False) - - if os.path.exists(cover_path): - if release_mbid and prefer_caa: - try: - existing_size = os.path.getsize(cover_path) - if existing_size > 200_000: - return - is_upgrade = True - except Exception: - return - else: - return - else: - is_upgrade = False - - image_data = None - if release_mbid and prefer_caa: - try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" - req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) - with urllib.request.urlopen(req, timeout=10) as response: - image_data = response.read() - if not image_data or len(image_data) <= 1000: - image_data = None - except Exception: - image_data = None - - if is_upgrade and not image_data: - logger_.error("CAA upgrade failed - keeping existing cover.jpg") - return - - if not image_data: - art_url = album_info.get("album_image_url") - if not art_url and context: - album_ctx = get_import_context_album(context) - art_url = album_ctx.get("image_url") - if not art_url and album_ctx.get("images"): - images = album_ctx.get("images", []) - if images and isinstance(images[0], dict): - art_url = images[0].get("url", "") - if art_url: - logger_.info("Using cover art URL from album context") - if art_url and "i.scdn.co" in art_url: - try: - from core.spotify_client import _upgrade_spotify_image_url - - art_url = _upgrade_spotify_image_url(art_url) - except Exception: - pass - elif art_url and "mzstatic.com" in art_url: - import re as _re - - art_url = _re.sub(r"\d+x\d+bb", "3000x3000bb", art_url) - if not art_url: - logger_.warning("No cover art URL available for download.") - return - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - - if not image_data: - return - - with open(cover_path, "wb") as handle: - handle.write(image_data) - logger_.info("Cover art downloaded to: %s", cover_path) - except Exception as exc: - logger_.error("Error downloading cover.jpg: %s", exc) - - -def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) - if cfg.get("metadata_enhancement.lrclib_enabled", True) is False: - return False - - try: - from core.lyrics_client import lyrics_client - - context = normalize_import_context(context) - original_search = get_import_original_search(context) - album_context = get_import_context_album(context) - track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track")) - - if isinstance(artist, dict): - artist_name = artist.get("name", "Unknown Artist") - elif hasattr(artist, "name"): - artist_name = artist.name - else: - artist_name = str(artist) if artist else "Unknown Artist" - - album_name = None - duration_seconds = None - if album_info and album_info.get("is_album"): - album_name = ( - get_import_clean_album(context, album_info=album_info, default="") - or album_info.get("album_name") - or album_context.get("name") - ) - - if original_search.get("duration_ms"): - duration_seconds = int(original_search["duration_ms"] / 1000) - - success = lyrics_client.create_lrc_file( - audio_file_path=file_path, - track_name=track_name, - artist_name=artist_name, - album_name=album_name, - duration_seconds=duration_seconds, - ) - - if success: - logger_.info("LRC file generated for: %s", track_name) - else: - logger_.warning("No lyrics found for: %s", track_name) - return success - except Exception as exc: - logger_.error("Error generating LRC file for %s: %s", file_path, exc) - return False - - -def wipe_source_tags(file_path: str, runtime=None) -> bool: - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) - _ = cfg # keep signature parallel with other helpers - - try: - _strip_all_non_audio_tags(file_path, runtime=runtime) - symbols = _get_mutagen_symbols(runtime) - if not symbols: - return False - - audio = symbols.File(file_path) - if audio is None: - return False - if hasattr(audio, "clear_pictures"): - audio.clear_pictures() - if audio.tags is not None: - tag_count = len(audio.tags) - audio.tags.clear() - else: - audio.add_tags() - tag_count = 0 - _save_audio_file(audio, symbols) - if tag_count > 0: - logger_.info("[Tag Wipe] Stripped %s source tags from: %s", tag_count, os.path.basename(file_path)) - return True - except Exception as exc: - logger_.error("[Tag Wipe] Failed (non-fatal): %s", exc) - return False - - -def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: - cfg = _get_config_manager(runtime) - logger_ = _get_logger(runtime) +def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: + cfg = _get_config_manager() + logger_ = _get_logger() if cfg.get("metadata_enhancement.enabled", True) is False: logger_.warning("Metadata enhancement disabled in config.") return True @@ -1352,7 +58,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf if album_info is None: album_info = {} - symbols = _get_mutagen_symbols(runtime) + symbols = _get_mutagen_symbols() if not symbols: logger_.error("Mutagen is unavailable, cannot enhance metadata.") return False @@ -1361,7 +67,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf with file_lock: logger_.info("Enhancing metadata for: %s", os.path.basename(file_path)) try: - _strip_all_non_audio_tags(file_path, runtime=runtime) + _strip_all_non_audio_tags(file_path) audio_file = symbols.File(file_path) if audio_file is None: logger_.error("Could not load audio file with Mutagen: %s", file_path) @@ -1380,7 +86,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf _save_audio_file(audio_file, symbols) - metadata = extract_source_metadata(context, artist, album_info, runtime=runtime) + metadata = extract_source_metadata(context, artist, album_info) if not metadata: logger_.error("Could not extract source metadata, saving with cleared tags.") _save_audio_file(audio_file, symbols) @@ -1443,13 +149,13 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf if metadata.get("disc_number"): audio_file["disk"] = [(metadata["disc_number"], 0)] - embed_source_ids(audio_file, metadata, context, runtime=runtime) + embed_source_ids(audio_file, metadata, context) if album_info is not None and metadata.get("musicbrainz_release_id"): album_info["musicbrainz_release_id"] = metadata["musicbrainz_release_id"] if cfg.get("metadata_enhancement.embed_album_art", True): - embed_album_art_metadata(audio_file, metadata, runtime=runtime) + embed_album_art_metadata(audio_file, metadata) quality = context.get("_audio_quality", "") if quality and cfg.get("metadata_enhancement.tags.quality_tag", True) is not False: @@ -1462,7 +168,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf _save_audio_file(audio_file, symbols) - verified = _verify_metadata_written(file_path, runtime=runtime) + verified = _verify_metadata_written(file_path) if verified: logger_.info("Metadata enhanced successfully.") else: @@ -1478,9 +184,3 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf logger_.warning("[Metadata Debug] Album info: %s", album_info.get("album_name", "MISSING") if album_info else "None") logger_.error("[Metadata Debug] Traceback:\n%s", traceback.format_exc()) return False - - -def _normalize_album_cache_key(album_name: str) -> str: - result = _EDITION_PAREN_RE.sub("", album_name or "") - result = _EDITION_BARE_RE.sub("", result) - return result.lower().strip() diff --git a/core/metadata_lyrics.py b/core/metadata_lyrics.py new file mode 100644 index 00000000..99a7612b --- /dev/null +++ b/core/metadata_lyrics.py @@ -0,0 +1,63 @@ +"""Lyrics export helpers for metadata enrichment.""" + +from __future__ import annotations + +from core.import_context import ( + get_import_clean_album, + get_import_clean_title, + get_import_context_album, + get_import_original_search, + normalize_import_context, +) +from core.metadata_common import _get_config_manager, _get_logger + + +def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: + cfg = _get_config_manager() + logger_ = _get_logger() + if cfg.get("metadata_enhancement.lrclib_enabled", True) is False: + return False + + try: + from core.lyrics_client import lyrics_client + + context = normalize_import_context(context) + original_search = get_import_original_search(context) + album_context = get_import_context_album(context) + track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track")) + + if isinstance(artist, dict): + artist_name = artist.get("name", "Unknown Artist") + elif hasattr(artist, "name"): + artist_name = artist.name + else: + artist_name = str(artist) if artist else "Unknown Artist" + + album_name = None + duration_seconds = None + if album_info and album_info.get("is_album"): + album_name = ( + get_import_clean_album(context, album_info=album_info, default="") + or album_info.get("album_name") + or album_context.get("name") + ) + + if original_search.get("duration_ms"): + duration_seconds = int(original_search["duration_ms"] / 1000) + + success = lyrics_client.create_lrc_file( + audio_file_path=file_path, + track_name=track_name, + artist_name=artist_name, + album_name=album_name, + duration_seconds=duration_seconds, + ) + + if success: + logger_.info("LRC file generated for: %s", track_name) + else: + logger_.warning("No lyrics found for: %s", track_name) + return success + except Exception as exc: + logger_.error("Error generating LRC file for %s: %s", file_path, exc) + return False diff --git a/core/metadata_service.py b/core/metadata_service.py index 861f0b08..68cf0b92 100644 --- a/core/metadata_service.py +++ b/core/metadata_service.py @@ -1,5 +1,5 @@ """ -Metadata Service - Centralized metadata source selection +Metadata Service - Centralized metadata source selection and provider access. ALL metadata source decisions flow through this module. Other files import get_primary_source() and get_primary_client() instead of reimplementing @@ -100,6 +100,35 @@ def get_source_priority(preferred_source: str): return ordered +def _get_source_chain_for_lookup(options: MetadataLookupOptions) -> List[str]: + primary_source = get_primary_source() + source_chain = list(get_source_priority(primary_source)) + override = (options.source_override or '').strip().lower() + + if override: + source_chain = [override] + [source for source in source_chain if source != override] + + if not options.allow_fallback: + source_chain = source_chain[:1] + + return source_chain + + +def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any: + if value is None: + return default + + for name in names: + if isinstance(value, dict): + if name in value and value[name] is not None: + return value[name] + else: + candidate = getattr(value, name, None) + if candidate is not None: + return candidate + return default + + def get_client_for_source(source: str): """Get the client object for an exact metadata source. @@ -235,35 +264,6 @@ def get_artist_albums_for_source( return None -def _get_source_chain_for_lookup(options: MetadataLookupOptions) -> List[str]: - primary_source = get_primary_source() - source_chain = list(get_source_priority(primary_source)) - override = (options.source_override or '').strip().lower() - - if override: - source_chain = [override] + [source for source in source_chain if source != override] - - if not options.allow_fallback: - source_chain = source_chain[:1] - - return source_chain - - -def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any: - if value is None: - return default - - for name in names: - if isinstance(value, dict): - if name in value and value[name] is not None: - return value[name] - else: - candidate = getattr(value, name, None) - if candidate is not None: - return candidate - return default - - def _normalize_artist_name(value: Any) -> str: return (value or '').strip().casefold() @@ -841,234 +841,6 @@ def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]: return normalized -def _build_single_import_context_payload( - track_data: Any, - source: Optional[str], - source_priority: List[str], - requested_title: str = '', - requested_artist: str = '', -) -> Dict[str, Any]: - album_data = _extract_lookup_value(track_data, 'album', default=None) - - track_id = str(_extract_lookup_value(track_data, 'id', 'track_id', 'trackId', default='') or '') - track_name = _extract_lookup_value(track_data, 'name', 'title', 'trackName', default='') or requested_title or 'Unknown Track' - track_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'artists', default=[])) - if not track_artists and requested_artist: - track_artists = [{'name': requested_artist}] - - primary_track_artist = track_artists[0] if track_artists else {} - primary_artist_name = primary_track_artist.get('name') or requested_artist or 'Unknown Artist' - primary_artist_id = str(primary_track_artist.get('id', '') or _extract_lookup_value(track_data, 'artist_id', 'artistId', default='') or '') - - album_name = _extract_lookup_value(track_data, 'album_name', 'collectionName', default='') or '' - album_id = str(_extract_lookup_value(track_data, 'album_id', 'collectionId', 'albumId', default='') or '') - release_date = str(_extract_lookup_value(track_data, 'release_date', default='') or '') - album_type = str(_extract_lookup_value(track_data, 'album_type', default='album') or 'album') - total_tracks = int(_extract_lookup_value(track_data, 'total_tracks', 'track_count', default=0) or 0) - album_images: List[Dict[str, Any]] = [] - album_image_url = str(_extract_lookup_value(track_data, 'image_url', 'thumb_url', default='') or '') - album_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'album_artists', 'artists', default=[])) - - if isinstance(album_data, dict): - album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name - album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id) - release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date) - album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type) - total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks) - album_images = _extract_lookup_value(album_data, 'images', default=[]) or [] - if not album_image_url: - album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '') - if not album_image_url and album_images: - album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') - album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) - elif album_data: - album_name = album_name or str(album_data) - - if not album_artists and primary_artist_name: - album_artists = [{'name': primary_artist_name}] - - if not album_image_url and album_images: - album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') - - track_info = { - 'id': track_id, - 'name': track_name, - 'track_number': int(_extract_lookup_value(track_data, 'track_number', 'trackNumber', default=1) or 1), - 'disc_number': int(_extract_lookup_value(track_data, 'disc_number', 'discNumber', default=1) or 1), - 'duration_ms': int(_extract_lookup_value(track_data, 'duration_ms', 'duration', 'trackTimeMillis', default=0) or 0), - 'artists': track_artists or [{'name': primary_artist_name}], - 'uri': str(_extract_lookup_value(track_data, 'uri', default='') or ''), - 'album': album_name, - 'album_id': album_id, - 'album_type': album_type, - 'release_date': release_date, - '_source': source or '', - } - - album_payload = { - 'id': album_id, - 'name': album_name, - 'release_date': release_date, - 'total_tracks': total_tracks or 1, - 'album_type': album_type, - 'image_url': album_image_url, - 'images': album_images, - 'artists': album_artists, - '_source': source or '', - } - - artist_payload = { - 'id': primary_artist_id, - 'name': primary_artist_name, - 'genres': [], - '_source': source or '', - } - - original_search = { - 'title': track_name, - 'artist': primary_artist_name, - 'album': album_name, - 'track_number': track_info['track_number'], - 'disc_number': track_info['disc_number'], - 'clean_title': track_name, - 'clean_album': album_name, - 'clean_artist': primary_artist_name, - 'artists': track_info['artists'], - 'duration_ms': track_info['duration_ms'], - 'id': track_id, - '_source': source or '', - } - - return { - 'success': bool(track_id or track_name != requested_title or album_name), - 'source': source, - 'source_priority': source_priority, - 'context': { - 'artist': artist_payload, - 'album': album_payload, - 'track_info': track_info, - 'original_search_result': original_search, - 'is_album_download': False, - 'has_clean_metadata': bool(track_id), - 'has_full_metadata': bool(track_id), - 'source': source, - 'source_priority': source_priority, - }, - } - - -def _build_single_import_fallback_context( - requested_title: str, - requested_artist: str, - source_priority: List[str], -) -> Dict[str, Any]: - artist_name = requested_artist or 'Unknown Artist' - title = requested_title or 'Unknown Track' - return { - 'success': False, - 'source': None, - 'source_priority': source_priority, - 'context': { - 'artist': { - 'id': '', - 'name': artist_name, - 'genres': [], - '_source': '', - }, - 'album': { - 'id': '', - 'name': '', - 'release_date': '', - 'total_tracks': 1, - 'album_type': 'album', - 'image_url': '', - 'images': [], - 'artists': [], - '_source': '', - }, - 'track_info': { - 'id': '', - 'name': title, - 'track_number': 1, - 'disc_number': 1, - 'duration_ms': 0, - 'artists': [{'name': artist_name}], - 'uri': '', - 'album': '', - 'album_id': '', - 'album_type': 'album', - 'release_date': '', - '_source': '', - }, - 'original_search_result': { - 'title': title, - 'artist': artist_name, - 'album': '', - 'track_number': 1, - 'disc_number': 1, - 'clean_title': title, - 'clean_album': '', - 'clean_artist': artist_name, - 'artists': [{'name': artist_name}], - 'duration_ms': 0, - 'id': '', - '_source': '', - }, - 'is_album_download': False, - 'has_clean_metadata': False, - 'has_full_metadata': False, - 'source': None, - 'source_priority': source_priority, - }, - } - - -def _build_track_search_query(source: str, title: str, artist: str) -> str: - base_query = " ".join(part for part in (title, artist) if part).strip() - if source == 'deezer' and title: - if artist: - return f'artist:"{artist}" track:"{title}"' - return f'track:"{title}"' - return base_query or title or artist - - -def _pick_best_track_match(search_results: List[Any], title: str, artist: str = '') -> Optional[Any]: - if not search_results: - return None - - target_title = str(title or '').strip().lower() - target_artist = str(artist or '').strip().lower() - - for candidate in search_results: - candidate_title = str(_extract_lookup_value(candidate, 'name', 'title', 'track_name', default='') or '').strip().lower() - if candidate_title != target_title: - continue - - if not target_artist: - return candidate - - candidate_artists = _normalize_context_artists(_extract_lookup_value(candidate, 'artists', default=[])) - candidate_artist_name = candidate_artists[0]['name'].strip().lower() if candidate_artists else '' - if candidate_artist_name == target_artist: - return candidate - - return search_results[0] - - -def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]: - if not client or not hasattr(client, 'search_tracks'): - return [] - - try: - kwargs = {'limit': limit} - if source == 'spotify': - kwargs['allow_fallback'] = False - return client.search_tracks(query, **kwargs) or [] - except Exception as exc: - logger.debug("Could not search %s for %s: %s", source, query, exc) - return [] - - def get_single_track_import_context( title: str, artist: str = '', @@ -1076,94 +848,16 @@ def get_single_track_import_context( override_source: str = 'spotify', source_override: Optional[str] = None, ) -> Dict[str, Any]: - """Build an import context for singles using source-priority metadata lookup.""" - options = MetadataLookupOptions(source_override=source_override, allow_fallback=True) - source_priority = _get_source_chain_for_lookup(options) - title = (title or '').strip() - artist = (artist or '').strip() + """Compatibility wrapper for the single-track import resolver.""" + from core.import_resolution import get_single_track_import_context as _get_single_track_import_context - if override_id: - chosen_source = (override_source or 'spotify').strip().lower() or 'spotify' - client = get_client_for_source(chosen_source) - if client and hasattr(client, 'get_track_details'): - try: - track_data = client.get_track_details(str(override_id)) - if track_data: - payload = _build_single_import_context_payload( - track_data, - chosen_source, - source_priority, - requested_title=title, - requested_artist=artist, - ) - if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): - try: - artist_details = client.get_artist(payload['context']['artist']['id']) - if artist_details: - payload['context']['artist']['genres'] = _extract_lookup_value( - artist_details, - 'genres', - default=[], - ) or [] - except Exception: - pass - return payload - except Exception as exc: - logger.debug("Override track lookup failed on %s for %s: %s", chosen_source, override_id, exc) - - for source in source_priority: - client = get_client_for_source(source) - if not client: - continue - - search_query = _build_track_search_query(source, title, artist) - if not search_query: - continue - - search_results = _search_tracks_for_source(source, client, search_query, limit=5) - if not search_results and search_query != title: - search_results = _search_tracks_for_source(source, client, title, limit=5) - if not search_results and artist and search_query != artist: - search_results = _search_tracks_for_source(source, client, artist, limit=5) - - if not search_results: - continue - - best_match = _pick_best_track_match(search_results, title or search_query, artist) - if not best_match: - continue - - resolved_track_id = str(_extract_lookup_value(best_match, 'id', 'track_id', 'trackId', default='') or '') - resolved_data = best_match - if resolved_track_id and hasattr(client, 'get_track_details'): - try: - detailed = client.get_track_details(resolved_track_id) - if detailed: - resolved_data = detailed - except Exception as exc: - logger.debug("Track detail lookup failed on %s for %s: %s", source, resolved_track_id, exc) - - payload = _build_single_import_context_payload( - resolved_data, - source, - source_priority, - requested_title=title, - requested_artist=artist, - ) - if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): - try: - artist_details = client.get_artist(payload['context']['artist']['id']) - if artist_details: - payload['context']['artist']['genres'] = _extract_lookup_value( - artist_details, - 'genres', - default=[], - ) or [] - except Exception: - pass - return payload - - return _build_single_import_fallback_context(title, artist, source_priority) + return _get_single_track_import_context( + title, + artist=artist, + override_id=override_id, + override_source=override_source, + source_override=source_override, + ) def resolve_album_reference( diff --git a/core/metadata_source.py b/core/metadata_source.py new file mode 100644 index 00000000..ce1ddbe4 --- /dev/null +++ b/core/metadata_source.py @@ -0,0 +1,896 @@ +"""Source metadata extraction and source-ID embedding helpers.""" + +from __future__ import annotations + +import re +import threading +import time +from typing import Any, Dict + +from core.import_context import ( + get_import_clean_artist, + get_import_clean_title, + get_import_context_album, + get_import_original_search, + get_import_source, + get_import_source_ids, + get_import_track_info, + get_source_tag_names, + normalize_import_context, +) +from core.metadata_common import ( + _extract_artist_name, + _get_config_manager, + _get_database, + _get_itunes_client, + _get_logger, + _get_mutagen_symbols, + _is_vorbis_like, +) + + +_MB_RELEASE_CACHE: Dict[tuple, str] = {} +_MB_RELEASE_CACHE_LOCK = threading.RLock() +_MB_RELEASE_DETAIL_CACHE: Dict[str, Dict[str, Any]] = {} +_MB_RELEASE_DETAIL_CACHE_LOCK = threading.RLock() + +_EDITION_PAREN_RE = re.compile( + r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' + r'limited|bonus|platinum|gold|super\s*deluxe|standard)' + r'(?:\s+(?:edition|version))?[^)\]]*[\)\]]', + re.IGNORECASE, +) +_EDITION_BARE_RE = re.compile( + r'\s+(?:-\s+)?(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' + r'limited|bonus|platinum|gold|super\s*deluxe|standard)' + r'(?:\s+(?:edition|version))?\s*$', + re.IGNORECASE, +) + + +def _normalize_album_cache_key(album_name: str) -> str: + result = _EDITION_PAREN_RE.sub("", album_name or "") + result = _EDITION_BARE_RE.sub("", result) + return result.lower().strip() + + +def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> dict: + if album_info is None: + album_info = {} + + cfg = _get_config_manager() + logger_ = _get_logger() + context = normalize_import_context(context) + original_search = get_import_original_search(context) + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + source = get_import_source(context) + source_ids = get_import_source_ids(context) + + artist_dict = artist if isinstance(artist, dict) else { + "name": _extract_artist_name(artist), + "id": getattr(artist, "id", ""), + "genres": list(getattr(artist, "genres", []) or []), + } + + metadata: Dict[str, Any] = { + "source": source, + "source_track_id": source_ids["track_id"], + "source_artist_id": source_ids["artist_id"], + "source_album_id": source_ids["album_id"], + } + + metadata["title"] = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "")) + if original_search.get("clean_title"): + logger_.info("Metadata: Using clean title: '%s'", metadata["title"]) + elif album_info.get("clean_track_name"): + logger_.info("Metadata: Using album info clean name: '%s'", metadata["title"]) + else: + logger_.warning("Metadata: Using original title as fallback: '%s'", metadata["title"]) + + artists = original_search.get("artists") + if isinstance(artists, list) and artists: + all_artists = [] + for artist_item in artists: + if isinstance(artist_item, dict) and artist_item.get("name"): + all_artists.append(artist_item["name"]) + elif isinstance(artist_item, str): + all_artists.append(artist_item) + else: + all_artists.append(str(artist_item)) + metadata["artist"] = ", ".join(all_artists) + logger_.info("Metadata: Using all artists: '%s'", metadata["artist"]) + else: + metadata["artist"] = artist_dict.get("name", "") or get_import_clean_artist(context) + logger_.info("Metadata: Using primary artist: '%s'", metadata["artist"]) + + raw_album_artist = artist_dict.get("name", "") or metadata["artist"] + track_info_ctx = track_info or {} + explicit_artist = track_info_ctx.get("_explicit_artist_context") if isinstance(track_info_ctx, dict) else None + album_artists_for_collab = None + + if isinstance(explicit_artist, dict) and explicit_artist.get("name"): + raw_album_artist = explicit_artist["name"] + album_artists_for_collab = [explicit_artist] + elif isinstance(explicit_artist, str) and explicit_artist: + raw_album_artist = explicit_artist + album_artists_for_collab = [{"name": explicit_artist}] + elif album_ctx and isinstance(album_ctx, dict): + album_artists = album_ctx.get("artists", []) + if album_artists: + first_album_artist = album_artists[0] + if isinstance(first_album_artist, dict) and first_album_artist.get("name"): + raw_album_artist = first_album_artist["name"] + elif isinstance(first_album_artist, str) and first_album_artist: + raw_album_artist = first_album_artist + album_artists_for_collab = album_artists + + collab_mode = cfg.get("file_organization.collab_artist_mode", "first") + if collab_mode == "first" and raw_album_artist: + context_artists = album_artists_for_collab or original_search.get("artists") or track_info_ctx.get("artists") or [] + if len(context_artists) > 1: + first = context_artists[0] + raw_album_artist = first.get("name", first) if isinstance(first, dict) else str(first) + elif len(context_artists) == 1 and ("," in raw_album_artist or " & " in raw_album_artist): + artist_id = str(artist_dict.get("id", "")) + if source == "itunes" and artist_id.isdigit(): + try: + itunes_client = _get_itunes_client() + if itunes_client and hasattr(itunes_client, "resolve_primary_artist"): + resolved = itunes_client.resolve_primary_artist(artist_id) + if resolved and resolved != raw_album_artist: + raw_album_artist = resolved + except Exception: + pass + metadata["album_artist"] = raw_album_artist + + if album_info.get("is_album"): + metadata["album"] = album_info.get("album_name", "Unknown Album") + metadata["track_number"] = album_info.get("track_number", 1) + metadata["total_tracks"] = album_ctx.get("total_tracks", 1) if album_ctx else 1 + logger_.info("[METADATA] Album track - track_number: %s, album: %s", metadata["track_number"], metadata["album"]) + else: + if album_ctx and album_ctx.get("name"): + logger_.info("[SAFEGUARD] Using album context name instead of track title for album metadata") + metadata["album"] = album_ctx["name"] + metadata["track_number"] = album_info.get("track_number", 1) if album_info else 1 + metadata["total_tracks"] = album_ctx.get("total_tracks", 1) + else: + metadata["album"] = metadata["title"] + metadata["track_number"] = 1 + metadata["total_tracks"] = 1 + + disc_num = original_search.get("disc_number") + if disc_num is None and album_info: + disc_num = album_info.get("disc_number") + metadata["disc_number"] = disc_num if disc_num is not None else 1 + + if album_ctx and album_ctx.get("release_date"): + metadata["date"] = album_ctx["release_date"][:4] + + genres = artist_dict.get("genres") or [] + if genres: + from core.genre_filter import filter_genres + + filtered = filter_genres(list(genres[:2]), cfg) + if filtered: + metadata["genre"] = ", ".join(filtered) + + metadata["album_art_url"] = album_info.get("album_image_url") if album_info else None + if not metadata["album_art_url"] and album_ctx: + album_image = album_ctx.get("image_url") + if not album_image and album_ctx.get("images"): + first_image = album_ctx["images"][0] + album_image = first_image.get("url") if isinstance(first_image, dict) else None + metadata["album_art_url"] = album_image + + logger_.info( + "[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | track=%s/%s | disc=%s", + metadata.get("title"), + metadata.get("artist"), + metadata.get("album_artist"), + metadata.get("album"), + metadata.get("track_number"), + metadata.get("total_tracks"), + metadata.get("disc_number"), + ) + + return metadata + + +def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=None): + cfg = _get_config_manager() + logger_ = _get_logger() + symbols = _get_mutagen_symbols() + if not symbols: + return + + try: + tag_config = { + "SPOTIFY_TRACK_ID": "spotify.tags.track_id", + "SPOTIFY_ARTIST_ID": "spotify.tags.artist_id", + "SPOTIFY_ALBUM_ID": "spotify.tags.album_id", + "ITUNES_TRACK_ID": "itunes.tags.track_id", + "ITUNES_ARTIST_ID": "itunes.tags.artist_id", + "ITUNES_ALBUM_ID": "itunes.tags.album_id", + "MUSICBRAINZ_RECORDING_ID": "musicbrainz.tags.recording_id", + "MUSICBRAINZ_ARTIST_ID": "musicbrainz.tags.artist_id", + "MUSICBRAINZ_RELEASE_ID": "musicbrainz.tags.release_id", + "MUSICBRAINZ_RELEASEGROUPID": "musicbrainz.tags.release_group_id", + "MUSICBRAINZ_ALBUMARTISTID": "musicbrainz.tags.album_artist_id", + "MUSICBRAINZ_RELEASETRACKID": "musicbrainz.tags.release_track_id", + "RELEASETYPE": "musicbrainz.tags.release_type", + "ORIGINALDATE": "musicbrainz.tags.original_date", + "RELEASESTATUS": "musicbrainz.tags.release_status", + "RELEASECOUNTRY": "musicbrainz.tags.release_country", + "BARCODE": "musicbrainz.tags.barcode", + "MEDIA": "musicbrainz.tags.media", + "TOTALDISCS": "musicbrainz.tags.total_discs", + "CATALOGNUMBER": "musicbrainz.tags.catalog_number", + "SCRIPT": "musicbrainz.tags.script", + "ASIN": "musicbrainz.tags.asin", + "DEEZER_TRACK_ID": "deezer.tags.track_id", + "DEEZER_ARTIST_ID": "deezer.tags.artist_id", + "AUDIODB_TRACK_ID": "audiodb.tags.track_id", + "TIDAL_TRACK_ID": "tidal.tags.track_id", + "TIDAL_ARTIST_ID": "tidal.tags.artist_id", + "QOBUZ_TRACK_ID": "qobuz.tags.track_id", + "QOBUZ_ARTIST_ID": "qobuz.tags.artist_id", + "GENIUS_TRACK_ID": "genius.tags.track_id", + } + + def _tag_enabled(path: str) -> bool: + return cfg.get(path, True) is not False + + def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: + if not a or not b: + return False + from difflib import SequenceMatcher + + norm = lambda s: re.sub(r"[^a-z0-9 ]", "", re.sub(r"\(.*?\)", "", s).lower()).strip() + return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold + + context = normalize_import_context(context) + source = (metadata.get("source") or "").strip().lower() + source_ids = {} + if source: + source_tag_names = get_source_tag_names(source) + source_track_id = metadata.get("source_track_id") + source_artist_id = metadata.get("source_artist_id") + source_album_id = metadata.get("source_album_id") + if cfg.get(f"{source}.embed_tags", True) is not False: + if source_tag_names.get("track") and source_track_id: + source_ids[source_tag_names["track"]] = source_track_id + if source_tag_names.get("artist") and source_artist_id: + source_ids[source_tag_names["artist"]] = source_artist_id + if source_tag_names.get("album") and source_album_id: + source_ids[source_tag_names["album"]] = source_album_id + + if not source_ids: + if cfg.get("spotify.embed_tags", True) is not False: + if metadata.get("spotify_track_id"): + source_ids["SPOTIFY_TRACK_ID"] = metadata["spotify_track_id"] + if metadata.get("spotify_artist_id"): + source_ids["SPOTIFY_ARTIST_ID"] = metadata["spotify_artist_id"] + if metadata.get("spotify_album_id"): + source_ids["SPOTIFY_ALBUM_ID"] = metadata["spotify_album_id"] + if cfg.get("itunes.embed_tags", True) is not False: + if metadata.get("itunes_track_id"): + source_ids["ITUNES_TRACK_ID"] = metadata["itunes_track_id"] + if metadata.get("itunes_artist_id"): + source_ids["ITUNES_ARTIST_ID"] = metadata["itunes_artist_id"] + if metadata.get("itunes_album_id"): + source_ids["ITUNES_ALBUM_ID"] = metadata["itunes_album_id"] + + track_title = metadata.get("title", "") + artist_name = metadata.get("album_artist", "") or metadata.get("artist", "") + track_info = get_import_track_info(context) + explicit_artist = (track_info or {}).get("_explicit_artist_context") if isinstance(track_info, dict) else None + batch_artist_name = None + if isinstance(explicit_artist, dict) and explicit_artist.get("name"): + batch_artist_name = explicit_artist["name"] + elif isinstance(explicit_artist, str) and explicit_artist: + batch_artist_name = explicit_artist + + pp = { + "id_tags": source_ids, + "track_title": track_title, + "artist_name": artist_name, + "batch_artist_name": batch_artist_name, + "metadata": metadata, + "recording_mbid": None, + "artist_mbid": None, + "release_mbid": "", + "mb_genres": [], + "isrc": None, + "deezer_bpm": None, + "deezer_isrc": None, + "audiodb_mood": None, + "audiodb_style": None, + "audiodb_genre": None, + "tidal_isrc": None, + "tidal_copyright": None, + "qobuz_isrc": None, + "qobuz_copyright": None, + "qobuz_label": None, + "lastfm_tags": [], + "lastfm_url": None, + "genius_url": None, + "release_year": None, + } + + source_order = cfg.get("metadata_enhancement.post_process_order", None) + if not isinstance(source_order, list) or not source_order: + source_order = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"] + + db = _get_database() + + for source_name in source_order: + if source_name == "musicbrainz": + if cfg.get("musicbrainz.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + mb_worker = getattr(runtime, "mb_worker", None) + mb_service = mb_worker.mb_service if mb_worker else None + if not mb_service: + continue + try: + result = mb_service.match_recording(track_title, artist_name) + if result and result.get("mbid"): + pp["recording_mbid"] = result["mbid"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"] + details = mb_service.mb_client.get_recording(pp["recording_mbid"], includes=["isrcs", "genres"]) + if details: + isrcs = details.get("isrcs", []) + if isrcs: + pp["isrc"] = isrcs[0] + pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)] + + track_artist_name = metadata.get("artist", "") or artist_name + if ", " in track_artist_name: + track_artist_name = track_artist_name.split(", ")[0] + artist_result = mb_service.match_artist(track_artist_name) + if artist_result and artist_result.get("mbid"): + pp["artist_mbid"] = artist_result["mbid"] + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"] + + album_name_for_mb = metadata.get("album", "") + if album_name_for_mb: + artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() + rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), artist_key) + rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) + with _MB_RELEASE_CACHE_LOCK: + cached = _MB_RELEASE_CACHE.get(rc_key_norm) + if cached is None: + cached = _MB_RELEASE_CACHE.get(rc_key_exact) + if cached is not None: + pp["release_mbid"] = cached + else: + try: + rc_result = mb_service.match_release(album_name_for_mb, artist_name) + pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" + except Exception: + pp["release_mbid"] = "" + _MB_RELEASE_CACHE[rc_key_norm] = pp["release_mbid"] + _MB_RELEASE_CACHE[rc_key_exact] = pp["release_mbid"] + if pp["release_mbid"]: + pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] + + if pp["release_mbid"]: + with _MB_RELEASE_DETAIL_CACHE_LOCK: + release_detail = _MB_RELEASE_DETAIL_CACHE.get(pp["release_mbid"]) + if release_detail is None: + release_detail = mb_service.mb_client.get_release( + pp["release_mbid"], + includes=["release-groups", "labels", "media", "artist-credits", "recordings"], + ) or {} + with _MB_RELEASE_DETAIL_CACHE_LOCK: + _MB_RELEASE_DETAIL_CACHE[pp["release_mbid"]] = release_detail + if release_detail: + rg = release_detail.get("release-group", {}) + if rg.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"] + ac = release_detail.get("artist-credit", []) + if ac and isinstance(ac[0], dict): + aa = ac[0].get("artist", {}) + if aa.get("id"): + pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"] + if rg.get("primary-type"): + pp["id_tags"]["RELEASETYPE"] = rg["primary-type"] + if rg.get("first-release-date"): + pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"] + if not pp["release_year"] and len(rg["first-release-date"]) >= 4: + year = rg["first-release-date"][:4] + if year.isdigit(): + pp["release_year"] = year + if release_detail.get("status"): + pp["id_tags"]["RELEASESTATUS"] = release_detail["status"] + if release_detail.get("country"): + pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"] + if release_detail.get("barcode"): + pp["id_tags"]["BARCODE"] = release_detail["barcode"] + media_list = release_detail.get("media", []) + if media_list: + fmt = media_list[0].get("format", "") + if fmt: + pp["id_tags"]["MEDIA"] = fmt + pp["id_tags"]["TOTALDISCS"] = str(len(media_list)) + label_info = release_detail.get("label-info", []) + if label_info and isinstance(label_info[0], dict): + cat = label_info[0].get("catalog-number", "") + if cat: + pp["id_tags"]["CATALOGNUMBER"] = cat + text_rep = release_detail.get("text-representation", {}) + if isinstance(text_rep, dict) and text_rep.get("script"): + pp["id_tags"]["SCRIPT"] = text_rep["script"] + if release_detail.get("asin"): + pp["id_tags"]["ASIN"] = release_detail["asin"] + track_num = metadata.get("track_number") + disc_num = metadata.get("disc_number") or 1 + if track_num and media_list: + try: + track_num_int = int(track_num) + disc_num_int = int(disc_num) + for medium in media_list: + if medium.get("position", 1) == disc_num_int: + for mtrack in (medium.get("tracks") or medium.get("track-list", [])): + if mtrack.get("position") == track_num_int: + if mtrack.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"] + release_recording = mtrack.get("recording", {}) + if release_recording.get("id"): + pp["recording_mbid"] = release_recording["id"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"] + break + break + except (ValueError, TypeError): + pass + except Exception as exc: + logger_.error("MusicBrainz lookup failed (non-fatal): %s", exc) + continue + + if source_name == "deezer": + if cfg.get("deezer.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + deezer_worker = getattr(runtime, "deezer_worker", None) + dz_client = deezer_worker.client if deezer_worker else None + if not dz_client: + continue + dz_result = dz_client.search_track(artist_name, track_title) + if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name): + dz_track_id = dz_result["id"] + pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id) + dz_artist_id = dz_result.get("artist", {}).get("id") + if dz_artist_id: + pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id) + dz_details = dz_client.get_track_details(dz_track_id) + if dz_details: + bpm_val = dz_details.get("bpm") + if bpm_val and bpm_val > 0: + pp["deezer_bpm"] = bpm_val + dz_isrc = dz_details.get("isrc") + if dz_isrc: + pp["deezer_isrc"] = dz_isrc + if not pp["release_year"]: + dz_album = dz_result.get("album", {}) + dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or "" + if len(dz_release) >= 4 and dz_release[:4].isdigit(): + pp["release_year"] = dz_release[:4] + except Exception as exc: + logger_.error("Deezer lookup failed (non-fatal): %s", exc) + continue + + if source_name == "audiodb": + if cfg.get("audiodb.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + audiodb_worker = getattr(runtime, "audiodb_worker", None) + adb_client = audiodb_worker.client if audiodb_worker else None + if not adb_client: + continue + adb_result = adb_client.search_track(artist_name, track_title) + if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name): + adb_track_id = adb_result.get("idTrack") + if adb_track_id: + pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id) + adb_mb_track = adb_result.get("strMusicBrainzID") + if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track + pp["recording_mbid"] = adb_mb_track + adb_mb_artist = adb_result.get("strMusicBrainzArtistID") + if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist + pp["artist_mbid"] = adb_mb_artist + pp["audiodb_mood"] = adb_result.get("strMood") or None + pp["audiodb_style"] = adb_result.get("strStyle") or None + pp["audiodb_genre"] = adb_result.get("strGenre") or None + except Exception as exc: + logger_.error("AudioDB lookup failed (non-fatal): %s", exc) + continue + + if source_name == "tidal": + if cfg.get("tidal.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + tidal_client = getattr(runtime, "tidal_client", None) + if not (tidal_client and tidal_client.is_authenticated()): + continue + td_result = tidal_client.search_track(artist_name, track_title) + if td_result and _names_match(td_result.get("title", ""), track_title): + td_track_id = td_result.get("id") + if td_track_id: + pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id) + td_artist = td_result.get("artist", {}) + if isinstance(td_artist, dict) and td_artist.get("id"): + pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"]) + if td_track_id: + td_details = tidal_client.get_track(str(td_track_id)) + if td_details: + pp["tidal_isrc"] = td_details.get("isrc") + td_copyright = td_details.get("copyright") + if isinstance(td_copyright, dict): + td_copyright = td_copyright.get("text", td_copyright.get("name", "")) + pp["tidal_copyright"] = td_copyright or None + if not pp["release_year"]: + td_album = td_result.get("album", {}) + td_release = "" + if isinstance(td_album, dict): + td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "") + if len(td_release) >= 4 and td_release[:4].isdigit(): + pp["release_year"] = td_release[:4] + except Exception as exc: + logger_.error("Tidal lookup failed (non-fatal): %s", exc) + continue + + if source_name == "qobuz": + if cfg.get("qobuz.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None) + qz_client = qobuz_worker.client if qobuz_worker else None + if not (qz_client and qz_client.is_authenticated()): + continue + qz_result = qz_client.search_track(artist_name, track_title) + if qz_result: + qz_performer = qz_result.get("performer") or {} + if not isinstance(qz_performer, dict): + qz_performer = {} + qz_artist_name = qz_performer.get("name", "") + if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name): + qz_track_id = qz_result.get("id") + if qz_track_id: + pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id) + if qz_performer.get("id"): + pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"]) + qz_isrc = qz_result.get("isrc") + if isinstance(qz_isrc, dict): + qz_isrc = qz_isrc.get("value", qz_isrc.get("id", "")) + if qz_isrc: + pp["qobuz_isrc"] = qz_isrc + qz_copyright = qz_result.get("copyright") + if isinstance(qz_copyright, dict): + qz_copyright = qz_copyright.get("text", qz_copyright.get("name", "")) + if isinstance(qz_copyright, str): + pp["qobuz_copyright"] = qz_copyright + qz_album = qz_result.get("album", {}) + if isinstance(qz_album, dict): + qz_label_info = qz_album.get("label", {}) + if isinstance(qz_label_info, dict) and qz_label_info.get("name"): + pp["qobuz_label"] = qz_label_info["name"] + if not pp["release_year"]: + qz_release = str(qz_album.get("release_date_original", "") or "") + if not qz_release: + qz_ts = qz_album.get("released_at") + if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: + import datetime as _dt + qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) + if len(qz_release) >= 4 and qz_release[:4].isdigit(): + pp["release_year"] = qz_release[:4] + except Exception as exc: + logger_.error("Qobuz lookup failed (non-fatal): %s", exc) + continue + + if source_name == "lastfm": + if cfg.get("lastfm.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + lastfm_worker = getattr(runtime, "lastfm_worker", None) + lf_client = lastfm_worker.client if lastfm_worker else None + if not lf_client: + continue + lf_result = lf_client.get_track_info(artist_name, track_title) + if lf_result: + lf_url = lf_result.get("url") + if lf_url: + pp["lastfm_url"] = lf_url + lf_toptags = lf_result.get("toptags", {}) + if isinstance(lf_toptags, dict): + tag_list = lf_toptags.get("tag", []) + if isinstance(tag_list, list): + pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")] + elif isinstance(tag_list, dict) and tag_list.get("name"): + pp["lastfm_tags"] = [tag_list["name"]] + except Exception as exc: + logger_.error("Last.fm lookup failed (non-fatal): %s", exc) + continue + + if source_name == "genius": + if cfg.get("genius.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + import core.genius_client as _genius_module + + if time.time() < _genius_module._rate_limit_until: + logger_.info("Genius rate-limited, skipping (non-blocking)") + continue + genius_worker = getattr(runtime, "genius_worker", None) + g_client = genius_worker.client if genius_worker else None + if not g_client: + continue + g_result = g_client.search_song(artist_name, track_title) + if g_result: + g_id = g_result.get("id") + if g_id: + pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id) + g_url = g_result.get("url") + if g_url: + pp["genius_url"] = g_url + except Exception as exc: + logger_.error("Genius lookup failed (non-fatal): %s", exc) + continue + + if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["audiodb_mood"] and not pp["audiodb_style"]: + return + + filtered_tags: Dict[str, str] = {} + for tag_name, value in pp["id_tags"].items(): + config_path = tag_config.get(tag_name) + if config_path and not _tag_enabled(config_path): + continue + filtered_tags[tag_name] = value + + written = [] + id3_tag_map = { + "MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"), + "MUSICBRAINZ_ARTIST_ID": ("TXXX", "MusicBrainz Artist Id"), + "MUSICBRAINZ_RELEASE_ID": ("TXXX", "MusicBrainz Album Id"), + "MUSICBRAINZ_RELEASEGROUPID": ("TXXX", "MusicBrainz Release Group Id"), + "MUSICBRAINZ_ALBUMARTISTID": ("TXXX", "MusicBrainz Album Artist Id"), + "MUSICBRAINZ_RELEASETRACKID": ("TXXX", "MusicBrainz Release Track Id"), + "RELEASETYPE": ("TXXX", "MusicBrainz Album Type"), + "RELEASESTATUS": ("TXXX", "MusicBrainz Album Status"), + "RELEASECOUNTRY": ("TXXX", "MusicBrainz Album Release Country"), + "ORIGINALDATE": ("TDOR", None), + "MEDIA": ("TMED", None), + } + vorbis_tag_map = { + "MUSICBRAINZ_RECORDING_ID": "MUSICBRAINZ_TRACKID", + "MUSICBRAINZ_ARTIST_ID": "MUSICBRAINZ_ARTISTID", + "MUSICBRAINZ_RELEASE_ID": "MUSICBRAINZ_ALBUMID", + "MUSICBRAINZ_RELEASEGROUPID": "MUSICBRAINZ_RELEASEGROUPID", + "MUSICBRAINZ_ALBUMARTISTID": "MUSICBRAINZ_ALBUMARTISTID", + "MUSICBRAINZ_RELEASETRACKID": "MUSICBRAINZ_RELEASETRACKID", + } + mp4_tag_map = { + "MUSICBRAINZ_RECORDING_ID": "MusicBrainz Track Id", + "MUSICBRAINZ_ARTIST_ID": "MusicBrainz Artist Id", + "MUSICBRAINZ_RELEASE_ID": "MusicBrainz Album Id", + "MUSICBRAINZ_RELEASEGROUPID": "MusicBrainz Release Group Id", + "MUSICBRAINZ_ALBUMARTISTID": "MusicBrainz Album Artist Id", + "MUSICBRAINZ_RELEASETRACKID": "MusicBrainz Release Track Id", + "RELEASETYPE": "MusicBrainz Album Type", + "RELEASESTATUS": "MusicBrainz Album Status", + "RELEASECOUNTRY": "MusicBrainz Album Release Country", + } + + if isinstance(audio_file.tags, symbols.ID3): + for tag_name, value in filtered_tags.items(): + spec = id3_tag_map.get(tag_name) + if spec: + frame_type, desc = spec + if frame_type == "UFID": + audio_file.tags.add(symbols.UFID(owner=desc, data=str(value).encode("ascii"))) + written.append(f"UFID:{desc}") + elif frame_type == "TDOR": + audio_file.tags.add(symbols.TDOR(encoding=3, text=[value])) + written.append("TDOR") + elif frame_type == "TMED": + audio_file.tags.add(symbols.TMED(encoding=3, text=[value])) + written.append("TMED") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=desc, text=[value])) + written.append(f"TXXX:{desc}") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=tag_name, text=[str(value)])) + written.append(f"TXXX:{tag_name}") + elif isinstance(audio_file, symbols.MP4): + # Keep the dedicated MP4 path last so the same tag maps can be reused. + for tag_name, value in filtered_tags.items(): + key = f"----:com.apple.iTunes:{mp4_tag_map.get(tag_name, tag_name)}" + audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))] + written.append(key) + elif _is_vorbis_like(audio_file, symbols): + for tag_name, value in filtered_tags.items(): + audio_file[vorbis_tag_map.get(tag_name, tag_name)] = [str(value)] + written.append(vorbis_tag_map.get(tag_name, tag_name)) + + if written: + logger_.info("Embedded IDs: %s", ", ".join(written)) + + release_year = pp["release_year"] + needs_date_tag = bool(release_year and not metadata.get("date")) + if needs_date_tag: + metadata["date"] = release_year + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["date"] = [release_year] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9day"] = [release_year] + logger_.info("Date tag: %s", release_year) + + if _tag_enabled("deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0: + bpm_int = int(pp["deezer_bpm"]) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["BPM"] = [str(bpm_int)] + elif isinstance(audio_file, symbols.MP4): + audio_file["tmpo"] = [bpm_int] + logger_.info("BPM: %s", bpm_int) + + if _tag_enabled("audiodb.tags.mood") and pp["audiodb_mood"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["MOOD"] = [pp["audiodb_mood"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))] + + if _tag_enabled("audiodb.tags.style") and pp["audiodb_style"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["STYLE"] = [pp["audiodb_style"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))] + + if _tag_enabled("metadata_enhancement.tags.genre_merge"): + enrichment_genres = [] + if _tag_enabled("musicbrainz.tags.genres"): + enrichment_genres += pp["mb_genres"] + if pp["audiodb_genre"] and _tag_enabled("audiodb.tags.genre"): + enrichment_genres.append(pp["audiodb_genre"]) + if _tag_enabled("lastfm.tags.genres"): + enrichment_genres += pp["lastfm_tags"] + if enrichment_genres: + from core.genre_filter import filter_genres as _filter_genres + + enrichment_genres = _filter_genres(enrichment_genres, cfg) + source_genres = [g.strip() for g in str(metadata.get("genre", "")).split(",") if g.strip()] + seen = set() + merged = [] + for genre in source_genres + enrichment_genres: + key = genre.strip().lower() + if key and key not in seen: + seen.add(key) + merged.append(genre.strip().title()) + if len(merged) >= 5: + break + if merged: + genre_string = ", ".join(merged) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["GENRE"] = [genre_string] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9gen"] = [genre_string] + logger_.info("Genres merged: %s", genre_string) + + isrc_candidates = [] + if pp["isrc"] and _tag_enabled("musicbrainz.tags.isrc"): + isrc_candidates.append(("MusicBrainz", pp["isrc"])) + if pp["deezer_isrc"] and _tag_enabled("deezer.tags.isrc"): + isrc_candidates.append(("Deezer", pp["deezer_isrc"])) + if pp["tidal_isrc"] and _tag_enabled("tidal.tags.isrc"): + isrc_candidates.append(("Tidal", pp["tidal_isrc"])) + if pp["qobuz_isrc"] and _tag_enabled("qobuz.tags.isrc"): + isrc_candidates.append(("Qobuz", pp["qobuz_isrc"])) + if isrc_candidates: + isrc_source, final_isrc = isrc_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["ISRC"] = [final_isrc] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))] + logger_.info("ISRC (%s): %s", isrc_source, final_isrc) + + copyright_candidates = [] + if pp["tidal_copyright"] and _tag_enabled("tidal.tags.copyright"): + copyright_candidates.append(("Tidal", pp["tidal_copyright"])) + if pp["qobuz_copyright"] and _tag_enabled("qobuz.tags.copyright"): + copyright_candidates.append(("Qobuz", pp["qobuz_copyright"])) + if copyright_candidates: + copyright_source, final_copyright = copyright_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["COPYRIGHT"] = [final_copyright] + elif isinstance(audio_file, symbols.MP4): + audio_file["cprt"] = [final_copyright] + logger_.info("Copyright (%s): %s", copyright_source, final_copyright[:60]) + + if _tag_enabled("qobuz.tags.label") and pp["qobuz_label"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["LABEL"] = [pp["qobuz_label"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))] + + if _tag_enabled("lastfm.tags.url") and pp["lastfm_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["LASTFM_URL"] = [pp["lastfm_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))] + + if _tag_enabled("genius.tags.url") and pp["genius_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["GENIUS_URL"] = [pp["genius_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))] + + release_id = pp["release_mbid"] + if release_id: + metadata["musicbrainz_release_id"] = release_id + if db is not None: + try: + album_name_for_db = metadata.get("album", "") + album_artist_for_db = metadata.get("album_artist", "") or metadata.get("artist", "") + if album_name_for_db and album_artist_for_db: + conn = db._get_connection() + try: + cursor = conn.cursor() + cursor.execute( + """ + UPDATE albums SET year = ? + WHERE (year IS NULL OR year = 0) + AND id IN ( + SELECT al.id FROM albums al + JOIN artists ar ON ar.id = al.artist_id + WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) + ) + """, + (int(release_year), album_name_for_db, album_artist_for_db), + ) + if cursor.rowcount > 0: + conn.commit() + logger_.info("Updated album year to %s in database", release_year) + else: + conn.rollback() + finally: + conn.close() + except Exception as exc: + logger_.error("Could not update album year in DB: %s", exc) + + except Exception as exc: + logger_.error("Error embedding source IDs (non-fatal): %s", exc) diff --git a/tests/test_metadata_enrichment.py b/tests/test_metadata_enrichment.py index 49cbe630..8f557972 100644 --- a/tests/test_metadata_enrichment.py +++ b/tests/test_metadata_enrichment.py @@ -1,9 +1,8 @@ -import logging import types -import pytest - from core import metadata_enrichment as me +from core import metadata_artwork as ma +from core import metadata_source as ms class _Config: @@ -110,20 +109,9 @@ def _fake_symbols(audio): ) -def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_for_non_itunes_sources(): - class _ItunesClient: - def __init__(self): - self.called = False - - def resolve_primary_artist(self, artist_id): - self.called = True - raise AssertionError("itunes fallback should not run for non-itunes sources") - - runtime = types.SimpleNamespace( - logger=logging.getLogger("test.metadata_enrichment"), - config_manager=_Config({"file_organization.collab_artist_mode": "first"}), - itunes_enrichment_worker=types.SimpleNamespace(client=_ItunesClient()), - ) +def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_for_non_itunes_sources(monkeypatch): + monkeypatch.setattr(ms, "_get_config_manager", lambda: _Config({"file_organization.collab_artist_mode": "first"})) + monkeypatch.setattr(ms, "_get_itunes_client", lambda: (_ for _ in ()).throw(AssertionError("itunes fallback should not run for non-itunes sources"))) context = { "source": "spotify", @@ -156,7 +144,6 @@ def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_ context, context["artist"], {"is_album": True, "album_name": "Album One", "track_number": 3, "disc_number": 2, "album_image_url": "https://img.example/album.jpg"}, - runtime=runtime, ) assert metadata["source"] == "spotify" @@ -168,27 +155,14 @@ def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_ assert metadata["total_tracks"] == 12 assert metadata["disc_number"] == 2 assert metadata["album_art_url"] == "https://img.example/album.jpg" - assert runtime.itunes_enrichment_worker.client.called is False def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatch): - runtime = types.SimpleNamespace( - logger=logging.getLogger("test.metadata_enrichment"), - config_manager=_Config(), - mb_worker=None, - deezer_worker=None, - audiodb_worker=None, - tidal_client=None, - qobuz_enrichment_worker=None, - lastfm_worker=None, - genius_worker=None, - itunes_enrichment_worker=None, - get_database=lambda: None, - ) - audio = _FakeAudio() symbols = _fake_symbols(audio) - monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + monkeypatch.setattr(ms, "_get_config_manager", lambda: _Config()) + monkeypatch.setattr(ms, "_get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "_get_database", lambda: None) current_metadata = { "source": "deezer", @@ -200,7 +174,7 @@ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatc "album_artist": "Artist One", "album": "Album One", } - me.embed_source_ids(audio, current_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime) + me.embed_source_ids(audio, current_metadata, context={"track_info": {}, "original_search_result": {}}) current_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"] assert "DEEZER_TRACK_ID" in current_descs @@ -208,7 +182,7 @@ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatc audio = _FakeAudio() symbols = _fake_symbols(audio) - monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + monkeypatch.setattr(ms, "_get_mutagen_symbols", lambda: symbols) legacy_metadata = { "source": "", @@ -223,7 +197,7 @@ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatc "album_artist": "Artist One", "album": "Album One", } - me.embed_source_ids(audio, legacy_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime) + me.embed_source_ids(audio, legacy_metadata, context={"track_info": {}, "original_search_result": {}}) legacy_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"] assert "SPOTIFY_TRACK_ID" in legacy_descs @@ -237,35 +211,25 @@ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatc def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch): audio = _FakeAudio() symbols = _fake_symbols(audio) - runtime = types.SimpleNamespace( - logger=logging.getLogger("test.metadata_enrichment"), - config_manager=_Config( - { - "metadata_enhancement.enabled": True, - "metadata_enhancement.embed_album_art": False, - "metadata_enhancement.tags.write_multi_artist": False, - } - ), - mb_worker=None, - deezer_worker=None, - audiodb_worker=None, - tidal_client=None, - qobuz_enrichment_worker=None, - lastfm_worker=None, - genius_worker=None, - itunes_enrichment_worker=None, - get_database=lambda: None, - ) - strip_calls = [] verify_calls = [] - monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) - monkeypatch.setattr(me, "_strip_all_non_audio_tags", lambda file_path, runtime=None: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0}) + monkeypatch.setattr(me, "_get_config_manager", lambda: _Config( + { + "metadata_enhancement.enabled": True, + "metadata_enhancement.embed_album_art": False, + "metadata_enhancement.tags.write_multi_artist": False, + } + )) + monkeypatch.setattr(ms, "_get_config_manager", lambda: _Config()) + monkeypatch.setattr(me, "_get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "_get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "_get_database", lambda: None) + monkeypatch.setattr(me, "_strip_all_non_audio_tags", lambda file_path: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0}) monkeypatch.setattr( me, "extract_source_metadata", - lambda context, artist, album_info, runtime=None: { + lambda context, artist, album_info: { "source": "deezer", "source_track_id": "dz-track", "source_artist_id": "dz-artist", @@ -283,7 +247,7 @@ def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch }, ) monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None) - monkeypatch.setattr(me, "_verify_metadata_written", lambda file_path, runtime=None: verify_calls.append(file_path) or True) + monkeypatch.setattr(me, "_verify_metadata_written", lambda file_path: verify_calls.append(file_path) or True) album_info = {} result = me.enhance_file_metadata( @@ -291,7 +255,6 @@ def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch {"_audio_quality": ""}, {"name": "Artist One"}, album_info, - runtime=runtime, ) assert result is True @@ -306,17 +269,14 @@ def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch): - runtime = types.SimpleNamespace( - logger=logging.getLogger("test.metadata_enrichment"), - config_manager=_Config( - { - "metadata_enhancement.cover_art_download": True, - "metadata_enhancement.prefer_caa_art": False, - } - ), - ) + monkeypatch.setattr(ma, "_get_config_manager", lambda: _Config( + { + "metadata_enhancement.cover_art_download": True, + "metadata_enhancement.prefer_caa_art": False, + } + )) - monkeypatch.setattr(me.urllib.request, "urlopen", lambda *args, **kwargs: _FakeResponse(b"cover-bytes")) + monkeypatch.setattr(ma.urllib.request, "urlopen", lambda *args, **kwargs: _FakeResponse(b"cover-bytes")) target_dir = tmp_path / "Album One" target_dir.mkdir() @@ -325,7 +285,6 @@ def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch): {}, str(target_dir), {"album": {"image_url": "https://img.example/album.jpg"}}, - runtime=runtime, ) cover_path = target_dir / "cover.jpg" diff --git a/web_server.py b/web_server.py index 504ba7b5..e6cd49c2 100644 --- a/web_server.py +++ b/web_server.py @@ -559,6 +559,8 @@ def _make_context_key(username, filename): normalized = filename.replace('\\', '/').lstrip('/') if filename else '' return f"{username}::{normalized}" +IS_SHUTTING_DOWN = False + # --- Initialize Core Application Components --- # Each client is initialized independently so one failure doesn't take down everything. # Previously, a single exception set ALL clients to None, breaking the entire app. @@ -650,6 +652,1563 @@ except Exception as e: logger.info("Core service initialization complete.") +# --- Shared Runtime State --- +# These globals are used by routes, background workers, and shutdown helpers. +# A prior refactor accidentally dropped this initializer block, so several +# modules and handlers were still referencing names that never got created. + +# Global Streaming State Management +stream_state = { + "status": "stopped", # States: stopped, loading, queued, ready, error + "progress": 0, + "track_info": None, + "file_path": None, # Path to the audio file in the Stream folder + "error_message": None, +} +stream_lock = threading.Lock() # Prevent race conditions +stream_background_task = None +stream_executor = ThreadPoolExecutor(max_workers=1) # Only one stream at a time + +# Global OAuth State Management +# Store PKCE values for Tidal OAuth flow +tidal_oauth_state = { + "code_verifier": None, + "code_challenge": None, +} +tidal_oauth_lock = threading.Lock() + +# Sync Page Globals +sync_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="SyncWorker") +active_sync_workers = {} # Key: playlist_id, Value: Future object +sync_states = {} # Key: playlist_id, Value: dict with progress info +sync_lock = threading.Lock() + +# Database Update / Tool Progress State +db_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DBUpdate") +db_update_worker = None +db_update_state = { + "status": "idle", # idle, running, finished, error + "phase": "Idle", + "progress": 0, + "current_item": "", + "processed": 0, + "total": 0, + "error_message": "", + "removed_artists": 0, + "removed_albums": 0, + "removed_tracks": 0, +} +_db_update_automation_id = None # Set when automation triggers DB update, used by callbacks +db_update_lock = threading.Lock() + +# Quality Scanner state +quality_scanner_state = { + "status": "idle", # idle, running, finished, error + "phase": "Ready to scan", + "progress": 0, + "processed": 0, + "total": 0, + "quality_met": 0, + "low_quality": 0, + "matched": 0, + "error_message": "", + "results": [], # List of low quality tracks with match status +} +quality_scanner_lock = threading.Lock() +quality_scanner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="QualityScanner") + +# Duplicate Cleaner state +duplicate_cleaner_state = { + "status": "idle", # idle, running, finished, error + "phase": "Ready to scan", + "progress": 0, + "files_scanned": 0, + "total_files": 0, + "duplicates_found": 0, + "deleted": 0, + "space_freed": 0, # in bytes + "error_message": "", +} +duplicate_cleaner_lock = threading.Lock() +duplicate_cleaner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DuplicateCleaner") + +# Retag Tool Globals +retag_state = { + "status": "idle", + "phase": "Ready", + "progress": 0, + "current_track": "", + "total_tracks": 0, + "processed": 0, + "error_message": "", +} +retag_lock = threading.Lock() +retag_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="RetagWorker") + +# Download Missing Tracks Modal State Management +# Thread-safe state tracking for modal download functionality. +# Shared task/batch state now lives in core.import_runtime_state. +missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker") + +# Automatic Wishlist / Watchlist Processing Flags +# Processing state flags (guards/recovery - timers are now managed by AutomationEngine) +wishlist_auto_processing = False +wishlist_auto_processing_timestamp = 0 +wishlist_timer_lock = threading.Lock() + +watchlist_auto_scanning = False +watchlist_auto_scanning_timestamp = 0 +watchlist_timer_lock = threading.Lock() + +# Beatport Data Cache +# Cache Beatport scraping data to reduce load times and avoid hammering Beatport.com +beatport_data_cache = { + 'homepage': { + 'hero_tracks': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'top_10_lists': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'top_10_releases': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'new_releases': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'hype_picks': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'featured_charts': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + 'dj_charts': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours + }, + 'genre': { + # Future expansion for genre-specific caching + # 'house': {'top_10': {...}, 'releases': {...}}, + # 'techno': {'top_10': {...}, 'releases': {...}} + }, + 'cache_lock': threading.Lock(), +} + +# Shared Soulseek transfer cache +# Keeps download status polling from hammering the API when multiple modals are open. +transfer_data_cache = { + 'data': {}, + 'last_update': 0, + 'update_lock': threading.Lock(), + 'cache_duration': 0.75, +} + + +def get_cached_transfer_data(): + """ + Return live Soulseek transfer data with a short cache window. + + The download modal, batch status endpoint, and socket emit loop all call + this helper, so we keep the API hit rate low while still refreshing often. + """ + current_time = time.time() + + with transfer_data_cache['update_lock']: + if (current_time - transfer_data_cache['last_update']) < transfer_data_cache['cache_duration']: + return transfer_data_cache['data'] + + if not soulseek_client: + return {} + + live_transfers_lookup = {} + try: + transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + if transfers_data: + for user_data in transfers_data: + if not isinstance(user_data, dict): + continue + username = user_data.get('username', 'Unknown') + for directory in user_data.get('directories', []) or []: + if not isinstance(directory, dict): + continue + for file_info in directory.get('files', []) or []: + if not isinstance(file_info, dict): + continue + transfer = dict(file_info) + transfer['username'] = username + lookup_key = _make_context_key(username, transfer.get('filename', '')) + live_transfers_lookup[lookup_key] = transfer + + transfer_data_cache['data'] = live_transfers_lookup + transfer_data_cache['last_update'] = current_time + except Exception as e: + logger.warning("Could not fetch live transfers (cached): %s", e) + return {} + + return live_transfers_lookup + + +def _regenerate_batch_m3u(batch, tracks): + """Regenerate an exported playlist M3U after downloads finish.""" + try: + from collections import defaultdict + from difflib import SequenceMatcher + + try: + from unidecode import unidecode as _unidecode + except ImportError: + _unidecode = lambda x: x + + from core.import_paths import sanitize_filename as _sanitize_filename + + def _norm(text): + return _unidecode(text).lower().strip() if text else '' + + def _track_name(track): + return track.get('name') or track.get('title') or 'Unknown' + + def _track_artist(track): + return track.get('artist') or track.get('artist_name') or 'Unknown' + + playlist_name = batch.get('playlist_name', 'Playlist') + db = get_database() + active_server = config_manager.get_active_media_server() + raw_base = config_manager.get('m3u_export.entry_base_path', '') or '' + entry_base_path = raw_base.rstrip('/\\') + + artist_groups = defaultdict(list) + for idx, track in enumerate(tracks): + artist_groups[_track_artist(track)].append((idx, track)) + + file_path_map = {} + for artist, group in artist_groups.items(): + if not artist or artist == 'Unknown': + for idx, _ in group: + file_path_map[idx] = None + continue + + try: + db_tracks = db.search_tracks(artist=artist, limit=500, server_source=active_server) + except Exception as search_err: + logger.debug("[M3U] Track lookup failed for %s: %s", artist, search_err) + db_tracks = [] + + if not db_tracks: + for idx, _ in group: + file_path_map[idx] = None + continue + + db_entries = [] + for db_track in db_tracks: + db_title = getattr(db_track, 'title', '') or '' + db_entries.append((_norm(db_title), db_track)) + + for idx, track in group: + name = _track_name(track) + if not name: + file_path_map[idx] = None + continue + + s_norm = _norm(name) + matched = None + for db_n, db_track in db_entries: + if not db_n: + continue + if s_norm == db_n or SequenceMatcher(None, s_norm, db_n).ratio() >= 0.7: + matched = db_track + break + + if matched is None: + file_path_map[idx] = None + else: + file_path = getattr(matched, 'file_path', None) + if file_path is None and isinstance(matched, dict): + file_path = matched.get('file_path') + file_path_map[idx] = file_path or None + + lines = ['#EXTM3U', f'#PLAYLIST:{playlist_name}', f'#GENERATED:{datetime.utcnow().isoformat()}Z', ''] + found = 0 + missing = 0 + for idx, track in enumerate(tracks): + duration_ms = track.get('duration_ms', 0) or 0 + artist = _track_artist(track) + name = _track_name(track) + lines.append(f'#EXTINF:{int(duration_ms / 1000)},{artist} - {name}') + file_path = file_path_map.get(idx) + if file_path: + path = f'{entry_base_path}/{file_path}' if entry_base_path else file_path + lines.append(path.replace('\\', '/')) + found += 1 + else: + lines.append(f'# MISSING: {artist} - {name}') + missing += 1 + lines.append('') + + if found == 0: + logger.info("[M3U] Skipping regeneration for %s: no library paths resolved", playlist_name) + return + + transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) + playlist_dir = _compute_m3u_folder(transfer_dir, 'playlist', playlist_name, '', '', '') + os.makedirs(playlist_dir, exist_ok=True) + m3u_path = os.path.join(playlist_dir, f'{_sanitize_filename(playlist_name)}.m3u') + + with open(m3u_path, 'w', encoding='utf-8') as f: + f.write('\n'.join(lines)) + + logger.info( + "[M3U] Regenerated M3U on batch complete: %s (%s/%s resolved, %s missing)", + m3u_path, + found, + len(tracks), + missing, + ) + except Exception as e: + logger.error("[M3U] Error in _regenerate_batch_m3u: %s", e) + + +def _sanitize_filename(filename: str) -> str: + """Sanitize a filename for filesystem use.""" + sanitized = re.sub(r'[<>:"/\\|?*]', '_', filename or '') + sanitized = re.sub(r'\s+', ' ', sanitized).strip() + sanitized = sanitized.rstrip('. ') or '_' + if re.match(r'^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)', sanitized, re.IGNORECASE): + sanitized = '_' + sanitized + return sanitized[:200] + + +def _compute_m3u_folder(transfer_dir, context_type, playlist_name, artist_name='', album_name='', year=''): + """Compute the target folder for an M3U file using the template system.""" + from core.import_paths import get_file_path_from_template + + if context_type == 'album' and artist_name and album_name: + template_context = { + 'artist': artist_name, + 'albumartist': artist_name, + 'album': album_name, + 'title': 'placeholder', + 'track_number': 1, + 'disc_number': 1, + 'year': year, + 'quality': '' + } + folder_path, _ = get_file_path_from_template(template_context, 'album_path') + if folder_path: + return os.path.join(transfer_dir, folder_path) + artist_sanitized = _sanitize_filename(artist_name) + album_sanitized = _sanitize_filename(album_name) + return os.path.join(transfer_dir, artist_sanitized, f"{artist_sanitized} - {album_sanitized}") + + template_context = { + 'artist': 'placeholder', + 'albumartist': 'placeholder', + 'album': 'placeholder', + 'title': 'placeholder', + 'playlist_name': playlist_name, + 'track_number': 1, + 'disc_number': 1, + 'year': '', + 'quality': '' + } + folder_path, _ = get_file_path_from_template(template_context, 'playlist_path') + if folder_path: + return os.path.join(transfer_dir, folder_path) + playlist_sanitized = _sanitize_filename(playlist_name) + return os.path.join(transfer_dir, playlist_sanitized) + +# --- Restored Web UI Helper State --- +session_completed_downloads = 0 +session_stats_lock = threading.Lock() + +batch_locks = {} +_processed_download_ids = set() +_orphaned_download_keys = set() + +_mb_release_cache = {} +_mb_release_cache_lock = threading.Lock() +_mb_release_detail_cache = {} +_mb_release_detail_cache_lock = threading.Lock() + +_enrichment_activity_log = {} +_idle_since = {} +_IDLE_GRACE_SECONDS = 5 + +_status_cache = { + 'spotify': {'connected': False, 'response_time': 0, 'source': 'itunes'}, + 'media_server': {'connected': False, 'response_time': 0, 'type': None}, + 'soulseek': {'connected': False, 'response_time': 0}, +} +_status_cache_timestamps = { + 'spotify': 0, + 'media_server': 0, + 'soulseek': 0, +} +STATUS_CACHE_TTL = 120 + +dev_mode_enabled = False +_hydrabase_ws = None +_hydrabase_lock = threading.Lock() + + +def get_cached_beatport_data(section_type, data_key, genre_slug=None): + current_time = time.time() + with beatport_data_cache['cache_lock']: + try: + if section_type == 'homepage': + cache_entry = beatport_data_cache['homepage'].get(data_key) + elif section_type == 'genre' and genre_slug: + cache_entry = beatport_data_cache['genre'].get(genre_slug, {}).get(data_key) + else: + return None + + if not cache_entry: + return None + + age = current_time - cache_entry['timestamp'] + if age < cache_entry['ttl'] and cache_entry['data'] is not None: + return cache_entry['data'] + return None + except Exception as e: + print(f"Cache lookup error for {section_type}/{data_key}: {e}") + return None + + +def set_cached_beatport_data(section_type, data_key, data, genre_slug=None): + current_time = time.time() + with beatport_data_cache['cache_lock']: + try: + if section_type == 'homepage': + if data_key in beatport_data_cache['homepage']: + beatport_data_cache['homepage'][data_key]['data'] = data + beatport_data_cache['homepage'][data_key]['timestamp'] = current_time + elif section_type == 'genre' and genre_slug: + if genre_slug not in beatport_data_cache['genre']: + beatport_data_cache['genre'][genre_slug] = {} + if data_key not in beatport_data_cache['genre'][genre_slug]: + beatport_data_cache['genre'][genre_slug][data_key] = { + 'data': None, + 'timestamp': 0, + 'ttl': 600, + } + beatport_data_cache['genre'][genre_slug][data_key]['data'] = data + beatport_data_cache['genre'][genre_slug][data_key]['timestamp'] = current_time + except Exception as e: + print(f"Cache storage error for {section_type}/{data_key}: {e}") + + +def add_cache_headers(response, cache_duration=300): + response.headers['Cache-Control'] = f'public, max-age={cache_duration}' + response.headers['Pragma'] = 'cache' + return response + + +def _get_max_concurrent(): + return config_manager.get('download_source.max_concurrent', 3) + + +def _get_batch_max_concurrent(is_album=False, source=None): + if is_album and source in ('soulseek', None): + if source == 'soulseek': + return 1 + mode = config_manager.get('download_source.mode', 'soulseek') + if mode == 'soulseek': + return 1 + return _get_max_concurrent() + + +def _update_task_status(task_id, new_status): + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = new_status + download_tasks[task_id]['status_change_time'] = time.time() + + +class WebUIDownloadMonitor: + def __init__(self): + self.monitoring = False + self.monitor_thread = None + self.monitored_batches = set() + self._lock = threading.Lock() + + def start_monitoring(self, batch_id): + with self._lock: + self.monitored_batches.add(batch_id) + if not self.monitoring: + self.monitoring = True + self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True) + self.monitor_thread.start() + + def stop_monitoring(self, batch_id): + with self._lock: + self.monitored_batches.discard(batch_id) + if not self.monitored_batches: + self.monitoring = False + + def shutdown(self): + with self._lock: + self.monitoring = False + self.monitored_batches.clear() + self.monitor_thread = None + + def _monitor_loop(self): + while self.monitoring and self.monitored_batches: + try: + if globals().get('IS_SHUTTING_DOWN', False): + self.monitoring = False + break + self._check_all_downloads() + time.sleep(1) + except Exception as e: + if 'interpreter shutdown' in str(e) or 'cannot schedule new futures' in str(e): + self.monitoring = False + break + print(f"Download monitor error: {e}") + + def _check_all_downloads(self): + current_time = time.time() + live_transfers_lookup = self._get_live_transfers() + exhausted_tasks = [] + completed_tasks = [] + deferred_ops = [] + + with tasks_lock: + for batch_id in list(self.monitored_batches): + if batch_id not in download_batches: + self.monitored_batches.discard(batch_id) + continue + + for task_id in download_batches[batch_id].get('queue', []): + task = download_tasks.get(task_id) + if not task or task['status'] not in ['downloading', 'queued']: + continue + + retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) + if retry_exhausted: + exhausted_tasks.append((batch_id, task_id)) + + task_filename = task.get('filename') or task.get('track_info', {}).get('filename') + task_username = task.get('username') or task.get('track_info', {}).get('username') + if task_filename and task_username: + lookup_key = _make_context_key(task_username, task_filename) + live_info = live_transfers_lookup.get(lookup_key) + if live_info: + state = live_info.get('state', '') + has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) + has_completion = ('Completed' in state or 'Succeeded' in state) + if has_completion and not has_error: + expected_size = live_info.get('size', 0) + transferred = live_info.get('bytesTransferred', 0) + if expected_size > 0 and transferred < expected_size: + if not task.get('_incomplete_warned'): + task['_incomplete_warned'] = True + continue + if has_completion and not has_error and task['status'] == 'downloading': + task.pop('_incomplete_warned', None) + task['status'] = 'post_processing' + task['status_change_time'] = current_time + completed_tasks.append((batch_id, task_id)) + + for op in deferred_ops: + try: + if op[0] == 'cancel_download': + _, download_id, username = op + run_async(soulseek_client.cancel_download(download_id, username, remove=True)) + elif op[0] == 'cleanup_orphan': + _, context_key = op + with matched_context_lock: + matched_downloads_context.pop(context_key, None) + elif op[0] == 'restart_worker': + _, task_id, batch_id = op + missing_download_executor.submit(_download_track_worker, task_id, batch_id) + except Exception as e: + print(f"[Deferred] Error executing deferred operation {op[0]}: {e}") + + for batch_id, task_id in completed_tasks: + try: + missing_download_executor.submit(_run_post_processing_worker, task_id, batch_id) + _on_download_completed(batch_id, task_id, success=True) + except Exception as e: + print(f"[Monitor] Error handling completed task {task_id}: {e}") + + for batch_id, task_id in exhausted_tasks: + try: + _on_download_completed(batch_id, task_id, success=False) + except Exception as e: + print(f"[Monitor] Error handling exhausted task {task_id}: {e}") + + self._validate_worker_counts() + + def _get_live_transfers(self): + try: + if not self.monitoring or not soulseek_client: + return {} + + live_transfers = {} + transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + if transfers_data: + for user_data in transfers_data: + username = user_data.get('username', 'Unknown') + if 'directories' in user_data: + for directory in user_data['directories']: + if 'files' in directory: + for file_info in directory['files']: + key = _make_context_key(username, file_info.get('filename', '')) + live_transfers[key] = file_info + + try: + all_downloads = run_async(soulseek_client.get_all_downloads()) + for download in all_downloads: + if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): + key = _make_context_key(download.username, download.filename) + live_transfers[key] = { + 'id': download.id, + 'filename': download.filename, + 'username': download.username, + 'state': download.state, + 'percentComplete': download.progress, + 'size': download.size, + 'bytesTransferred': download.transferred, + 'averageSpeed': download.speed, + } + except Exception as yt_error: + print(f"Monitor: Could not fetch streaming source downloads: {yt_error}") + + return live_transfers + except Exception as e: + if ('interpreter shutdown' in str(e) or 'cannot schedule new futures' in str(e) or 'Event loop is closed' in str(e)): + self.monitoring = False + return {} + print(f"Monitor: Could not fetch live transfers: {e}") + return {} + + def _should_retry_task(self, task_id, task, live_transfers_lookup, current_time, deferred_ops): + ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + task_filename = task.get('filename') or ti.get('filename') + task_username = task.get('username') or ti.get('username') + + if not task_filename or not task_username: + return False + + lookup_key = _make_context_key(task_username, task_filename) + live_info = live_transfers_lookup.get(lookup_key) + + if not live_info: + if current_time - task.get('status_change_time', current_time) > 90: + retry_count = task.get('stuck_retry_count', 0) + last_retry = task.get('last_retry_time', 0) + if retry_count < 3 and (current_time - last_retry) > 30: + task['stuck_retry_count'] = retry_count + 1 + task['last_retry_time'] = current_time + download_id = task.get('download_id') + if task_username and download_id: + deferred_ops.append(('cancel_download', download_id, task_username)) + if task_username and task_filename: + used_sources = task.get('used_sources', set()) + source_key = f"{task_username}_{task_filename}" + used_sources.add(source_key) + task['used_sources'] = used_sources + if task_username and task_filename: + _orphaned_download_keys.add(lookup_key) + deferred_ops.append(('cleanup_orphan', lookup_key)) + task.pop('download_id', None) + task.pop('username', None) + task.pop('filename', None) + task['status'] = 'searching' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task['status_change_time'] = current_time + batch_id = task.get('batch_id') + if task_id and batch_id: + deferred_ops.append(('restart_worker', task_id, batch_id)) + return False + elif retry_count < 3: + return False + else: + track_label = task.get('track_info', {}).get('name', 'Unknown') + tried_sources = task.get('used_sources', set()) + sources_str = f" (tried {len(tried_sources)} source{'s' if len(tried_sources) != 1 else ''})" if tried_sources else '' + task['status'] = 'failed' + task['error_message'] = f'Download disappeared from transfer list 3 times for "{track_label}"{sources_str} — source may be unavailable' + return bool(task.get('batch_id')) + return False + + state_str = live_info.get('state', '') + progress = live_info.get('percentComplete', 0) + + if any(token in state_str for token in ('Errored', 'Failed', 'Rejected', 'TimedOut')): + retry_count = task.get('error_retry_count', 0) + last_retry = task.get('last_error_retry_time', 0) + if retry_count < 3 and (current_time - last_retry) > 5: + task['error_retry_count'] = retry_count + 1 + task['last_error_retry_time'] = current_time + _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or _ti.get('username') + filename = task.get('filename') or _ti.get('filename') + download_id = task.get('download_id') + if username and download_id: + deferred_ops.append(('cancel_download', download_id, username)) + if username and filename: + used_sources = task.get('used_sources', set()) + used_sources.add(f"{username}_{filename}") + task['used_sources'] = used_sources + if username and filename: + old_context_key = _make_context_key(username, filename) + _orphaned_download_keys.add(old_context_key) + deferred_ops.append(('cleanup_orphan', old_context_key)) + task.pop('download_id', None) + task.pop('username', None) + task.pop('filename', None) + task['status'] = 'searching' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task['status_change_time'] = current_time + batch_id = task.get('batch_id') + if task_id and batch_id: + deferred_ops.append(('restart_worker', task_id, batch_id)) + return False + elif retry_count < 3: + return False + task['status'] = 'failed' + task['error_message'] = f'Soulseek transfer errored 3 times for "{task.get("track_info", {}).get("name", "Unknown")}"' + return bool(task.get('batch_id')) + + elif 'Queued' in state_str or task['status'] == 'queued': + if 'queued_start_time' not in task: + task['queued_start_time'] = current_time + return False + queue_time = current_time - task['queued_start_time'] + timeout_threshold = 15.0 if task.get('track_info', {}).get('is_album_download', False) else 90.0 + if queue_time > timeout_threshold: + retry_count = task.get('stuck_retry_count', 0) + last_retry = task.get('last_retry_time', 0) + if retry_count < 3 and (current_time - last_retry) > 30: + task['stuck_retry_count'] = retry_count + 1 + task['last_retry_time'] = current_time + _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or _ti.get('username') + filename = task.get('filename') or _ti.get('filename') + download_id = task.get('download_id') + if username and download_id: + deferred_ops.append(('cancel_download', download_id, username)) + if username and filename: + used_sources = task.get('used_sources', set()) + used_sources.add(f"{username}_{filename}") + task['used_sources'] = used_sources + if username and filename: + old_context_key = _make_context_key(username, filename) + _orphaned_download_keys.add(old_context_key) + deferred_ops.append(('cleanup_orphan', old_context_key)) + task.pop('download_id', None) + task.pop('username', None) + task.pop('filename', None) + task['status'] = 'searching' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task['status_change_time'] = current_time + batch_id = task.get('batch_id') + if task_id and batch_id: + deferred_ops.append(('restart_worker', task_id, batch_id)) + return False + elif retry_count < 3: + return False + task['status'] = 'failed' + task['error_message'] = f'Download stayed queued too long 3 times for "{task.get("track_info", {}).get("name", "Unknown")}"' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + return bool(task.get('batch_id')) + + elif 'InProgress' in state_str and progress < 1: + if 'downloading_start_time' not in task: + task['downloading_start_time'] = current_time + return False + download_time = current_time - task['downloading_start_time'] + timeout_threshold = 15.0 if task.get('track_info', {}).get('is_album_download', False) else 90.0 + if download_time > timeout_threshold: + retry_count = task.get('stuck_retry_count', 0) + last_retry = task.get('last_retry_time', 0) + if retry_count < 3 and (current_time - last_retry) > 30: + task['stuck_retry_count'] = retry_count + 1 + task['last_retry_time'] = current_time + _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or _ti.get('username') + filename = task.get('filename') or _ti.get('filename') + download_id = task.get('download_id') + if username and download_id: + deferred_ops.append(('cancel_download', download_id, username)) + if username and filename: + used_sources = task.get('used_sources', set()) + used_sources.add(f"{username}_{filename}") + task['used_sources'] = used_sources + if username and filename: + old_context_key = _make_context_key(username, filename) + _orphaned_download_keys.add(old_context_key) + deferred_ops.append(('cleanup_orphan', old_context_key)) + task.pop('download_id', None) + task.pop('username', None) + task.pop('filename', None) + task['status'] = 'searching' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task['status_change_time'] = current_time + batch_id = task.get('batch_id') + if task_id and batch_id: + deferred_ops.append(('restart_worker', task_id, batch_id)) + return False + elif retry_count < 3: + return False + task['status'] = 'failed' + task['error_message'] = f'Download stuck at 0% three times for "{task.get("track_info", {}).get("name", "Unknown")}"' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + return bool(task.get('batch_id')) + else: + bytes_transferred = live_info.get('bytesTransferred', 0) + if progress >= 1 or bytes_transferred > 0: + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task.pop('stuck_retry_count', None) + else: + if 'downloading_start_time' not in task: + task['downloading_start_time'] = current_time + download_time = current_time - task['downloading_start_time'] + timeout_threshold = 15.0 if task.get('track_info', {}).get('is_album_download', False) else 90.0 + if download_time > timeout_threshold: + retry_count = task.get('stuck_retry_count', 0) + last_retry = task.get('last_retry_time', 0) + if retry_count < 3 and (current_time - last_retry) > 30: + task['stuck_retry_count'] = retry_count + 1 + task['last_retry_time'] = current_time + _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} + username = task.get('username') or _ti.get('username') + filename = task.get('filename') or _ti.get('filename') + download_id = task.get('download_id') + if username and download_id: + deferred_ops.append(('cancel_download', download_id, username)) + if username and filename: + used_sources = task.get('used_sources', set()) + used_sources.add(f"{username}_{filename}") + task['used_sources'] = used_sources + if username and filename: + old_context_key = _make_context_key(username, filename) + _orphaned_download_keys.add(old_context_key) + deferred_ops.append(('cleanup_orphan', old_context_key)) + task.pop('download_id', None) + task.pop('username', None) + task.pop('filename', None) + task['status'] = 'searching' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + task['status_change_time'] = current_time + batch_id = task.get('batch_id') + if task_id and batch_id: + deferred_ops.append(('restart_worker', task_id, batch_id)) + return False + elif retry_count >= 3: + task['status'] = 'failed' + task['error_message'] = f'Download stuck in "{state_str}" state 3 times for "{task.get("track_info", {}).get("name", "Unknown")}"' + task.pop('queued_start_time', None) + task.pop('downloading_start_time', None) + return bool(task.get('batch_id')) + + return False + + def _validate_worker_counts(self): + try: + batches_needing_workers = [] + with tasks_lock: + for batch_id in list(self.monitored_batches): + if batch_id not in download_batches: + continue + batch = download_batches[batch_id] + reported_active = batch['active_count'] + max_concurrent = batch['max_concurrent'] + queue = batch.get('queue', []) + queue_index = batch.get('queue_index', 0) + actually_active = 0 + orphaned_tasks = [] + completed_task_ids = batch.get('_completed_task_ids', set()) + for task_id in queue: + if task_id in download_tasks: + task_status = download_tasks[task_id]['status'] + if task_status in ['searching', 'downloading', 'queued', 'post_processing']: + if task_id not in completed_task_ids: + actually_active += 1 + elif task_status in ['failed', 'completed', 'cancelled', 'not_found'] and task_id in queue[queue_index:]: + orphaned_tasks.append(task_id) + if reported_active != actually_active or orphaned_tasks: + if reported_active != actually_active: + batch['active_count'] = actually_active + if actually_active < max_concurrent and queue_index < len(queue): + batches_needing_workers.append(batch_id) + for batch_id in batches_needing_workers: + try: + _start_next_batch_of_downloads(batch_id) + except Exception as e: + print(f"[Worker Validation] Error starting workers for {batch_id}: {e}") + except Exception as validation_error: + print(f"Error in worker count validation: {validation_error}") + + +download_monitor = WebUIDownloadMonitor() + + +def _is_explicit_blocked(track_data): + if config_manager.get('content_filter.allow_explicit', True): + return False + if track_data.get('explicit', False): + return True + sp_data = track_data.get('spotify_data', {}) + if isinstance(sp_data, str): + try: + sp_data = json.loads(sp_data) + except Exception: + sp_data = {} + return sp_data.get('explicit', False) + + +def fix_artist_image_url(thumb_url): + if not thumb_url: + return None + + try: + needs_fixing = ( + thumb_url.startswith('http://localhost:') or + thumb_url.startswith('https://localhost:') or + thumb_url.startswith('/library/') or + thumb_url.startswith('/Items/') or + thumb_url.startswith('/api/') or + thumb_url.startswith('/rest/') + ) + + if needs_fixing: + active_server = config_manager.get_active_media_server() + if active_server == 'plex': + plex_config = config_manager.get_plex_config() + plex_base_url = plex_config.get('base_url', '') + plex_token = plex_config.get('token', '') + if plex_base_url and plex_token: + if thumb_url.startswith('/library/'): + path = thumb_url + else: + from urllib.parse import urlparse + path = urlparse(thumb_url).path + return f"{plex_base_url.rstrip('/')}{path}?X-Plex-Token={plex_token}" + + elif active_server == 'jellyfin': + jellyfin_config = config_manager.get_jellyfin_config() + jellyfin_base_url = jellyfin_config.get('base_url', '') + jellyfin_token = jellyfin_config.get('api_key', '') + if jellyfin_base_url: + if thumb_url.startswith('/Items/') or thumb_url.startswith('/api/'): + path = thumb_url + else: + from urllib.parse import urlparse + path = urlparse(thumb_url).path + if jellyfin_token: + separator = '&' if '?' in path else '?' + return f"{jellyfin_base_url.rstrip('/')}{path}{separator}X-Emby-Token={jellyfin_token}" + return f"{jellyfin_base_url.rstrip('/')}{path}" + + elif active_server == 'navidrome': + navidrome_config = config_manager.get_navidrome_config() + navidrome_base_url = navidrome_config.get('base_url', '') + navidrome_username = navidrome_config.get('username', '') + navidrome_password = navidrome_config.get('password', '') + if navidrome_base_url and navidrome_username and navidrome_password: + if thumb_url.startswith('/rest/'): + path = thumb_url + else: + from urllib.parse import urlparse + path = urlparse(thumb_url).path + import hashlib + import secrets + salt = secrets.token_hex(6) + token = hashlib.md5((navidrome_password + salt).encode()).hexdigest() + separator = '&' if '?' in path else '?' + auth_params = f"u={navidrome_username}&t={token}&s={salt}&v=1.16.1&c=SoulSync&f=json" + return f"{navidrome_base_url.rstrip('/')}{path}{separator}{auth_params}" + + return thumb_url + except Exception as e: + print(f"Error fixing image URL '{thumb_url}': {e}") + return thumb_url + + +def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): + import os + import re + from difflib import SequenceMatcher + from unidecode import unidecode + + if '||' in api_filename: + _, title = api_filename.split('||', 1) + api_filename = title + + def normalize_for_finding(text: str) -> str: + if not text: + return "" + text = unidecode(text).lower() + text = re.sub(r'[._/]', ' ', text) + text = re.sub(r'[\[\(].*?[\]\)]', '', text) + text = re.sub(r'[^a-z0-9\s-]', '', text) + return ' '.join(text.split()).strip() + + def _path_matches_api_dirs(file_path): + path_parts = set(p.lower() for p in file_path.replace('\\', '/').split('/')) + return all(d in path_parts for d in api_dir_parts) + + def search_in_directory(search_dir, location_name): + best_fuzzy_path = None + highest_fuzzy_similarity = 0.0 + exact_matches = [] + for root, dirs, files in os.walk(search_dir): + dirs[:] = [d for d in dirs if d != 'ss_quarantine'] + for file in files: + if os.path.basename(file) == target_basename: + file_path = os.path.join(root, file) + if api_dir_parts and _path_matches_api_dirs(file_path): + return file_path, 1.0 + if not api_dir_parts: + return file_path, 1.0 + exact_matches.append(file_path) + continue + + file_stem, file_ext_part = os.path.splitext(file) + stripped_stem = re.sub(r'_\d{10,}$', '', file_stem) + if stripped_stem != file_stem and stripped_stem + file_ext_part == target_basename: + file_path = os.path.join(root, file) + if api_dir_parts and _path_matches_api_dirs(file_path): + return file_path, 1.0 + if not api_dir_parts: + return file_path, 1.0 + exact_matches.append(file_path) + continue + + normalized_file = normalize_for_finding(file) + similarity = SequenceMatcher(None, normalized_target, normalized_file).ratio() + if similarity > highest_fuzzy_similarity: + highest_fuzzy_similarity = similarity + best_fuzzy_path = os.path.join(root, file) + + if exact_matches: + if len(exact_matches) == 1: + return exact_matches[0], 1.0 + best = exact_matches[0] + best_score = -1 + for m in exact_matches: + m_parts = set(p.lower() for p in m.replace('\\', '/').split('/')) + score = sum(1 for d in api_dir_parts if d in m_parts) + if score > best_score: + best_score = score + best = m + return best, 1.0 + + return best_fuzzy_path, highest_fuzzy_similarity + + target_basename = extract_filename(api_filename) + normalized_target = normalize_for_finding(target_basename) + api_path_normalized = api_filename.replace('\\', '/') if api_filename else '' + api_dir_parts = [p.lower() for p in api_path_normalized.split('/')[:-1] if p] + + best_downloads_path, downloads_similarity = search_in_directory(download_dir, 'downloads') + if downloads_similarity > 0.85: + return (best_downloads_path, 'downloads') + + transfer_similarity = 0.0 + if transfer_dir and os.path.exists(transfer_dir): + best_transfer_path, transfer_similarity = search_in_directory(transfer_dir, 'transfer') + if transfer_similarity > 0.85: + return (best_transfer_path, 'transfer') + + return (None, None) + + +def _cleanup_empty_directories(download_path, moved_file_path): + try: + current_dir = os.path.dirname(moved_file_path) + while current_dir != download_path and current_dir.startswith(download_path): + is_empty = not any(not f.startswith('.') for f in os.listdir(current_dir)) + if is_empty: + os.rmdir(current_dir) + current_dir = os.path.dirname(current_dir) + else: + break + except Exception as e: + print(f"Warning: An error occurred during directory cleanup: {e}") + + +def _sweep_empty_download_directories(): + try: + download_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + if not os.path.isdir(download_path): + return 0 + + removed = 0 + for dirpath, _dirnames, _filenames in os.walk(download_path, topdown=False): + if os.path.normpath(dirpath) == os.path.normpath(download_path): + continue + try: + entries = os.listdir(dirpath) + except OSError: + continue + visible = [e for e in entries if not e.startswith('.')] + if not visible: + try: + for hidden in entries: + try: + os.remove(os.path.join(dirpath, hidden)) + except Exception: + pass + os.rmdir(dirpath) + removed += 1 + except OSError: + pass + + return removed + except Exception as e: + print(f"[Folder Cleanup] Error sweeping empty directories: {e}") + return 0 + + +def _get_audio_quality_string(file_path): + try: + ext = os.path.splitext(file_path)[1].lower() + if ext == '.flac': + audio = FLAC(file_path) + return f"FLAC {audio.info.bits_per_sample}bit" + if ext == '.mp3': + from mutagen.mp3 import MP3, BitrateMode + audio = MP3(file_path) + bitrate_kbps = audio.info.bitrate // 1000 + if audio.info.bitrate_mode == BitrateMode.VBR: + return "MP3-VBR" + return f"MP3-{bitrate_kbps}" + if ext in ('.m4a', '.aac', '.mp4'): + audio = MP4(file_path) + return f"M4A-{audio.info.bitrate // 1000}" + if ext == '.ogg': + audio = OggVorbis(file_path) + return f"OGG-{audio.info.bitrate // 1000}" + if ext == '.opus': + from mutagen.oggopus import OggOpus + audio = OggOpus(file_path) + return f"OPUS-{audio.info.bitrate // 1000}" + return '' + except Exception as e: + logger.debug(f"Could not determine audio quality for {file_path}: {e}") + return '' + + +def parse_youtube_playlist(url): + try: + import re + + def clean_youtube_artist(artist_string): + if not artist_string: + return artist_string + original_artist = artist_string + artist_string = artist_string.replace('"', '').replace("'", '') + artist_string = re.sub(r'\s*\([^)]*\)', '', artist_string).strip() + artist_string = re.sub(r'\s*\[[^\]]*\]', '', artist_string).strip() + for suffix in [ + r'\s*-\s*Topic\s*$', + r'\s*VEVO\s*$', + r'\s*Music\s*$', + r'\s*Official\s*$', + r'\s*Records\s*$', + r'\s*Entertainment\s*$', + r'\s*TV\s*$', + r'\s*Channel\s*$', + ]: + artist_string = re.sub(suffix, '', artist_string, flags=re.IGNORECASE).strip() + for sep in [',', '&', ' and ', ' x ', ' X ', ' feat.', ' ft.', ' featuring', ' with', ' vs ', ' vs.']: + if sep in artist_string: + artist_string = artist_string.split(sep)[0].strip() + break + artist_string = re.sub(r'\s+', ' ', artist_string).strip() + artist_string = re.sub(r'^\-\s*|\s*\-$', '', artist_string).strip() + artist_string = re.sub(r'^,\s*|\s*,$', '', artist_string).strip() + return artist_string or original_artist + + def clean_youtube_track_title(title, artist_name=None): + if not title: + return title + original_title = title + artist_removed = False + if artist_name and '-' in title: + artist_pattern = r'^' + re.escape(artist_name.strip()) + r'(?:\s*[&,x]\s*[^-]+)?\s*[-–—]\s*' + cleaned_title = re.sub(artist_pattern, '', title, flags=re.IGNORECASE).strip() + if cleaned_title != title: + title = cleaned_title + artist_removed = True + else: + artist_end_pattern = r'\s*[-–—]\s*' + re.escape(artist_name.strip()) + r'(?:\s*[&,x]\s*[^-]+)?\s*$' + cleaned_title = re.sub(artist_end_pattern, '', title, flags=re.IGNORECASE).strip() + if cleaned_title != title: + title = cleaned_title + artist_removed = True + title = re.sub(r'【[^】]*】', '', title) + title = re.sub(r'\s*\([^)]*\)', '', title) + title = re.sub(r'\s*\(.*$', '', title) + title = re.sub(r'\[[^\]]*\]', '', title) + title = re.sub(r'\{[^}]*\}', '', title) + title = re.sub(r'<[^>]*>', '', title) + if artist_removed: + title = re.sub(r'\s*-\s*.*$', '', title) + title = re.split(r'\s*\|\s*', title)[0].strip() + for pattern in [ + r'\bapple\s+music\b', + r'\bfull\s+video\b', + r'\bmusic\s+video\b', + r'\bofficial\s+video\b', + r'\bofficial\s+music\s+video\b', + r'\bofficial\b', + r'\bcensored\s+version\b', + r'\buncensored\s+version\b', + r'\bexplicit\s+version\b', + r'\blive\s+version\b', + r'\bversion\b', + r'\btopic\b', + r'\baudio\b', + r'\blyrics?\b', + r'\blyric\s+video\b', + r'\bwith\s+lyrics?\b', + r'\bvisuali[sz]er\b', + r'\bmv\b', + r'\bdirectors?\s+cut\b', + r'\bremaster(ed)?\b', + r'\bremix\b', + ]: + title = re.sub(pattern, '', title, flags=re.IGNORECASE) + if artist_name: + collab_pattern = rf'\b{re.escape(artist_name)}\s*[&,]\s*\w+|[\w\s]+[&,]\s*{re.escape(artist_name)}\b' + if not re.search(collab_pattern, title, flags=re.IGNORECASE): + title = re.sub(rf'\b{re.escape(artist_name)}\b', '', title, flags=re.IGNORECASE) + title = re.sub(rf'\b{re.escape(artist_name)}\s*[-–—:]\s*', '', title, flags=re.IGNORECASE) + title = re.sub(rf'^{re.escape(artist_name)}\s*[-–—:]\s*', '', title, flags=re.IGNORECASE) + title = re.sub(r'\s+prod\.?\s+\S+', '', title, flags=re.IGNORECASE) + title = re.sub(r'["\']', '', title) + for pattern in [ + r'\s+feat\.?\s+.+$', + r'\s+ft\.?\s+.+$', + r'\s+featuring\s+.+$', + r'\s+with\s+.+$', + ]: + title = re.sub(pattern, '', title, flags=re.IGNORECASE).strip() + title = re.sub(r'\s+', ' ', title).strip() + title = re.sub(r'^[-–—:,.\s]+|[-–—:,.\s]+$', '', title).strip() + return title or original_title + + ydl_opts = { + 'quiet': True, + 'no_warnings': True, + 'extract_flat': 'in_playlist', + 'skip_download': True, + 'lazy_playlist': False, + } + + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + playlist_info = ydl.extract_info(url, download=False) + if not playlist_info: + return None + + playlist_name = playlist_info.get('title', 'Unknown Playlist') + playlist_id = playlist_info.get('id', 'unknown_id') + entries = list(playlist_info.get('entries', []) or []) + tracks = [] + + for entry in entries: + if not entry: + continue + raw_title = entry.get('title', 'Unknown Track') + raw_uploader = entry.get('uploader', 'Unknown Artist') + duration = entry.get('duration', 0) + video_id = entry.get('id', '') + cleaned_artist = clean_youtube_artist(raw_uploader) + cleaned_title = clean_youtube_track_title(raw_title, cleaned_artist) + tracks.append({ + 'id': video_id, + 'name': cleaned_title, + 'artists': [cleaned_artist], + 'duration_ms': duration * 1000 if duration else 0, + 'raw_title': raw_title, + 'raw_artist': raw_uploader, + 'url': f"https://www.youtube.com/watch?v={video_id}", + }) + + return { + 'id': playlist_id, + 'name': playlist_name, + 'tracks': tracks, + 'track_count': len(tracks), + 'url': url, + 'source': 'youtube', + } + except Exception as e: + print(f"Error parsing YouTube playlist: {e}") + return None + + +def get_download_status(): + if not soulseek_client: + return jsonify({"transfers": []}) + + try: + global _processed_download_ids + transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + all_transfers = [] + completed_matched_downloads = [] + _files_claimed_this_cycle = set() + + if transfers_data: + for user_data in transfers_data: + username = user_data.get('username', 'Unknown') + if 'directories' in user_data: + for directory in user_data['directories']: + if 'files' in directory: + for file_info in directory['files']: + file_info['username'] = username + all_transfers.append(file_info) + state = file_info.get('state', '').lower() + if ('succeeded' in state or 'completed' in state) and 'errored' not in state and 'rejected' not in state: + _fi_size = file_info.get('size', 0) + _fi_transferred = file_info.get('bytesTransferred', 0) + if _fi_size > 0 and _fi_transferred < _fi_size: + continue + filename_from_api = file_info.get('filename') + if not filename_from_api: + continue + + context_key = _make_context_key(username, filename_from_api) + if context_key in _orphaned_download_keys: + with matched_context_lock: + has_active_context = context_key in matched_downloads_context + if has_active_context: + _orphaned_download_keys.discard(context_key) + else: + download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + found_result = _find_completed_file_robust(download_dir, filename_from_api) + found_path = found_result[0] if found_result and found_result[0] else None + orphan_cleaned = False + if found_path: + try: + os.remove(found_path) + orphan_cleaned = True + except Exception as e: + print(f"Failed to delete orphaned file (will retry next poll): {e}") + else: + orphan_cleaned = True + if orphan_cleaned: + transfer_id = file_info.get('id') + if transfer_id: + try: + run_async(soulseek_client.cancel_download(str(transfer_id), username, remove=True)) + except Exception: + pass + _orphaned_download_keys.discard(context_key) + continue + + if context_key in _processed_download_ids: + continue + + with matched_context_lock: + context = matched_downloads_context.get(context_key) + available_keys = list(matched_downloads_context.keys())[:5] if not context else None + + if context and context_key not in _stale_transfer_keys: + pass + + if context: + download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + found_result = _find_completed_file_robust(download_dir, filename_from_api) + found_path = found_result[0] if found_result and found_result[0] else None + + if found_path: + _norm_path = os.path.normpath(found_path) + if _norm_path not in _files_claimed_this_cycle: + _files_claimed_this_cycle.add(_norm_path) + completed_matched_downloads.append((context_key, context, found_path)) + with _download_retry_lock: + if context_key in _download_retry_attempts: + del _download_retry_attempts[context_key] + else: + with _download_retry_lock: + if context_key not in _download_retry_attempts: + _download_retry_attempts[context_key] = {'count': 1, 'first_attempt': time.time()} + else: + _download_retry_attempts[context_key]['count'] += 1 + retry_count = _download_retry_attempts[context_key]['count'] + if retry_count >= _download_retry_max: + _processed_download_ids.add(context_key) + del _download_retry_attempts[context_key] + if completed_matched_downloads: + def process_completed_downloads(): + for context_key, context, found_path in completed_matched_downloads: + try: + _pp_task_id = context.get('task_id') + _pp_batch_id = context.get('batch_id') + if _pp_task_id and _pp_batch_id: + _pp_target = _post_process_matched_download_with_verification + _pp_args = (context_key, context, found_path, _pp_task_id, _pp_batch_id) + else: + _pp_target = _post_process_matched_download + _pp_args = (context_key, context, found_path) + thread = threading.Thread(target=_pp_target, args=_pp_args) + thread.daemon = True + thread.start() + _processed_download_ids.add(context_key) + except Exception as e: + print(f"Error starting post-processing thread for {context_key}: {e}") + processing_thread = threading.Thread(target=process_completed_downloads) + processing_thread.daemon = True + processing_thread.start() + + return jsonify({"transfers": all_transfers}) + except Exception as e: + logger.error(f"Error building download status: {e}") + return jsonify({"error": str(e)}), 500 + + +def _get_windowed_calls(key, current_total): + now = time.time() + history = _enrichment_activity_log.setdefault(key, collections.deque(maxlen=17300)) + history.append((now, current_total)) + + cutoff_1h = now - 3600 + cutoff_24h = now - 86400 + oldest_1h_total = current_total + oldest_24h_total = current_total + found_24h = False + for ts, total in history: + if not found_24h and ts >= cutoff_24h: + oldest_24h_total = total + found_24h = True + if ts >= cutoff_1h: + oldest_1h_total = total + break + + return max(0, current_total - oldest_1h_total), max(0, current_total - oldest_24h_total) + + +def _get_enrichment_status(): + services = {} + workers_info = [ + ('musicbrainz', 'MusicBrainz', lambda: mb_worker), + ('spotify_enrichment', 'Spotify', lambda: spotify_enrichment_worker), + ('itunes_enrichment', 'iTunes', lambda: itunes_enrichment_worker), + ('deezer_enrichment', 'Deezer', lambda: deezer_worker), + ('tidal_enrichment', 'Tidal', lambda: tidal_enrichment_worker), + ('qobuz_enrichment', 'Qobuz', lambda: qobuz_enrichment_worker), + ('lastfm', 'Last.fm', lambda: lastfm_worker), + ('genius', 'Genius', lambda: genius_worker), + ('audiodb', 'AudioDB', lambda: audiodb_worker), + ('discogs', 'Discogs', lambda: discogs_worker), + ] + configured_checks = { + 'spotify_enrichment': lambda: bool(config_manager.get('spotify.client_id') and config_manager.get('spotify.client_secret')), + 'tidal_enrichment': lambda: bool(tidal_client and getattr(tidal_client, 'access_token', None)), + 'qobuz_enrichment': lambda: bool(qobuz_enrichment_worker and qobuz_enrichment_worker.client and qobuz_enrichment_worker.client.user_auth_token), + 'lastfm': lambda: bool(config_manager.get('lastfm.api_key', '')), + 'genius': lambda: bool(config_manager.get('genius.access_token', '')), + } + + for key, name, get_worker in workers_info: + worker = get_worker() + if worker is not None: + is_alive = worker.thread is not None and worker.thread.is_alive() + try: + configured = configured_checks.get(key, lambda: True)() + except Exception: + configured = False + + stats = worker.stats + total_processed = stats.get('matched', 0) + stats.get('not_found', 0) + stats.get('errors', 0) + calls_1h, calls_24h = _get_windowed_calls(key, total_processed) + + has_item = getattr(worker, 'current_item', None) is not None + if has_item: + _idle_since.pop(key, None) + is_idle = False + else: + if key not in _idle_since: + _idle_since[key] = time.time() + is_idle = (time.time() - _idle_since[key]) >= _IDLE_GRACE_SECONDS + + svc_data = { + 'name': name, + 'configured': configured, + 'running': worker.running and is_alive and not worker.paused, + 'paused': worker.paused, + 'idle': is_alive and not worker.paused and is_idle, + 'calls_1h': calls_1h, + 'calls_24h': calls_24h, + } + if key == 'spotify_enrichment': + try: + svc_data['daily_budget'] = worker._get_daily_budget_info() + except Exception: + pass + services[key] = svc_data + else: + services[key] = { + 'name': name, + 'configured': False, + 'running': False, + 'paused': False, + 'idle': False, + 'calls_1h': 0, + 'calls_24h': 0, + } + + services['acoustid'] = { + 'name': 'AcoustID', + 'configured': bool(config_manager.get('acoustid.api_key', '')), + } + services['listenbrainz'] = { + 'name': 'ListenBrainz', + 'configured': bool(config_manager.get('listenbrainz.token', '')), + } + return services + + +def _build_system_stats(): + try: + import psutil + except Exception: + psutil = None + + from datetime import timedelta + + start_time = getattr(app, 'start_time', time.time()) + uptime = str(timedelta(seconds=int(time.time() - start_time))) + memory_usage = '0%' + if psutil is not None: + memory_usage = f"{psutil.virtual_memory().percent}%" + + active_downloads = len([batch_id for batch_id, batch_data in download_batches.items() if batch_data.get('phase') == 'downloading']) + with session_stats_lock: + finished_downloads = session_completed_downloads + + total_download_speed = 0.0 + try: + transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) + if transfers_data: + for user_data in transfers_data: + if 'directories' in user_data: + for directory in user_data['directories']: + if 'files' in directory: + for file_info in directory['files']: + state = file_info.get('state', '').lower() + if 'inprogress' in state or 'downloading' in state or 'transferring' in state: + speed = file_info.get('averageSpeed', 0) + if isinstance(speed, (int, float)) and speed > 0: + total_download_speed += float(speed) + except Exception: + pass + + return { + 'uptime': uptime, + 'memory_usage': memory_usage, + 'active_downloads': active_downloads, + 'finished_downloads': finished_downloads, + 'download_speed': f"{total_download_speed / (1024 * 1024):.1f} MB/s" if total_download_speed > 1024 * 1024 else f"{total_download_speed / 1024:.1f} KB/s", + } + + +def _is_hydrabase_active(): + try: + if hydrabase_client is None or not hydrabase_client.is_connected(): + return False + return dev_mode_enabled + except (NameError, Exception): + return False + # --- Automation Engine --- try: automation_engine = AutomationEngine(get_database()) @@ -658,6 +2217,48 @@ except Exception as e: logger.error(f"Automation engine failed to initialize: {e}") automation_engine = None +# --- Automation Progress Tracking --- +automation_progress_states = {} # automation_id (int) -> state dict +automation_progress_lock = threading.Lock() +_scan_library_automation_id = None + +def _init_automation_progress(automation_id, automation_name, action_type): + """Initialize progress state when an automation starts running.""" + with automation_progress_lock: + automation_progress_states[automation_id] = { + 'status': 'running', + 'action_type': action_type, + 'progress': 0, 'phase': 'Starting...', 'current_item': '', + 'processed': 0, 'total': 0, + 'log': [{'type': 'info', 'text': f'Starting {automation_name}'}], + 'started_at': datetime.now().isoformat(), + 'finished_at': None, + } + + +def _update_automation_progress(automation_id, **kwargs): + """Update progress state from handler threads. Thread-safe.""" + if automation_id is None: + return + with automation_progress_lock: + state = automation_progress_states.get(automation_id) + if not state: + return + for k, v in kwargs.items(): + if k == 'log_line': + state['log'].append({'type': kwargs.get('log_type', 'info'), 'text': v}) + if len(state['log']) > 50: + state['log'] = state['log'][-50:] + elif k != 'log_type': + state[k] = v + # Immediate emit on finish so frontend gets final state without waiting for loop + if kwargs.get('status') in ('finished', 'error'): + state['finished_at'] = datetime.now().isoformat() + try: + socketio.emit('automation:progress', {str(automation_id): dict(state)}) + except Exception: + pass + def _register_automation_handlers(): """Register real SoulSync action handlers with the automation engine.""" if not automation_engine: @@ -18089,31 +19690,6 @@ def _wipe_source_tags(file_path: str) -> bool: def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: return metadata_enrichment.enhance_file_metadata(file_path, context, artist, album_info) -def _get_image_dimensions(data: bytes): - """Extract width/height from JPEG or PNG image data without PIL.""" - try: - if data[:8] == b'\x89PNG\r\n\x1a\n': - # PNG: width and height at bytes 16-23 - import struct - w, h = struct.unpack('>II', data[16:24]) - return w, h - if data[:2] == b'\xff\xd8': - # JPEG: scan for SOF0/SOF2 marker - import struct - i = 2 - while i < len(data) - 9: - if data[i] != 0xFF: - break - marker = data[i + 1] - if marker in (0xC0, 0xC2): - h, w = struct.unpack('>HH', data[i + 5:i + 9]) - return w, h - length = struct.unpack('>H', data[i + 2:i + 4])[0] - i += 2 + length - except Exception: - pass - return None, None - def _download_cover_art(album_info: dict, target_dir: str, context: dict = None): return metadata_enrichment.download_cover_art( @@ -18323,8 +19899,34 @@ def _post_process_matched_download(context_key, context, file_path): from core.import_pipeline import post_process_matched_download return post_process_matched_download(context_key, context, file_path, _build_import_pipeline_runtime()) -# Keep track of processed downloads to avoid re-processing -_processed_download_ids = set() +def _build_import_pipeline_runtime(): + """Collect the live controller dependencies needed by core.import_pipeline.""" + return types.SimpleNamespace( + automation_engine=automation_engine, + on_download_completed=_on_download_completed, + web_scan_manager=web_scan_manager, + repair_worker=repair_worker, + ) + +def _wipe_source_tags(file_path: str) -> bool: + return metadata_enrichment.wipe_source_tags(file_path) + + +def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: + return metadata_enrichment.enhance_file_metadata( + file_path, + context, + artist, + album_info, + ) + + +def _download_cover_art(album_info: dict, target_dir: str, context: dict = None): + return metadata_enrichment.download_cover_art( + album_info, + target_dir, + context, + ) # Track stale transfer keys (completed in slskd but no context — e.g., from before app restart) # so we only log the warning once per key instead of spamming every poll cycle From b9269b4f16c14c2115ae21acdb84473bec5224fe Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Fri, 24 Apr 2026 21:52:06 +0300 Subject: [PATCH 04/19] Tighten metadata helper boundaries - remove stale wrapper helpers from web_server and metadata_common - import provider helpers directly in metadata_source - keep the metadata modules' public surface explicit --- core/import_pipeline.py | 10 ++-- core/metadata_artwork.py | 25 ++++++---- core/metadata_common.py | 78 +++++++++++++------------------ core/metadata_enrichment.py | 73 ++++++++++------------------- core/metadata_lyrics.py | 10 ++-- core/metadata_source.py | 59 ++++++++++++----------- tests/test_metadata_enrichment.py | 30 ++++++------ 7 files changed, 131 insertions(+), 154 deletions(-) diff --git a/core/import_pipeline.py b/core/import_pipeline.py index 1574fe69..62bc1f74 100644 --- a/core/import_pipeline.py +++ b/core/import_pipeline.py @@ -51,18 +51,16 @@ from core.import_runtime_state import ( _processed_download_ids, tasks_lock, ) -from core.metadata_enrichment import ( - download_cover_art, - enhance_file_metadata, - generate_lrc_file, - wipe_source_tags, -) +from core.metadata_artwork import download_cover_art +from core.metadata_common import wipe_source_tags +from core.metadata_enrichment import enhance_file_metadata from core.import_paths import ( build_final_path_for_track, build_simple_download_destination, docker_resolve_path, ) from core.import_album_naming import resolve_album_group +from core.metadata_lyrics import generate_lrc_file from database.music_database import get_database from utils.logging_config import get_logger diff --git a/core/metadata_artwork.py b/core/metadata_artwork.py index 4c83f325..b84f5274 100644 --- a/core/metadata_artwork.py +++ b/core/metadata_artwork.py @@ -8,17 +8,22 @@ import urllib.request from core.import_context import get_import_context_album from core.metadata_common import ( - _get_config_manager, - _get_image_dimensions, - _get_logger, - _get_mutagen_symbols, + get_config_manager, + get_image_dimensions, + get_logger, + get_mutagen_symbols, ) +__all__ = [ + "embed_album_art_metadata", + "download_cover_art", +] + def embed_album_art_metadata(audio_file, metadata: dict): - cfg = _get_config_manager() - logger_ = _get_logger() - symbols = _get_mutagen_symbols() + cfg = get_config_manager() + logger_ = get_logger() + symbols = get_mutagen_symbols() if not symbols: return @@ -59,7 +64,7 @@ def embed_album_art_metadata(audio_file, metadata: dict): picture.data = image_data picture.type = 3 picture.mime = mime_type - width, height = _get_image_dimensions(image_data) + width, height = get_image_dimensions(image_data) picture.width = width or 640 picture.height = height or 640 picture.depth = 24 @@ -74,8 +79,8 @@ def embed_album_art_metadata(audio_file, metadata: dict): def download_cover_art(album_info: dict, target_dir: str, context: dict = None): - cfg = _get_config_manager() - logger_ = _get_logger() + cfg = get_config_manager() + logger_ = get_logger() if cfg.get("metadata_enhancement.cover_art_download", True) is False: return diff --git a/core/metadata_common.py b/core/metadata_common.py index 975a8e36..5e27bc16 100644 --- a/core/metadata_common.py +++ b/core/metadata_common.py @@ -7,10 +7,24 @@ import threading from types import SimpleNamespace from typing import Any, Dict -from utils.logging_config import get_logger +from utils.logging_config import get_logger as _create_logger -logger = get_logger("metadata_enrichment") +logger = _create_logger("metadata_common") + +__all__ = [ + "get_logger", + "get_config_manager", + "get_mutagen_symbols", + "get_file_lock", + "is_ogg_opus", + "is_vorbis_like", + "save_audio_file", + "get_image_dimensions", + "strip_all_non_audio_tags", + "verify_metadata_written", + "wipe_source_tags", +] _FILE_LOCKS: Dict[str, threading.Lock] = {} _FILE_LOCKS_LOCK = threading.Lock() @@ -21,11 +35,11 @@ class _NullConfigManager: return default -def _get_logger(): +def get_logger(): return logger -def _get_config_manager(): +def get_config_manager(): try: from config.settings import config_manager as settings_config_manager @@ -34,33 +48,7 @@ def _get_config_manager(): return _NullConfigManager() -def _get_database(): - try: - from database.music_database import get_database - - return get_database() - except Exception: - return None - - -def _get_itunes_client(): - try: - from core.metadata_service import get_itunes_client - - return get_itunes_client() - except Exception: - return None - - -def _extract_artist_name(artist: Any) -> str: - if isinstance(artist, dict): - return str(artist.get("name", "") or "") - if hasattr(artist, "name"): - return str(getattr(artist, "name") or "") - return str(artist) if artist else "" - - -def _get_mutagen_symbols(): +def get_mutagen_symbols(): """Lazy mutagen import so tests can monkeypatch this without the package installed.""" try: from mutagen import File as MutagenFile @@ -128,7 +116,7 @@ def _get_mutagen_symbols(): ) -def _get_file_lock(file_path: str) -> threading.Lock: +def get_file_lock(file_path: str) -> threading.Lock: with _FILE_LOCKS_LOCK: lock = _FILE_LOCKS.get(file_path) if lock is None: @@ -137,21 +125,21 @@ def _get_file_lock(file_path: str) -> threading.Lock: return lock -def _is_ogg_opus(audio_file: Any) -> bool: +def is_ogg_opus(audio_file: Any) -> bool: return type(audio_file).__name__ == "OggOpus" -def _is_vorbis_like(audio_file: Any, symbols: Any) -> bool: +def is_vorbis_like(audio_file: Any, symbols: Any) -> bool: vorbis_classes = tuple( cls for cls in ( getattr(symbols, "FLAC", None), getattr(symbols, "OggVorbis", None), ) if cls is not None ) - return bool(vorbis_classes) and isinstance(audio_file, vorbis_classes) or _is_ogg_opus(audio_file) + return bool(vorbis_classes) and isinstance(audio_file, vorbis_classes) or is_ogg_opus(audio_file) -def _save_audio_file(audio_file: Any, symbols: Any) -> None: +def save_audio_file(audio_file: Any, symbols: Any) -> None: if isinstance(audio_file.tags, symbols.ID3): audio_file.save(v1=0, v2_version=4) elif isinstance(audio_file, symbols.FLAC): @@ -160,7 +148,7 @@ def _save_audio_file(audio_file: Any, symbols: Any) -> None: audio_file.save() -def _get_image_dimensions(data: bytes): +def get_image_dimensions(data: bytes): try: if data[:8] == b"\x89PNG\r\n\x1a\n": import struct @@ -185,12 +173,12 @@ def _get_image_dimensions(data: bytes): return None, None -def _strip_all_non_audio_tags(file_path: str) -> dict: +def strip_all_non_audio_tags(file_path: str) -> dict: summary = {"apev2_stripped": False, "apev2_tag_count": 0} if os.path.splitext(file_path)[1].lower() != ".mp3": return summary - symbols = _get_mutagen_symbols() + symbols = get_mutagen_symbols() if not symbols: return summary @@ -209,8 +197,8 @@ def _strip_all_non_audio_tags(file_path: str) -> dict: return summary -def _verify_metadata_written(file_path: str) -> bool: - symbols = _get_mutagen_symbols() +def verify_metadata_written(file_path: str) -> bool: + symbols = get_mutagen_symbols() if not symbols: return False @@ -231,7 +219,7 @@ def _verify_metadata_written(file_path: str) -> bool: return False except symbols.APENoHeaderError: pass - elif _is_vorbis_like(check, symbols): + elif is_vorbis_like(check, symbols): title_found = bool(check.get("title")) artist_found = bool(check.get("artist")) elif isinstance(check, symbols.MP4): @@ -251,8 +239,8 @@ def _verify_metadata_written(file_path: str) -> bool: def wipe_source_tags(file_path: str) -> bool: try: - _strip_all_non_audio_tags(file_path) - symbols = _get_mutagen_symbols() + strip_all_non_audio_tags(file_path) + symbols = get_mutagen_symbols() if not symbols: return False @@ -267,7 +255,7 @@ def wipe_source_tags(file_path: str) -> bool: else: audio.add_tags() tag_count = 0 - _save_audio_file(audio, symbols) + save_audio_file(audio, symbols) if tag_count > 0: logger.info("[Tag Wipe] Stripped %s source tags from: %s", tag_count, os.path.basename(file_path)) return True diff --git a/core/metadata_enrichment.py b/core/metadata_enrichment.py index 89651f1d..fe3fd906 100644 --- a/core/metadata_enrichment.py +++ b/core/metadata_enrichment.py @@ -4,53 +4,30 @@ from __future__ import annotations import os -from core.metadata_artwork import ( - download_cover_art as _download_cover_art_impl, - embed_album_art_metadata, -) +from core.metadata_artwork import embed_album_art_metadata from core.metadata_common import ( - _get_config_manager, - _get_file_lock, - _get_image_dimensions as _common_get_image_dimensions, - _get_logger, - _get_mutagen_symbols, - _is_ogg_opus as _common_is_ogg_opus, - _is_vorbis_like, - _save_audio_file, - _strip_all_non_audio_tags, - _verify_metadata_written, - wipe_source_tags as _common_wipe_source_tags, + get_config_manager, + get_file_lock, + get_logger, + get_mutagen_symbols, + is_vorbis_like, + save_audio_file, + strip_all_non_audio_tags, + verify_metadata_written, ) -from core.metadata_lyrics import generate_lrc_file as _generate_lrc_file_impl from core.metadata_source import embed_source_ids, extract_source_metadata -logger = _get_logger() - - -def _get_image_dimensions(data: bytes): - return _common_get_image_dimensions(data) - - -def _is_ogg_opus(audio_file): - return _common_is_ogg_opus(audio_file) - - -def wipe_source_tags(file_path: str) -> bool: - return _common_wipe_source_tags(file_path) - - -def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: - return _generate_lrc_file_impl(file_path, context, artist, album_info) - - -def download_cover_art(album_info: dict, target_dir: str, context: dict = None): - return _download_cover_art_impl(album_info, target_dir, context) +__all__ = [ + "enhance_file_metadata", + "extract_source_metadata", + "embed_source_ids", +] def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: - cfg = _get_config_manager() - logger_ = _get_logger() + cfg = get_config_manager() + logger_ = get_logger() if cfg.get("metadata_enhancement.enabled", True) is False: logger_.warning("Metadata enhancement disabled in config.") return True @@ -58,16 +35,16 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf if album_info is None: album_info = {} - symbols = _get_mutagen_symbols() + symbols = get_mutagen_symbols() if not symbols: logger_.error("Mutagen is unavailable, cannot enhance metadata.") return False - file_lock = _get_file_lock(file_path) + file_lock = get_file_lock(file_path) with file_lock: logger_.info("Enhancing metadata for: %s", os.path.basename(file_path)) try: - _strip_all_non_audio_tags(file_path) + strip_all_non_audio_tags(file_path) audio_file = symbols.File(file_path) if audio_file is None: logger_.error("Could not load audio file with Mutagen: %s", file_path) @@ -84,12 +61,12 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf else: audio_file.add_tags() - _save_audio_file(audio_file, symbols) + save_audio_file(audio_file, symbols) metadata = extract_source_metadata(context, artist, album_info) if not metadata: logger_.error("Could not extract source metadata, saving with cleared tags.") - _save_audio_file(audio_file, symbols) + save_audio_file(audio_file, symbols) return True track_num_str = f"{metadata.get('track_number', 1)}/{metadata.get('total_tracks', 1)}" @@ -114,7 +91,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf audio_file.tags.add(symbols.TRCK(encoding=3, text=[track_num_str])) if metadata.get("disc_number"): audio_file.tags.add(symbols.TPOS(encoding=3, text=[str(metadata["disc_number"])])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): if metadata.get("title"): audio_file["title"] = [metadata["title"]] if metadata.get("artist"): @@ -161,14 +138,14 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf if quality and cfg.get("metadata_enhancement.tags.quality_tag", True) is not False: if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TXXX(encoding=3, desc="QUALITY", text=[quality])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): audio_file["quality"] = [quality] elif isinstance(audio_file, symbols.MP4): audio_file["----:com.apple.iTunes:QUALITY"] = [symbols.MP4FreeForm(quality.encode("utf-8"))] - _save_audio_file(audio_file, symbols) + save_audio_file(audio_file, symbols) - verified = _verify_metadata_written(file_path) + verified = verify_metadata_written(file_path) if verified: logger_.info("Metadata enhanced successfully.") else: diff --git a/core/metadata_lyrics.py b/core/metadata_lyrics.py index 99a7612b..ead86fc9 100644 --- a/core/metadata_lyrics.py +++ b/core/metadata_lyrics.py @@ -9,12 +9,16 @@ from core.import_context import ( get_import_original_search, normalize_import_context, ) -from core.metadata_common import _get_config_manager, _get_logger +from core.metadata_common import get_config_manager, get_logger + +__all__ = [ + "generate_lrc_file", +] def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: - cfg = _get_config_manager() - logger_ = _get_logger() + cfg = get_config_manager() + logger_ = get_logger() if cfg.get("metadata_enhancement.lrclib_enabled", True) is False: return False diff --git a/core/metadata_source.py b/core/metadata_source.py index ce1ddbe4..a7c9d86d 100644 --- a/core/metadata_source.py +++ b/core/metadata_source.py @@ -8,6 +8,7 @@ import time from typing import Any, Dict from core.import_context import ( + extract_artist_name, get_import_clean_artist, get_import_clean_title, get_import_context_album, @@ -18,16 +19,20 @@ from core.import_context import ( get_source_tag_names, normalize_import_context, ) +from core.metadata_service import get_itunes_client +from database.music_database import get_database from core.metadata_common import ( - _extract_artist_name, - _get_config_manager, - _get_database, - _get_itunes_client, - _get_logger, - _get_mutagen_symbols, - _is_vorbis_like, + get_config_manager, + get_logger, + get_mutagen_symbols, + is_vorbis_like, ) +__all__ = [ + "extract_source_metadata", + "embed_source_ids", +] + _MB_RELEASE_CACHE: Dict[tuple, str] = {} _MB_RELEASE_CACHE_LOCK = threading.RLock() @@ -48,7 +53,7 @@ _EDITION_BARE_RE = re.compile( ) -def _normalize_album_cache_key(album_name: str) -> str: +def normalize_album_cache_key(album_name: str) -> str: result = _EDITION_PAREN_RE.sub("", album_name or "") result = _EDITION_BARE_RE.sub("", result) return result.lower().strip() @@ -58,8 +63,8 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di if album_info is None: album_info = {} - cfg = _get_config_manager() - logger_ = _get_logger() + cfg = get_config_manager() + logger_ = get_logger() context = normalize_import_context(context) original_search = get_import_original_search(context) album_ctx = get_import_context_album(context) @@ -68,7 +73,7 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di source_ids = get_import_source_ids(context) artist_dict = artist if isinstance(artist, dict) else { - "name": _extract_artist_name(artist), + "name": extract_artist_name(artist), "id": getattr(artist, "id", ""), "genres": list(getattr(artist, "genres", []) or []), } @@ -135,7 +140,7 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di artist_id = str(artist_dict.get("id", "")) if source == "itunes" and artist_id.isdigit(): try: - itunes_client = _get_itunes_client() + itunes_client = get_itunes_client() if itunes_client and hasattr(itunes_client, "resolve_primary_artist"): resolved = itunes_client.resolve_primary_artist(artist_id) if resolved and resolved != raw_album_artist: @@ -199,9 +204,9 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=None): - cfg = _get_config_manager() - logger_ = _get_logger() - symbols = _get_mutagen_symbols() + cfg = get_config_manager() + logger_ = get_logger() + symbols = get_mutagen_symbols() if not symbols: return @@ -323,7 +328,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N if not isinstance(source_order, list) or not source_order: source_order = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"] - db = _get_database() + db = get_database() for source_name in source_order: if source_name == "musicbrainz": @@ -723,7 +728,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N key = f"----:com.apple.iTunes:{mp4_tag_map.get(tag_name, tag_name)}" audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))] written.append(key) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): for tag_name, value in filtered_tags.items(): audio_file[vorbis_tag_map.get(tag_name, tag_name)] = [str(value)] written.append(vorbis_tag_map.get(tag_name, tag_name)) @@ -737,7 +742,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N metadata["date"] = release_year if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): audio_file["date"] = [release_year] elif isinstance(audio_file, symbols.MP4): audio_file["\xa9day"] = [release_year] @@ -747,7 +752,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N bpm_int = int(pp["deezer_bpm"]) if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): audio_file["BPM"] = [str(bpm_int)] elif isinstance(audio_file, symbols.MP4): audio_file["tmpo"] = [bpm_int] @@ -756,7 +761,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N if _tag_enabled("audiodb.tags.mood") and pp["audiodb_mood"]: if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): audio_file["MOOD"] = [pp["audiodb_mood"]] elif isinstance(audio_file, symbols.MP4): audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))] @@ -764,7 +769,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N if _tag_enabled("audiodb.tags.style") and pp["audiodb_style"]: if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): audio_file["STYLE"] = [pp["audiodb_style"]] elif isinstance(audio_file, symbols.MP4): audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))] @@ -795,7 +800,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N genre_string = ", ".join(merged) if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): audio_file["GENRE"] = [genre_string] elif isinstance(audio_file, symbols.MP4): audio_file["\xa9gen"] = [genre_string] @@ -814,7 +819,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N isrc_source, final_isrc = isrc_candidates[0] if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): audio_file["ISRC"] = [final_isrc] elif isinstance(audio_file, symbols.MP4): audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))] @@ -829,7 +834,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N copyright_source, final_copyright = copyright_candidates[0] if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): audio_file["COPYRIGHT"] = [final_copyright] elif isinstance(audio_file, symbols.MP4): audio_file["cprt"] = [final_copyright] @@ -838,7 +843,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N if _tag_enabled("qobuz.tags.label") and pp["qobuz_label"]: if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): audio_file["LABEL"] = [pp["qobuz_label"]] elif isinstance(audio_file, symbols.MP4): audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))] @@ -846,7 +851,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N if _tag_enabled("lastfm.tags.url") and pp["lastfm_url"]: if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): audio_file["LASTFM_URL"] = [pp["lastfm_url"]] elif isinstance(audio_file, symbols.MP4): audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))] @@ -854,7 +859,7 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N if _tag_enabled("genius.tags.url") and pp["genius_url"]: if isinstance(audio_file.tags, symbols.ID3): audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]])) - elif _is_vorbis_like(audio_file, symbols): + elif is_vorbis_like(audio_file, symbols): audio_file["GENIUS_URL"] = [pp["genius_url"]] elif isinstance(audio_file, symbols.MP4): audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))] diff --git a/tests/test_metadata_enrichment.py b/tests/test_metadata_enrichment.py index 8f557972..577bda98 100644 --- a/tests/test_metadata_enrichment.py +++ b/tests/test_metadata_enrichment.py @@ -110,8 +110,8 @@ def _fake_symbols(audio): def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_for_non_itunes_sources(monkeypatch): - monkeypatch.setattr(ms, "_get_config_manager", lambda: _Config({"file_organization.collab_artist_mode": "first"})) - monkeypatch.setattr(ms, "_get_itunes_client", lambda: (_ for _ in ()).throw(AssertionError("itunes fallback should not run for non-itunes sources"))) + monkeypatch.setattr(ms, "get_config_manager", lambda: _Config({"file_organization.collab_artist_mode": "first"})) + monkeypatch.setattr(ms, "get_itunes_client", lambda: (_ for _ in ()).throw(AssertionError("itunes fallback should not run for non-itunes sources"))) context = { "source": "spotify", @@ -160,9 +160,9 @@ def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatch): audio = _FakeAudio() symbols = _fake_symbols(audio) - monkeypatch.setattr(ms, "_get_config_manager", lambda: _Config()) - monkeypatch.setattr(ms, "_get_mutagen_symbols", lambda: symbols) - monkeypatch.setattr(ms, "_get_database", lambda: None) + monkeypatch.setattr(ms, "get_config_manager", lambda: _Config()) + monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "get_database", lambda: None) current_metadata = { "source": "deezer", @@ -182,7 +182,7 @@ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatc audio = _FakeAudio() symbols = _fake_symbols(audio) - monkeypatch.setattr(ms, "_get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols) legacy_metadata = { "source": "", @@ -214,18 +214,18 @@ def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch strip_calls = [] verify_calls = [] - monkeypatch.setattr(me, "_get_config_manager", lambda: _Config( + monkeypatch.setattr(me, "get_config_manager", lambda: _Config( { "metadata_enhancement.enabled": True, "metadata_enhancement.embed_album_art": False, "metadata_enhancement.tags.write_multi_artist": False, } )) - monkeypatch.setattr(ms, "_get_config_manager", lambda: _Config()) - monkeypatch.setattr(me, "_get_mutagen_symbols", lambda: symbols) - monkeypatch.setattr(ms, "_get_mutagen_symbols", lambda: symbols) - monkeypatch.setattr(ms, "_get_database", lambda: None) - monkeypatch.setattr(me, "_strip_all_non_audio_tags", lambda file_path: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0}) + monkeypatch.setattr(ms, "get_config_manager", lambda: _Config()) + monkeypatch.setattr(me, "get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "get_database", lambda: None) + monkeypatch.setattr(me, "strip_all_non_audio_tags", lambda file_path: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0}) monkeypatch.setattr( me, "extract_source_metadata", @@ -247,7 +247,7 @@ def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch }, ) monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None) - monkeypatch.setattr(me, "_verify_metadata_written", lambda file_path: verify_calls.append(file_path) or True) + monkeypatch.setattr(me, "verify_metadata_written", lambda file_path: verify_calls.append(file_path) or True) album_info = {} result = me.enhance_file_metadata( @@ -269,7 +269,7 @@ def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch): - monkeypatch.setattr(ma, "_get_config_manager", lambda: _Config( + monkeypatch.setattr(ma, "get_config_manager", lambda: _Config( { "metadata_enhancement.cover_art_download": True, "metadata_enhancement.prefer_caa_art": False, @@ -281,7 +281,7 @@ def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch): target_dir = tmp_path / "Album One" target_dir.mkdir() - me.download_cover_art( + ma.download_cover_art( {}, str(target_dir), {"album": {"image_url": "https://img.example/album.jpg"}}, From e10df4caf28c819a7a93be5b936d6174e94b6103 Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sat, 25 Apr 2026 12:36:24 +0300 Subject: [PATCH 05/19] Rehome import helpers into core/imports - Move import flow modules into a dedicated package - Update app and test imports to the new namespace - Group the import-focused tests under tests/imports --- api/downloads.py | 2 +- api/system.py | 4 +-- core/imports/__init__.py | 1 + core/{import_album.py => imports/album.py} | 6 ++-- .../album_naming.py} | 4 +-- .../{import_context.py => imports/context.py} | 0 .../file_ops.py} | 10 ++----- .../filename.py} | 0 core/{import_guards.py => imports/guards.py} | 6 ++-- core/{import_paths.py => imports/paths.py} | 8 +++--- .../pipeline.py} | 18 ++++++------ .../resolution.py} | 2 +- .../runtime_state.py} | 2 +- .../side_effects.py} | 4 +-- .../{import_staging.py => imports/staging.py} | 6 ++-- core/metadata_artwork.py | 2 +- core/metadata_lyrics.py | 2 +- core/metadata_service.py | 2 +- core/metadata_source.py | 2 +- core/soulseek_client.py | 2 +- tests/{ => imports}/test_import_album.py | 2 +- tests/{ => imports}/test_import_context.py | 2 +- tests/{ => imports}/test_import_file_ops.py | 6 ++-- tests/{ => imports}/test_import_filename.py | 2 +- tests/{ => imports}/test_import_guards.py | 2 +- tests/{ => imports}/test_import_paths.py | 4 +-- tests/{ => imports}/test_import_pipeline.py | 6 ++-- .../{ => imports}/test_import_side_effects.py | 2 +- tests/{ => imports}/test_import_staging.py | 2 +- tests/test_downloads_pagination.py | 2 +- web_server.py | 28 +++++++++---------- 31 files changed, 68 insertions(+), 73 deletions(-) create mode 100644 core/imports/__init__.py rename core/{import_album.py => imports/album.py} (99%) rename core/{import_album_naming.py => imports/album_naming.py} (98%) rename core/{import_context.py => imports/context.py} (100%) rename core/{import_file_ops.py => imports/file_ops.py} (97%) rename core/{import_filename.py => imports/filename.py} (100%) rename core/{import_guards.py => imports/guards.py} (97%) rename core/{import_paths.py => imports/paths.py} (99%) rename core/{import_pipeline.py => imports/pipeline.py} (99%) rename core/{import_resolution.py => imports/resolution.py} (99%) rename core/{import_runtime_state.py => imports/runtime_state.py} (99%) rename core/{import_side_effects.py => imports/side_effects.py} (99%) rename core/{import_staging.py => imports/staging.py} (99%) rename tests/{ => imports}/test_import_album.py (99%) rename tests/{ => imports}/test_import_context.py (99%) rename tests/{ => imports}/test_import_file_ops.py (94%) rename tests/{ => imports}/test_import_filename.py (94%) rename tests/{ => imports}/test_import_guards.py (97%) rename tests/{ => imports}/test_import_paths.py (98%) rename tests/{ => imports}/test_import_pipeline.py (97%) rename tests/{ => imports}/test_import_side_effects.py (98%) rename tests/{ => imports}/test_import_staging.py (99%) diff --git a/api/downloads.py b/api/downloads.py index fd36e91e..88fd690c 100644 --- a/api/downloads.py +++ b/api/downloads.py @@ -5,7 +5,7 @@ Download management endpoints — list, cancel active downloads. from flask import request, current_app from .auth import require_api_key from .helpers import api_success, api_error -from core.import_runtime_state import download_tasks, tasks_lock +from core.imports.runtime_state import download_tasks, tasks_lock def _serialize_download(task_id, task): diff --git a/api/system.py b/api/system.py index b6a72141..9bdf107b 100644 --- a/api/system.py +++ b/api/system.py @@ -55,7 +55,7 @@ def register_routes(bp): def system_activity(): """Recent activity feed.""" try: - from core.import_runtime_state import activity_feed + from core.imports.runtime_state import activity_feed items = list(activity_feed) if activity_feed else [] return api_success({"activities": items}) except Exception as e: @@ -74,7 +74,7 @@ def register_routes(bp): # Active download count download_count = 0 try: - from core.import_runtime_state import download_tasks, tasks_lock + from core.imports.runtime_state import download_tasks, tasks_lock with tasks_lock: download_count = sum( 1 for t in download_tasks.values() diff --git a/core/imports/__init__.py b/core/imports/__init__.py new file mode 100644 index 00000000..d5d5418f --- /dev/null +++ b/core/imports/__init__.py @@ -0,0 +1 @@ +"""Import flow helpers package.""" diff --git a/core/import_album.py b/core/imports/album.py similarity index 99% rename from core/import_album.py rename to core/imports/album.py index 1f439f1b..db6c7d01 100644 --- a/core/import_album.py +++ b/core/imports/album.py @@ -4,12 +4,12 @@ from __future__ import annotations from typing import Any, Dict, Iterable, List, Optional, Set -from core.import_context import normalize_import_context -from core.import_staging import collect_staging_files +from core.imports.context import normalize_import_context +from core.imports.staging import collect_staging_files from utils.logging_config import get_logger -logger = get_logger("import_album") +logger = get_logger("imports.album") def get_client_for_source(source: str): from core.metadata_service import get_client_for_source as _get_client_for_source diff --git a/core/import_album_naming.py b/core/imports/album_naming.py similarity index 98% rename from core/import_album_naming.py rename to core/imports/album_naming.py index 338f326d..6d3f379a 100644 --- a/core/import_album_naming.py +++ b/core/imports/album_naming.py @@ -6,11 +6,11 @@ import re import threading from typing import Any, Dict -from core.import_context import extract_artist_name +from core.imports.context import extract_artist_name from utils.logging_config import get_logger -logger = get_logger("import_album_naming") +logger = get_logger("imports.album_naming") _album_cache_lock = threading.Lock() _album_editions: dict[str, str] = {} diff --git a/core/import_context.py b/core/imports/context.py similarity index 100% rename from core/import_context.py rename to core/imports/context.py diff --git a/core/import_file_ops.py b/core/imports/file_ops.py similarity index 97% rename from core/import_file_ops.py rename to core/imports/file_ops.py index 21cf9318..73185aac 100644 --- a/core/import_file_ops.py +++ b/core/imports/file_ops.py @@ -1,4 +1,4 @@ -"""Shared file and path helpers for import processing.""" +"""File operation helpers for the import flow.""" from __future__ import annotations @@ -8,14 +8,8 @@ import shutil import subprocess import time from pathlib import Path -from typing import Any, Dict, Optional -# Backward-compatible re-exports; canonical homes are core.import_filename -# and core.import_staging. -from core.import_filename import extract_track_number_from_filename -from core.import_staging import read_staging_file_metadata - -logger = logging.getLogger("import_file_ops") +logger = logging.getLogger("imports.file_ops") def _get_config_manager(): diff --git a/core/import_filename.py b/core/imports/filename.py similarity index 100% rename from core/import_filename.py rename to core/imports/filename.py diff --git a/core/import_guards.py b/core/imports/guards.py similarity index 97% rename from core/import_guards.py rename to core/imports/guards.py index de2a3185..4d97c9fb 100644 --- a/core/import_guards.py +++ b/core/imports/guards.py @@ -8,7 +8,7 @@ from datetime import datetime from pathlib import Path from typing import Any, Dict, Optional -from core.import_context import ( +from core.imports.context import ( get_import_clean_artist, get_import_clean_title, get_import_context_artist, @@ -16,12 +16,12 @@ from core.import_context import ( get_import_track_info, normalize_import_context, ) -from core.import_file_ops import safe_move_file +from core.imports.file_ops import safe_move_file from database.music_database import MusicDatabase from utils.logging_config import get_logger -logger = get_logger("import_guards") +logger = get_logger("imports.guards") def _get_config_manager(): diff --git a/core/import_paths.py b/core/imports/paths.py similarity index 99% rename from core/import_paths.py rename to core/imports/paths.py index 39e280c8..5ac357d1 100644 --- a/core/import_paths.py +++ b/core/imports/paths.py @@ -9,10 +9,10 @@ import re from pathlib import Path from typing import Any -# Album grouping lives in core.import_album_naming; this module keeps the +# Album grouping lives in core.imports.album_naming; this module keeps the # imported helper because the path builder still needs it. -from core.import_album_naming import resolve_album_group -from core.import_context import ( +from core.imports.album_naming import resolve_album_group +from core.imports.context import ( extract_artist_name, get_import_clean_title, get_import_context_album, @@ -22,7 +22,7 @@ from core.import_context import ( normalize_import_context, ) -logger = logging.getLogger("import_paths") +logger = logging.getLogger("imports.paths") def _get_config_manager(): diff --git a/core/import_pipeline.py b/core/imports/pipeline.py similarity index 99% rename from core/import_pipeline.py rename to core/imports/pipeline.py index 62bc1f74..99eee1a5 100644 --- a/core/import_pipeline.py +++ b/core/imports/pipeline.py @@ -8,7 +8,7 @@ import threading import time from config.settings import config_manager -from core.import_file_ops import ( +from core.imports.file_ops import ( cleanup_empty_directories, create_lossy_copy, downsample_hires_flac, @@ -16,7 +16,7 @@ from core.import_file_ops import ( get_quality_tier_from_extension, safe_move_file, ) -from core.import_context import ( +from core.imports.context import ( build_import_album_info, extract_artist_name, get_import_clean_artist, @@ -28,9 +28,9 @@ from core.import_context import ( get_import_track_info, normalize_import_context, ) -from core.import_filename import extract_track_number_from_filename -from core.import_guards import check_flac_bit_depth, move_to_quarantine -from core.import_side_effects import ( +from core.imports.filename import extract_track_number_from_filename +from core.imports.guards import check_flac_bit_depth, move_to_quarantine +from core.imports.side_effects import ( check_and_remove_from_wishlist, emit_track_downloaded, record_download_provenance, @@ -38,7 +38,7 @@ from core.import_side_effects import ( record_retag_download, record_soulsync_library_entry, ) -from core.import_runtime_state import ( +from core.imports.runtime_state import ( add_activity_item, detect_album_info_web, download_batches, @@ -54,18 +54,18 @@ from core.import_runtime_state import ( from core.metadata_artwork import download_cover_art from core.metadata_common import wipe_source_tags from core.metadata_enrichment import enhance_file_metadata -from core.import_paths import ( +from core.imports.paths import ( build_final_path_for_track, build_simple_download_destination, docker_resolve_path, ) -from core.import_album_naming import resolve_album_group +from core.imports.album_naming import resolve_album_group from core.metadata_lyrics import generate_lrc_file from database.music_database import get_database from utils.logging_config import get_logger -logger = get_logger("import_pipeline") +logger = get_logger("imports.pipeline") pp_logger = get_logger("post_processing") diff --git a/core/import_resolution.py b/core/imports/resolution.py similarity index 99% rename from core/import_resolution.py rename to core/imports/resolution.py index 8f9c712c..40e4149c 100644 --- a/core/import_resolution.py +++ b/core/imports/resolution.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional from utils.logging_config import get_logger -logger = get_logger("import_resolution") +logger = get_logger("imports.resolution") def _get_metadata_service(): diff --git a/core/import_runtime_state.py b/core/imports/runtime_state.py similarity index 99% rename from core/import_runtime_state.py rename to core/imports/runtime_state.py index 9abf2baa..4e24d06f 100644 --- a/core/import_runtime_state.py +++ b/core/imports/runtime_state.py @@ -6,7 +6,7 @@ import threading import time from typing import Any, Dict, Optional -from core.import_context import ( +from core.imports.context import ( build_import_album_info, extract_artist_name, get_import_clean_artist, diff --git a/core/import_side_effects.py b/core/imports/side_effects.py similarity index 99% rename from core/import_side_effects.py rename to core/imports/side_effects.py index 6c456e3d..dbc5bbf7 100644 --- a/core/import_side_effects.py +++ b/core/imports/side_effects.py @@ -7,7 +7,7 @@ import json import os from typing import Any, Dict, List, Optional -from core.import_context import ( +from core.imports.context import ( extract_artist_name, get_import_clean_album, get_import_clean_artist, @@ -26,7 +26,7 @@ from database.music_database import get_database from utils.logging_config import get_logger -logger = get_logger("import_side_effects") +logger = get_logger("imports.side_effects") def _get_config_manager(): diff --git a/core/import_staging.py b/core/imports/staging.py similarity index 99% rename from core/import_staging.py rename to core/imports/staging.py index 8f8fc6f0..44edb0bf 100644 --- a/core/import_staging.py +++ b/core/imports/staging.py @@ -6,11 +6,11 @@ import os import threading from typing import Any, Dict, Iterable, List, Optional, Tuple -from core.import_paths import docker_resolve_path -from core.import_filename import extract_track_number_from_filename +from core.imports.paths import docker_resolve_path +from core.imports.filename import extract_track_number_from_filename from utils.logging_config import get_logger -logger = get_logger("import_staging") +logger = get_logger("imports.staging") AUDIO_EXTENSIONS = {".mp3", ".flac", ".ogg", ".opus", ".m4a", ".aac", ".wav", ".wma", ".aiff", ".aif", ".ape"} diff --git a/core/metadata_artwork.py b/core/metadata_artwork.py index b84f5274..8d43b0af 100644 --- a/core/metadata_artwork.py +++ b/core/metadata_artwork.py @@ -6,7 +6,7 @@ import os import re import urllib.request -from core.import_context import get_import_context_album +from core.imports.context import get_import_context_album from core.metadata_common import ( get_config_manager, get_image_dimensions, diff --git a/core/metadata_lyrics.py b/core/metadata_lyrics.py index ead86fc9..00719c5c 100644 --- a/core/metadata_lyrics.py +++ b/core/metadata_lyrics.py @@ -2,7 +2,7 @@ from __future__ import annotations -from core.import_context import ( +from core.imports.context import ( get_import_clean_album, get_import_clean_title, get_import_context_album, diff --git a/core/metadata_service.py b/core/metadata_service.py index 68cf0b92..66487d60 100644 --- a/core/metadata_service.py +++ b/core/metadata_service.py @@ -849,7 +849,7 @@ def get_single_track_import_context( source_override: Optional[str] = None, ) -> Dict[str, Any]: """Compatibility wrapper for the single-track import resolver.""" - from core.import_resolution import get_single_track_import_context as _get_single_track_import_context + from core.imports.resolution import get_single_track_import_context as _get_single_track_import_context return _get_single_track_import_context( title, diff --git a/core/metadata_source.py b/core/metadata_source.py index a7c9d86d..7f4411e9 100644 --- a/core/metadata_source.py +++ b/core/metadata_source.py @@ -7,7 +7,7 @@ import threading import time from typing import Any, Dict -from core.import_context import ( +from core.imports.context import ( extract_artist_name, get_import_clean_artist, get_import_clean_title, diff --git a/core/soulseek_client.py b/core/soulseek_client.py index e859fd4e..3110c305 100644 --- a/core/soulseek_client.py +++ b/core/soulseek_client.py @@ -8,7 +8,7 @@ import time from pathlib import Path from utils.logging_config import get_logger from config.settings import config_manager -from core.import_filename import parse_filename_metadata +from core.imports.filename import parse_filename_metadata logger = get_logger("soulseek_client") diff --git a/tests/test_import_album.py b/tests/imports/test_import_album.py similarity index 99% rename from tests/test_import_album.py rename to tests/imports/test_import_album.py index 45706da3..5b11ca8e 100644 --- a/tests/test_import_album.py +++ b/tests/imports/test_import_album.py @@ -1,4 +1,4 @@ -import core.import_album as import_album +import core.imports.album as import_album class _FakeEngine: diff --git a/tests/test_import_context.py b/tests/imports/test_import_context.py similarity index 99% rename from tests/test_import_context.py rename to tests/imports/test_import_context.py index f91acf19..7e2f242a 100644 --- a/tests/test_import_context.py +++ b/tests/imports/test_import_context.py @@ -1,6 +1,6 @@ import pytest -from core.import_context import ( +from core.imports.context import ( build_import_album_info, get_import_clean_album, get_import_clean_artist, diff --git a/tests/test_import_file_ops.py b/tests/imports/test_import_file_ops.py similarity index 94% rename from tests/test_import_file_ops.py rename to tests/imports/test_import_file_ops.py index 056c95bf..a07c5c4a 100644 --- a/tests/test_import_file_ops.py +++ b/tests/imports/test_import_file_ops.py @@ -1,12 +1,12 @@ import sys import types -from core.import_file_ops import ( +from core.imports.file_ops import ( cleanup_empty_directories, safe_move_file, ) -from core.import_filename import extract_track_number_from_filename -from core.import_staging import read_staging_file_metadata +from core.imports.filename import extract_track_number_from_filename +from core.imports.staging import read_staging_file_metadata def test_extract_track_number_from_filename_handles_common_patterns(): diff --git a/tests/test_import_filename.py b/tests/imports/test_import_filename.py similarity index 94% rename from tests/test_import_filename.py rename to tests/imports/test_import_filename.py index a5b32ff2..59c29dcf 100644 --- a/tests/test_import_filename.py +++ b/tests/imports/test_import_filename.py @@ -1,6 +1,6 @@ import pytest -from core.import_filename import parse_filename_metadata +from core.imports.filename import parse_filename_metadata @pytest.mark.parametrize( diff --git a/tests/test_import_guards.py b/tests/imports/test_import_guards.py similarity index 97% rename from tests/test_import_guards.py rename to tests/imports/test_import_guards.py index a7b95372..ee71a1dd 100644 --- a/tests/test_import_guards.py +++ b/tests/imports/test_import_guards.py @@ -1,6 +1,6 @@ from types import SimpleNamespace -from core import import_guards as guards +from core.imports import guards class _FakeDB: diff --git a/tests/test_import_paths.py b/tests/imports/test_import_paths.py similarity index 98% rename from tests/test_import_paths.py rename to tests/imports/test_import_paths.py index a3e766f9..656279f9 100644 --- a/tests/test_import_paths.py +++ b/tests/imports/test_import_paths.py @@ -1,5 +1,5 @@ -import core.import_album_naming as album_naming -import core.import_paths as import_paths +import core.imports.album_naming as album_naming +import core.imports.paths as import_paths class _Config: diff --git a/tests/test_import_pipeline.py b/tests/imports/test_import_pipeline.py similarity index 97% rename from tests/test_import_pipeline.py rename to tests/imports/test_import_pipeline.py index 768a8788..7c50c942 100644 --- a/tests/test_import_pipeline.py +++ b/tests/imports/test_import_pipeline.py @@ -2,9 +2,9 @@ import logging import sys import types -import core.import_pipeline as import_pipeline -import core.import_paths as import_paths -import core.import_runtime_state as runtime_state +import core.imports.pipeline as import_pipeline +import core.imports.paths as import_paths +import core.imports.runtime_state as runtime_state class _Config: diff --git a/tests/test_import_side_effects.py b/tests/imports/test_import_side_effects.py similarity index 98% rename from tests/test_import_side_effects.py rename to tests/imports/test_import_side_effects.py index 510a9ab0..697d845b 100644 --- a/tests/test_import_side_effects.py +++ b/tests/imports/test_import_side_effects.py @@ -1,7 +1,7 @@ import sqlite3 from types import SimpleNamespace -from core import import_side_effects as side_effects +from core.imports import side_effects class _FakeDB: diff --git a/tests/test_import_staging.py b/tests/imports/test_import_staging.py similarity index 99% rename from tests/test_import_staging.py rename to tests/imports/test_import_staging.py index 1cd5e15b..56571ede 100644 --- a/tests/test_import_staging.py +++ b/tests/imports/test_import_staging.py @@ -1,6 +1,6 @@ from types import SimpleNamespace -import core.import_staging as import_staging +import core.imports.staging as import_staging class FakeClient: diff --git a/tests/test_downloads_pagination.py b/tests/test_downloads_pagination.py index f85dd96d..24340b66 100644 --- a/tests/test_downloads_pagination.py +++ b/tests/test_downloads_pagination.py @@ -44,7 +44,7 @@ _install_flask_limiter_stub() from flask import Flask, Blueprint # noqa: E402 from api import downloads as downloads_mod # noqa: E402 -import core.import_runtime_state as runtime_state # noqa: E402 +import core.imports.runtime_state as runtime_state # noqa: E402 def _make_task(status="downloading", when=None): diff --git a/web_server.py b/web_server.py index e6cd49c2..6d7da994 100644 --- a/web_server.py +++ b/web_server.py @@ -97,7 +97,7 @@ from core.database_update_worker import DatabaseUpdateWorker from core.web_scan_manager import WebScanManager from core.lyrics_client import lyrics_client from core.metadata_cache import get_metadata_cache -from core.import_context import ( +from core.imports.context import ( build_import_album_info, get_import_clean_album, get_import_clean_artist, @@ -114,14 +114,14 @@ from core.import_context import ( get_source_tag_names, normalize_import_context, ) -from core.import_album import ( +from core.imports.album import ( build_album_import_context, build_album_import_match_payload, resolve_album_artist_context, ) -from core.import_album_naming import resolve_album_group as _resolve_album_group -from core.import_filename import extract_track_number_from_filename, parse_filename_metadata -from core.import_staging import ( +from core.imports.album_naming import resolve_album_group as _resolve_album_group +from core.imports.filename import extract_track_number_from_filename, parse_filename_metadata +from core.imports.staging import ( get_import_suggestions_cache, get_primary_source, get_staging_path, @@ -131,9 +131,9 @@ from core.import_staging import ( search_import_tracks, start_import_suggestions_cache, ) -from core.import_paths import build_final_path_for_track as _build_final_path_for_track +from core.imports.paths import build_final_path_for_track as _build_final_path_for_track from core.metadata_common import get_file_lock -from core.import_runtime_state import ( +from core.imports.runtime_state import ( activity_feed, activity_feed_lock, add_activity_item, @@ -747,7 +747,7 @@ retag_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="RetagWork # Download Missing Tracks Modal State Management # Thread-safe state tracking for modal download functionality. -# Shared task/batch state now lives in core.import_runtime_state. +# Shared task/batch state now lives in core.imports.runtime_state. missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker") # Automatic Wishlist / Watchlist Processing Flags @@ -845,7 +845,7 @@ def _regenerate_batch_m3u(batch, tracks): except ImportError: _unidecode = lambda x: x - from core.import_paths import sanitize_filename as _sanitize_filename + from core.imports.paths import sanitize_filename as _sanitize_filename def _norm(text): return _unidecode(text).lower().strip() if text else '' @@ -965,7 +965,7 @@ def _sanitize_filename(filename: str) -> str: def _compute_m3u_folder(transfer_dir, context_type, playlist_name, artist_name='', album_name='', year=''): """Compute the target folder for an M3U file using the template system.""" - from core.import_paths import get_file_path_from_template + from core.imports.paths import get_file_path_from_template if context_type == 'album' and artist_name and album_name: template_context = { @@ -3967,7 +3967,7 @@ def _update_automation_progress(automation_id, **kwargs): pass # --- Global Matched Downloads Context Management --- -# Shared with core.import_runtime_state so the refactored pipeline and web +# Shared with core.imports.runtime_state so the refactored pipeline and web # server operate on the same context registry. _orphaned_download_keys = set() # Context keys of downloads abandoned during retry @@ -19773,7 +19773,7 @@ def _post_process_matched_download_with_verification(context_key, context, file_ NEW VERIFICATION WORKFLOW: Enhanced post-processing with file verification. Only sets task status to 'completed' after successful file verification and move operation. """ - from core.import_pipeline import post_process_matched_download_with_verification + from core.imports.pipeline import post_process_matched_download_with_verification return post_process_matched_download_with_verification( context_key, context, @@ -19896,11 +19896,11 @@ def _post_process_matched_download(context_key, context, file_path): Also handles simple downloads (from search page "Download" button) which just move files to /Transfer without metadata enhancement. """ - from core.import_pipeline import post_process_matched_download + from core.imports.pipeline import post_process_matched_download return post_process_matched_download(context_key, context, file_path, _build_import_pipeline_runtime()) def _build_import_pipeline_runtime(): - """Collect the live controller dependencies needed by core.import_pipeline.""" + """Collect the live controller dependencies needed by core.imports.pipeline.""" return types.SimpleNamespace( automation_engine=automation_engine, on_download_completed=_on_download_completed, From bdef127dd67f38b7dcd532242651e5603e7a42bf Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sat, 25 Apr 2026 13:11:55 +0300 Subject: [PATCH 06/19] Lift shared runtime state into core - Move app-wide task and activity registries out of core/imports - Share one runtime-state module across the web server, API, and import pipeline - Keep import-specific helpers focused on context and post-processing --- api/downloads.py | 2 +- api/system.py | 4 +- core/imports/context.py | 46 ++++++++++ core/imports/pipeline.py | 26 +++--- core/imports/runtime_state.py | 121 -------------------------- core/runtime_state.py | 65 ++++++++++++++ tests/imports/test_import_pipeline.py | 18 ++-- tests/test_downloads_pagination.py | 2 +- web_server.py | 34 ++++---- 9 files changed, 154 insertions(+), 164 deletions(-) delete mode 100644 core/imports/runtime_state.py create mode 100644 core/runtime_state.py diff --git a/api/downloads.py b/api/downloads.py index 88fd690c..2f087874 100644 --- a/api/downloads.py +++ b/api/downloads.py @@ -5,7 +5,7 @@ Download management endpoints — list, cancel active downloads. from flask import request, current_app from .auth import require_api_key from .helpers import api_success, api_error -from core.imports.runtime_state import download_tasks, tasks_lock +from core.runtime_state import download_tasks, tasks_lock def _serialize_download(task_id, task): diff --git a/api/system.py b/api/system.py index 9bdf107b..290da76e 100644 --- a/api/system.py +++ b/api/system.py @@ -55,7 +55,7 @@ def register_routes(bp): def system_activity(): """Recent activity feed.""" try: - from core.imports.runtime_state import activity_feed + from core.runtime_state import activity_feed items = list(activity_feed) if activity_feed else [] return api_success({"activities": items}) except Exception as e: @@ -74,7 +74,7 @@ def register_routes(bp): # Active download count download_count = 0 try: - from core.imports.runtime_state import download_tasks, tasks_lock + from core.runtime_state import download_tasks, tasks_lock with tasks_lock: download_count = sum( 1 for t in download_tasks.values() diff --git a/core/imports/context.py b/core/imports/context.py index cf9504ff..c19aa180 100644 --- a/core/imports/context.py +++ b/core/imports/context.py @@ -324,3 +324,49 @@ def build_import_album_info( "album_type": album_type, "total_tracks": int(total_tracks) if str(total_tracks).isdigit() else total_tracks, } + + +def detect_album_info_web(context, artist_context=None): + """Best-effort album detection for single-track downloads.""" + context = normalize_import_context(context) + if artist_context is None: + artist_context = context.get("artist") or {} + + album_info = build_import_album_info(context) + if album_info.get("is_album"): + return album_info + + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + + album_name = ( + album_ctx.get("name") + or track_info.get("album") + or original_search.get("album") + or "" + ) + track_name = ( + track_info.get("name") + or original_search.get("title") + or "" + ) + artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="") + + if album_name and track_name and album_name.strip().lower() not in { + track_name.strip().lower(), + artist_name.strip().lower(), + }: + return build_import_album_info( + context, + album_info={ + "album_name": album_name, + "track_number": track_info.get("track_number", 1), + "disc_number": track_info.get("disc_number", 1), + "album_image_url": album_ctx.get("image_url", ""), + "confidence": 0.5, + }, + force_album=True, + ) + + return None diff --git a/core/imports/pipeline.py b/core/imports/pipeline.py index 99eee1a5..68429c85 100644 --- a/core/imports/pipeline.py +++ b/core/imports/pipeline.py @@ -18,6 +18,7 @@ from core.imports.file_ops import ( ) from core.imports.context import ( build_import_album_info, + detect_album_info_web, extract_artist_name, get_import_clean_artist, get_import_clean_title, @@ -38,17 +39,16 @@ from core.imports.side_effects import ( record_retag_download, record_soulsync_library_entry, ) -from core.imports.runtime_state import ( +from core.runtime_state import ( add_activity_item, - detect_album_info_web, download_batches, download_tasks, matched_context_lock, matched_downloads_context, mark_task_completed as _mark_task_completed, - _post_process_locks, - _post_process_locks_lock, - _processed_download_ids, + post_process_locks, + post_process_locks_lock, + processed_download_ids, tasks_lock, ) from core.metadata_artwork import download_cover_art @@ -79,10 +79,10 @@ def post_process_matched_download(context_key, context, file_path, runtime): if on_download_completed: on_download_completed(batch_id, task_id, success=success) - with _post_process_locks_lock: - if context_key not in _post_process_locks: - _post_process_locks[context_key] = threading.Lock() - file_lock = _post_process_locks[context_key] + with post_process_locks_lock: + if context_key not in post_process_locks: + post_process_locks[context_key] = threading.Lock() + file_lock = post_process_locks[context_key] file_lock.acquire() try: @@ -747,8 +747,8 @@ def post_process_matched_download(context_key, context, file_path, runtime): source_exists = os.path.exists(file_path) if file_path else False if source_exists: - if context_key in _processed_download_ids: - _processed_download_ids.remove(context_key) + if context_key in processed_download_ids: + processed_download_ids.remove(context_key) logger.warning(f"Removed {context_key} from processed set - will retry on next check") with matched_context_lock: if context_key not in matched_downloads_context: @@ -758,8 +758,8 @@ def post_process_matched_download(context_key, context, file_path, runtime): logger.warning(f"Source file gone, not retrying: {context_key}") finally: file_lock.release() - with _post_process_locks_lock: - _post_process_locks.pop(context_key, None) + with post_process_locks_lock: + post_process_locks.pop(context_key, None) def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime): diff --git a/core/imports/runtime_state.py b/core/imports/runtime_state.py deleted file mode 100644 index 4e24d06f..00000000 --- a/core/imports/runtime_state.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Shared runtime state and tiny helpers for import/post-processing code.""" - -from __future__ import annotations - -import threading -import time -from typing import Any, Dict, Optional - -from core.imports.context import ( - build_import_album_info, - extract_artist_name, - get_import_clean_artist, - get_import_context_album, - get_import_original_search, - get_import_track_info, - normalize_import_context, -) - -matched_context_lock = threading.Lock() -matched_downloads_context: Dict[str, Dict[str, Any]] = {} -tasks_lock = threading.Lock() -download_tasks: Dict[str, Dict[str, Any]] = {} -download_batches: Dict[str, Dict[str, Any]] = {} -_processed_download_ids = set() -_post_process_locks: Dict[str, threading.Lock] = {} -_post_process_locks_lock = threading.Lock() - -activity_feed = [] -activity_feed_lock = threading.Lock() -_activity_toast_emitter = None - - -def set_activity_toast_emitter(emitter) -> None: - """Set the WebSocket-style emitter used by add_activity_item.""" - global _activity_toast_emitter - _activity_toast_emitter = emitter - - -def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True): - """Append an activity item and emit a toast if an emitter is configured.""" - activity_item = { - "icon": icon, - "title": title, - "subtitle": subtitle, - "time": time_ago, - "timestamp": time.time(), - "show_toast": show_toast, - } - with activity_feed_lock: - activity_feed.append(activity_item) - if len(activity_feed) > 20: - activity_feed.pop(0) - - if show_toast and _activity_toast_emitter is not None: - try: - _activity_toast_emitter("dashboard:toast", activity_item) - except Exception: - pass - - return activity_item - - -def mark_task_completed(task_id: str, track_info: Optional[Dict[str, Any]] = None) -> bool: - """Mark a download task as completed in the shared task registry.""" - with tasks_lock: - task = download_tasks.get(task_id) - if not task: - return False - - task["status"] = "completed" - task["stream_processed"] = True - task["status_change_time"] = time.time() - if track_info is not None: - task["track_info"] = track_info - return True - - -def detect_album_info_web(context, artist_context=None): - """Best-effort album detection for single-track downloads.""" - context = normalize_import_context(context) - if artist_context is None: - artist_context = context.get("artist") or {} - - album_info = build_import_album_info(context) - if album_info.get("is_album"): - return album_info - - album_ctx = get_import_context_album(context) - track_info = get_import_track_info(context) - original_search = get_import_original_search(context) - - album_name = ( - album_ctx.get("name") - or track_info.get("album") - or original_search.get("album") - or "" - ) - track_name = ( - track_info.get("name") - or original_search.get("title") - or "" - ) - artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="") - - if album_name and track_name and album_name.strip().lower() not in { - track_name.strip().lower(), - artist_name.strip().lower(), - }: - return build_import_album_info( - context, - album_info={ - "album_name": album_name, - "track_number": track_info.get("track_number", 1), - "disc_number": track_info.get("disc_number", 1), - "album_image_url": album_ctx.get("image_url", ""), - "confidence": 0.5, - }, - force_album=True, - ) - - return None diff --git a/core/runtime_state.py b/core/runtime_state.py new file mode 100644 index 00000000..8e4f4b64 --- /dev/null +++ b/core/runtime_state.py @@ -0,0 +1,65 @@ +"""Shared runtime state and tiny helpers for the app.""" + +from __future__ import annotations + +import threading +import time +from typing import Any, Dict, Optional + +matched_context_lock = threading.Lock() +matched_downloads_context: Dict[str, Dict[str, Any]] = {} +tasks_lock = threading.Lock() +download_tasks: Dict[str, Dict[str, Any]] = {} +download_batches: Dict[str, Dict[str, Any]] = {} +processed_download_ids = set() +post_process_locks: Dict[str, threading.Lock] = {} +post_process_locks_lock = threading.Lock() + +activity_feed = [] +activity_feed_lock = threading.Lock() +_activity_toast_emitter = None + + +def set_activity_toast_emitter(emitter) -> None: + """Set the WebSocket-style emitter used by add_activity_item.""" + global _activity_toast_emitter + _activity_toast_emitter = emitter + + +def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True): + """Append an activity item and emit a toast if an emitter is configured.""" + activity_item = { + "icon": icon, + "title": title, + "subtitle": subtitle, + "time": time_ago, + "timestamp": time.time(), + "show_toast": show_toast, + } + with activity_feed_lock: + activity_feed.append(activity_item) + if len(activity_feed) > 20: + activity_feed.pop(0) + + if show_toast and _activity_toast_emitter is not None: + try: + _activity_toast_emitter("dashboard:toast", activity_item) + except Exception: + pass + + return activity_item + + +def mark_task_completed(task_id: str, track_info: Optional[Dict[str, Any]] = None) -> bool: + """Mark a download task as completed in the shared task registry.""" + with tasks_lock: + task = download_tasks.get(task_id) + if not task: + return False + + task["status"] = "completed" + task["stream_processed"] = True + task["status_change_time"] = time.time() + if track_info is not None: + task["track_info"] = track_info + return True diff --git a/tests/imports/test_import_pipeline.py b/tests/imports/test_import_pipeline.py index 7c50c942..20b71692 100644 --- a/tests/imports/test_import_pipeline.py +++ b/tests/imports/test_import_pipeline.py @@ -4,7 +4,7 @@ import types import core.imports.pipeline as import_pipeline import core.imports.paths as import_paths -import core.imports.runtime_state as runtime_state +import core.runtime_state as runtime_state class _Config: @@ -63,14 +63,14 @@ def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch): original_matched_context = dict(runtime_state.matched_downloads_context) original_download_tasks = dict(runtime_state.download_tasks) original_download_batches = dict(runtime_state.download_batches) - original_processed_ids = set(runtime_state._processed_download_ids) - original_post_process_locks = dict(runtime_state._post_process_locks) + original_processed_ids = set(runtime_state.processed_download_ids) + original_post_locks = dict(runtime_state.post_process_locks) runtime_state.matched_downloads_context.clear() runtime_state.download_tasks.clear() runtime_state.download_batches.clear() - runtime_state._processed_download_ids.clear() - runtime_state._post_process_locks.clear() + runtime_state.processed_download_ids.clear() + runtime_state.post_process_locks.clear() runtime = types.SimpleNamespace( automation_engine=None, @@ -122,7 +122,7 @@ def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch): runtime_state.download_tasks.update(original_download_tasks) runtime_state.download_batches.clear() runtime_state.download_batches.update(original_download_batches) - runtime_state._processed_download_ids.clear() - runtime_state._processed_download_ids.update(original_processed_ids) - runtime_state._post_process_locks.clear() - runtime_state._post_process_locks.update(original_post_process_locks) + runtime_state.processed_download_ids.clear() + runtime_state.processed_download_ids.update(original_processed_ids) + runtime_state.post_process_locks.clear() + runtime_state.post_process_locks.update(original_post_locks) diff --git a/tests/test_downloads_pagination.py b/tests/test_downloads_pagination.py index 24340b66..2a9c9f05 100644 --- a/tests/test_downloads_pagination.py +++ b/tests/test_downloads_pagination.py @@ -44,7 +44,7 @@ _install_flask_limiter_stub() from flask import Flask, Blueprint # noqa: E402 from api import downloads as downloads_mod # noqa: E402 -import core.imports.runtime_state as runtime_state # noqa: E402 +import core.runtime_state as runtime_state # noqa: E402 def _make_task(status="downloading", when=None): diff --git a/web_server.py b/web_server.py index 6d7da994..daa7c687 100644 --- a/web_server.py +++ b/web_server.py @@ -133,7 +133,7 @@ from core.imports.staging import ( ) from core.imports.paths import build_final_path_for_track as _build_final_path_for_track from core.metadata_common import get_file_lock -from core.imports.runtime_state import ( +from core.runtime_state import ( activity_feed, activity_feed_lock, add_activity_item, @@ -142,6 +142,7 @@ from core.imports.runtime_state import ( matched_context_lock, matched_downloads_context, mark_task_completed as _core_mark_task_completed, + processed_download_ids, set_activity_toast_emitter, tasks_lock, ) @@ -747,7 +748,7 @@ retag_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="RetagWork # Download Missing Tracks Modal State Management # Thread-safe state tracking for modal download functionality. -# Shared task/batch state now lives in core.imports.runtime_state. +# Shared task/batch state now lives in core.runtime_state. missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker") # Automatic Wishlist / Watchlist Processing Flags @@ -1007,7 +1008,6 @@ session_completed_downloads = 0 session_stats_lock = threading.Lock() batch_locks = {} -_processed_download_ids = set() _orphaned_download_keys = set() _mb_release_cache = {} @@ -1940,7 +1940,7 @@ def get_download_status(): return jsonify({"transfers": []}) try: - global _processed_download_ids + global processed_download_ids transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) all_transfers = [] completed_matched_downloads = [] @@ -1994,7 +1994,7 @@ def get_download_status(): _orphaned_download_keys.discard(context_key) continue - if context_key in _processed_download_ids: + if context_key in processed_download_ids: continue with matched_context_lock: @@ -2025,7 +2025,7 @@ def get_download_status(): _download_retry_attempts[context_key]['count'] += 1 retry_count = _download_retry_attempts[context_key]['count'] if retry_count >= _download_retry_max: - _processed_download_ids.add(context_key) + processed_download_ids.add(context_key) del _download_retry_attempts[context_key] if completed_matched_downloads: def process_completed_downloads(): @@ -2042,7 +2042,7 @@ def get_download_status(): thread = threading.Thread(target=_pp_target, args=_pp_args) thread.daemon = True thread.start() - _processed_download_ids.add(context_key) + processed_download_ids.add(context_key) except Exception as e: print(f"Error starting post-processing thread for {context_key}: {e}") processing_thread = threading.Thread(target=process_completed_downloads) @@ -3967,7 +3967,7 @@ def _update_automation_progress(automation_id, **kwargs): pass # --- Global Matched Downloads Context Management --- -# Shared with core.imports.runtime_state so the refactored pipeline and web +# Shared with core.runtime_state so the refactored pipeline and web # server operate on the same context registry. _orphaned_download_keys = set() # Context keys of downloads abandoned during retry @@ -11755,7 +11755,7 @@ def get_download_status(): return jsonify({"transfers": []}) try: - global _processed_download_ids + global processed_download_ids # Skip slskd API call if Soulseek is already known to be disconnected soulseek_known_down = not _status_cache.get('soulseek', {}).get('connected', True) transfers_data = None @@ -11835,7 +11835,7 @@ def get_download_status(): continue # Skip normal post-processing either way # Skip downloads we've already processed (prevents log spam) - if context_key in _processed_download_ids: + if context_key in processed_download_ids: continue with matched_context_lock: @@ -11865,7 +11865,7 @@ def get_download_status(): _files_claimed_this_cycle.add(_norm_path) logger.info(f"Found completed matched file on disk: {found_path}") completed_matched_downloads.append((context_key, context, found_path)) - # Don't add to _processed_download_ids yet - wait until thread starts successfully + # Don't add to processed_download_ids yet - wait until thread starts successfully # Clean up retry tracking if file was found after retries with _download_retry_lock: @@ -11894,7 +11894,7 @@ def get_download_status(): if retry_count >= _download_retry_max: # Max retries reached, give up logger.error(f"CRITICAL: Could not find '{os.path.basename(filename_from_api)}' after {retry_count} attempts over {elapsed:.1f}s. Giving up.") - _processed_download_ids.add(context_key) + processed_download_ids.add(context_key) # Clean up retry tracking del _download_retry_attempts[context_key] else: @@ -11921,7 +11921,7 @@ def get_download_status(): thread.start() # Only mark as processed AFTER thread starts successfully - _processed_download_ids.add(context_key) + processed_download_ids.add(context_key) logger.info(f"Marked as processed: {context_key}") # DON'T remove context immediately - verification worker needs it @@ -11968,7 +11968,7 @@ def get_download_status(): with matched_context_lock: context = matched_downloads_context.get(context_key) - if context and context_key not in _processed_download_ids: + if context and context_key not in processed_download_ids: download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) found_result = _find_completed_file_robust(download_dir, download.filename) found_path = found_result[0] if found_result and found_result[0] else None @@ -11997,7 +11997,7 @@ def get_download_status(): thread = threading.Thread(target=_st_target, args=_st_args) thread.daemon = True thread.start() - _processed_download_ids.add(_ctx_key) + processed_download_ids.add(_ctx_key) logger.info(f"[{_label}] Marked as processed: {_ctx_key}") except Exception as e: logger.error(f"[{_label}] Error starting post-processing thread for {_ctx_key}: {e}") @@ -12008,7 +12008,7 @@ def get_download_status(): else: # File not found - likely already processed and moved to library # Mark as processed to prevent infinite checking - _processed_download_ids.add(context_key) + processed_download_ids.add(context_key) except Exception as streaming_error: import traceback logger.error(f"Could not fetch YouTube/Tidal downloads for status: {streaming_error}") @@ -18630,7 +18630,7 @@ def _search_track_in_album_context_web(context: dict, spotify_artist: dict) -> d if similarity > threshold: logger.info(f"FOUND: '{track_name}' (track #{track_number}) matches '{clean_track}' (similarity: {similarity:.2f})") - # Classify as album vs single using same logic as _detect_album_info_web + # Classify as album vs single using the shared detect_album_info_web helper ctx_album_type = getattr(album, 'album_type', 'album') or 'album' ctx_total_tracks = getattr(album, 'total_tracks', 1) or 1 ctx_is_album = ( From 8319c6679fbddaf00ca415f0ae630c05950a0b91 Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sat, 25 Apr 2026 13:45:46 +0300 Subject: [PATCH 07/19] Move new metadata helpers into a package - Keep existing metadata_cache and metadata_service at the top level for now - Move the new branch-local metadata helpers under core/metadata - Share MusicBrainz release cache state from core.metadata.source and update import sites --- core/imports/pipeline.py | 6 +- core/metadata/__init__.py | 2 + .../artwork.py} | 0 .../enrichment.py} | 4 +- .../lyrics.py} | 0 .../source.py} | 32 ++++++----- tests/test_metadata_enrichment.py | 6 +- web_server.py | 55 ++++++++----------- 8 files changed, 51 insertions(+), 54 deletions(-) create mode 100644 core/metadata/__init__.py rename core/{metadata_artwork.py => metadata/artwork.py} (100%) rename core/{metadata_enrichment.py => metadata/enrichment.py} (98%) rename core/{metadata_lyrics.py => metadata/lyrics.py} (100%) rename core/{metadata_source.py => metadata/source.py} (97%) diff --git a/core/imports/pipeline.py b/core/imports/pipeline.py index 68429c85..df18b81c 100644 --- a/core/imports/pipeline.py +++ b/core/imports/pipeline.py @@ -51,16 +51,16 @@ from core.runtime_state import ( processed_download_ids, tasks_lock, ) -from core.metadata_artwork import download_cover_art +from core.metadata.artwork import download_cover_art from core.metadata_common import wipe_source_tags -from core.metadata_enrichment import enhance_file_metadata +from core.metadata.enrichment import enhance_file_metadata from core.imports.paths import ( build_final_path_for_track, build_simple_download_destination, docker_resolve_path, ) from core.imports.album_naming import resolve_album_group -from core.metadata_lyrics import generate_lrc_file +from core.metadata.lyrics import generate_lrc_file from database.music_database import get_database from utils.logging_config import get_logger diff --git a/core/metadata/__init__.py b/core/metadata/__init__.py new file mode 100644 index 00000000..1961bd7b --- /dev/null +++ b/core/metadata/__init__.py @@ -0,0 +1,2 @@ +"""Metadata helper package.""" + diff --git a/core/metadata_artwork.py b/core/metadata/artwork.py similarity index 100% rename from core/metadata_artwork.py rename to core/metadata/artwork.py diff --git a/core/metadata_enrichment.py b/core/metadata/enrichment.py similarity index 98% rename from core/metadata_enrichment.py rename to core/metadata/enrichment.py index fe3fd906..0e5d5ec9 100644 --- a/core/metadata_enrichment.py +++ b/core/metadata/enrichment.py @@ -4,7 +4,7 @@ from __future__ import annotations import os -from core.metadata_artwork import embed_album_art_metadata +from core.metadata.artwork import embed_album_art_metadata from core.metadata_common import ( get_config_manager, get_file_lock, @@ -15,7 +15,7 @@ from core.metadata_common import ( strip_all_non_audio_tags, verify_metadata_written, ) -from core.metadata_source import embed_source_ids, extract_source_metadata +from core.metadata.source import embed_source_ids, extract_source_metadata __all__ = [ diff --git a/core/metadata_lyrics.py b/core/metadata/lyrics.py similarity index 100% rename from core/metadata_lyrics.py rename to core/metadata/lyrics.py diff --git a/core/metadata_source.py b/core/metadata/source.py similarity index 97% rename from core/metadata_source.py rename to core/metadata/source.py index 7f4411e9..483150ad 100644 --- a/core/metadata_source.py +++ b/core/metadata/source.py @@ -31,13 +31,17 @@ from core.metadata_common import ( __all__ = [ "extract_source_metadata", "embed_source_ids", + "normalize_album_cache_key", + "mb_release_cache", + "mb_release_cache_lock", + "mb_release_detail_cache", + "mb_release_detail_cache_lock", ] - -_MB_RELEASE_CACHE: Dict[tuple, str] = {} -_MB_RELEASE_CACHE_LOCK = threading.RLock() -_MB_RELEASE_DETAIL_CACHE: Dict[str, Dict[str, Any]] = {} -_MB_RELEASE_DETAIL_CACHE_LOCK = threading.RLock() +mb_release_cache: Dict[tuple, str] = {} +mb_release_cache_lock = threading.RLock() +mb_release_detail_cache: Dict[str, Dict[str, Any]] = {} +mb_release_detail_cache_lock = threading.RLock() _EDITION_PAREN_RE = re.compile( r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' @@ -365,10 +369,10 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), artist_key) rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) - with _MB_RELEASE_CACHE_LOCK: - cached = _MB_RELEASE_CACHE.get(rc_key_norm) + with mb_release_cache_lock: + cached = mb_release_cache.get(rc_key_norm) if cached is None: - cached = _MB_RELEASE_CACHE.get(rc_key_exact) + cached = mb_release_cache.get(rc_key_exact) if cached is not None: pp["release_mbid"] = cached else: @@ -377,21 +381,21 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" except Exception: pp["release_mbid"] = "" - _MB_RELEASE_CACHE[rc_key_norm] = pp["release_mbid"] - _MB_RELEASE_CACHE[rc_key_exact] = pp["release_mbid"] + mb_release_cache[rc_key_norm] = pp["release_mbid"] + mb_release_cache[rc_key_exact] = pp["release_mbid"] if pp["release_mbid"]: pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] if pp["release_mbid"]: - with _MB_RELEASE_DETAIL_CACHE_LOCK: - release_detail = _MB_RELEASE_DETAIL_CACHE.get(pp["release_mbid"]) + with mb_release_detail_cache_lock: + release_detail = mb_release_detail_cache.get(pp["release_mbid"]) if release_detail is None: release_detail = mb_service.mb_client.get_release( pp["release_mbid"], includes=["release-groups", "labels", "media", "artist-credits", "recordings"], ) or {} - with _MB_RELEASE_DETAIL_CACHE_LOCK: - _MB_RELEASE_DETAIL_CACHE[pp["release_mbid"]] = release_detail + with mb_release_detail_cache_lock: + mb_release_detail_cache[pp["release_mbid"]] = release_detail if release_detail: rg = release_detail.get("release-group", {}) if rg.get("id"): diff --git a/tests/test_metadata_enrichment.py b/tests/test_metadata_enrichment.py index 577bda98..35ffcd4b 100644 --- a/tests/test_metadata_enrichment.py +++ b/tests/test_metadata_enrichment.py @@ -1,8 +1,8 @@ import types -from core import metadata_enrichment as me -from core import metadata_artwork as ma -from core import metadata_source as ms +from core.metadata import enrichment as me +from core.metadata import artwork as ma +from core.metadata import source as ms class _Config: diff --git a/web_server.py b/web_server.py index daa7c687..bf7c7d4a 100644 --- a/web_server.py +++ b/web_server.py @@ -133,6 +133,13 @@ from core.imports.staging import ( ) from core.imports.paths import build_final_path_for_track as _build_final_path_for_track from core.metadata_common import get_file_lock +from core.metadata.source import ( + mb_release_cache, + mb_release_cache_lock, + mb_release_detail_cache, + mb_release_detail_cache_lock, + normalize_album_cache_key, +) from core.runtime_state import ( activity_feed, activity_feed_lock, @@ -146,8 +153,7 @@ from core.runtime_state import ( set_activity_toast_emitter, tasks_lock, ) -from core import metadata_enrichment -from core.metadata_source import normalize_album_cache_key +from core.metadata import enrichment as metadata_enrichment from database.music_database import get_database, MusicDatabase from services.sync_service import PlaylistSyncService @@ -1010,11 +1016,6 @@ session_stats_lock = threading.Lock() batch_locks = {} _orphaned_download_keys = set() -_mb_release_cache = {} -_mb_release_cache_lock = threading.Lock() -_mb_release_detail_cache = {} -_mb_release_detail_cache_lock = threading.Lock() - _enrichment_activity_log = {} _idle_since = {} _IDLE_GRACE_SECONDS = 5 @@ -5423,16 +5424,6 @@ album_artists = {} # album_key -> artist_name album_editions = {} # album_key -> "standard" or "deluxe" album_name_cache = {} # album_key -> cached_final_name -# Thread-safe cache for MusicBrainz release MBID lookups. -# Prevents concurrent post-processing threads from getting different -# release IDs for the same album, which causes players like Navidrome -# to split one album into multiple entries. -_mb_release_cache = {} # (album_lower, artist_lower) -> mbid_string_or_empty -_mb_release_cache_lock = threading.Lock() -_mb_release_detail_cache = {} # mbid -> release detail dict from get_release() -_mb_release_detail_cache_lock = threading.Lock() - - # Regexes to strip edition suffixes for cache key normalization. # Prevents Navidrome splitting albums when tracks get different MusicBrainz release IDs. import re as _re @@ -17817,11 +17808,11 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar if _pf_release and _pf_release.get('id'): _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() - with _mb_release_cache_lock: - _mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid - _mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[_pf_mbid] = _pf_release + with mb_release_cache_lock: + mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid + mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid + with mb_release_detail_cache_lock: + mb_release_detail_cache[_pf_mbid] = _pf_release logger.info(f"[Preflight] Pre-cached MB release for '{spotify_album['name']}': " f"'{_pf_release.get('title', '')}' ({_pf_mbid[:8]}...)") except Exception as pf_err: @@ -17958,11 +17949,11 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album): if _pf_release and _pf_release.get('id'): _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() - with _mb_release_cache_lock: - _mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid - _mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[_pf_mbid] = _pf_release + with mb_release_cache_lock: + mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid + mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid + with mb_release_detail_cache_lock: + mb_release_detail_cache[_pf_mbid] = _pf_release logger.info(f"[Preflight] Pre-cached MB release for '{spotify_album['name']}': " f"'{_pf_release.get('title', '')}' ({_pf_mbid[:8]}...)") except Exception as pf_err: @@ -25148,12 +25139,12 @@ def _run_full_missing_tracks_process(batch_id, playlist_id, tracks_json): _artist_key = artist_name_pf.lower().strip() _rc_key_norm = (normalize_album_cache_key(album_name_pf), _artist_key) _rc_key_exact = (album_name_pf.lower().strip(), _artist_key) - with _mb_release_cache_lock: - _mb_release_cache[_rc_key_norm] = release_mbid - _mb_release_cache[_rc_key_exact] = release_mbid + with mb_release_cache_lock: + mb_release_cache[_rc_key_norm] = release_mbid + mb_release_cache[_rc_key_exact] = release_mbid # Also cache the full release detail for tag extraction - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[release_mbid] = release + with mb_release_detail_cache_lock: + mb_release_detail_cache[release_mbid] = release logger.info(f"[Preflight] Pre-cached MB release for '{album_name_pf}': " f"'{release.get('title', '')}' ({release_mbid[:8]}...)") else: From 9656dbd46a4fb2fee9e28223318f5e0b0ed3c00a Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sat, 25 Apr 2026 14:13:43 +0300 Subject: [PATCH 08/19] Thread runtime through metadata enrichment - Pass the live runtime bundle into the shared metadata facade so worker-backed source enrichment can actually run. - Forward runtime from the import pipeline and web-server wrapper into embed_source_ids. - Add a regression test that verifies the runtime object reaches the source-ID embedding path. --- core/imports/pipeline.py | 4 +- core/metadata/enrichment.py | 4 +- core/metadata/source.py | 1340 +++++++++++++++-------------- tests/test_metadata_enrichment.py | 54 ++ web_server.py | 31 +- 5 files changed, 781 insertions(+), 652 deletions(-) diff --git a/core/imports/pipeline.py b/core/imports/pipeline.py index df18b81c..b915042f 100644 --- a/core/imports/pipeline.py +++ b/core/imports/pipeline.py @@ -364,7 +364,7 @@ def post_process_matched_download(context_key, context, file_path, runtime): f"[Metadata Input] Playlist mode - artist: '{artist_context.get('name', 'MISSING')}' " f"(id: {artist_context.get('id', 'MISSING')})" ) - enhance_file_metadata(file_path, context, artist_context, None) + enhance_file_metadata(file_path, context, artist_context, None, runtime=runtime) except Exception as meta_err: import traceback pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") @@ -529,7 +529,7 @@ def post_process_matched_download(context_key, context, file_path, runtime): ) else: logger.info("[Metadata Input] album_info: None (single track)") - enhance_file_metadata(file_path, context, artist_context, album_info) + enhance_file_metadata(file_path, context, artist_context, album_info, runtime=runtime) except Exception as meta_err: import traceback pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") diff --git a/core/metadata/enrichment.py b/core/metadata/enrichment.py index 0e5d5ec9..f44d706a 100644 --- a/core/metadata/enrichment.py +++ b/core/metadata/enrichment.py @@ -25,7 +25,7 @@ __all__ = [ ] -def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: +def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: cfg = get_config_manager() logger_ = get_logger() if cfg.get("metadata_enhancement.enabled", True) is False: @@ -126,7 +126,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf if metadata.get("disc_number"): audio_file["disk"] = [(metadata["disc_number"], 0)] - embed_source_ids(audio_file, metadata, context) + embed_source_ids(audio_file, metadata, context, runtime=runtime) if album_info is not None and metadata.get("musicbrainz_release_id"): album_info["musicbrainz_release_id"] = metadata["musicbrainz_release_id"] diff --git a/core/metadata/source.py b/core/metadata/source.py index 483150ad..0661d11a 100644 --- a/core/metadata/source.py +++ b/core/metadata/source.py @@ -23,10 +23,10 @@ from core.metadata_service import get_itunes_client from database.music_database import get_database from core.metadata_common import ( get_config_manager, - get_logger, get_mutagen_symbols, is_vorbis_like, ) +from utils.logging_config import get_logger as _create_logger __all__ = [ "extract_source_metadata", @@ -42,6 +42,7 @@ mb_release_cache: Dict[tuple, str] = {} mb_release_cache_lock = threading.RLock() mb_release_detail_cache: Dict[str, Dict[str, Any]] = {} mb_release_detail_cache_lock = threading.RLock() +logger = _create_logger("metadata.source") _EDITION_PAREN_RE = re.compile( r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' @@ -63,12 +64,690 @@ def normalize_album_cache_key(album_name: str) -> str: return result.lower().strip() +SOURCE_TAG_CONFIG = { + "SPOTIFY_TRACK_ID": "spotify.tags.track_id", + "SPOTIFY_ARTIST_ID": "spotify.tags.artist_id", + "SPOTIFY_ALBUM_ID": "spotify.tags.album_id", + "ITUNES_TRACK_ID": "itunes.tags.track_id", + "ITUNES_ARTIST_ID": "itunes.tags.artist_id", + "ITUNES_ALBUM_ID": "itunes.tags.album_id", + "MUSICBRAINZ_RECORDING_ID": "musicbrainz.tags.recording_id", + "MUSICBRAINZ_ARTIST_ID": "musicbrainz.tags.artist_id", + "MUSICBRAINZ_RELEASE_ID": "musicbrainz.tags.release_id", + "MUSICBRAINZ_RELEASEGROUPID": "musicbrainz.tags.release_group_id", + "MUSICBRAINZ_ALBUMARTISTID": "musicbrainz.tags.album_artist_id", + "MUSICBRAINZ_RELEASETRACKID": "musicbrainz.tags.release_track_id", + "RELEASETYPE": "musicbrainz.tags.release_type", + "ORIGINALDATE": "musicbrainz.tags.original_date", + "RELEASESTATUS": "musicbrainz.tags.release_status", + "RELEASECOUNTRY": "musicbrainz.tags.release_country", + "BARCODE": "musicbrainz.tags.barcode", + "MEDIA": "musicbrainz.tags.media", + "TOTALDISCS": "musicbrainz.tags.total_discs", + "CATALOGNUMBER": "musicbrainz.tags.catalog_number", + "SCRIPT": "musicbrainz.tags.script", + "ASIN": "musicbrainz.tags.asin", + "DEEZER_TRACK_ID": "deezer.tags.track_id", + "DEEZER_ARTIST_ID": "deezer.tags.artist_id", + "AUDIODB_TRACK_ID": "audiodb.tags.track_id", + "TIDAL_TRACK_ID": "tidal.tags.track_id", + "TIDAL_ARTIST_ID": "tidal.tags.artist_id", + "QOBUZ_TRACK_ID": "qobuz.tags.track_id", + "QOBUZ_ARTIST_ID": "qobuz.tags.artist_id", + "GENIUS_TRACK_ID": "genius.tags.track_id", +} + +DEFAULT_SOURCE_ORDER = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"] + +ID3_TAG_MAP = { + "MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"), + "MUSICBRAINZ_ARTIST_ID": ("TXXX", "MusicBrainz Artist Id"), + "MUSICBRAINZ_RELEASE_ID": ("TXXX", "MusicBrainz Album Id"), + "MUSICBRAINZ_RELEASEGROUPID": ("TXXX", "MusicBrainz Release Group Id"), + "MUSICBRAINZ_ALBUMARTISTID": ("TXXX", "MusicBrainz Album Artist Id"), + "MUSICBRAINZ_RELEASETRACKID": ("TXXX", "MusicBrainz Release Track Id"), + "RELEASETYPE": ("TXXX", "MusicBrainz Album Type"), + "RELEASESTATUS": ("TXXX", "MusicBrainz Album Status"), + "RELEASECOUNTRY": ("TXXX", "MusicBrainz Album Release Country"), + "ORIGINALDATE": ("TDOR", None), + "MEDIA": ("TMED", None), +} + +VORBIS_TAG_MAP = { + "MUSICBRAINZ_RECORDING_ID": "MUSICBRAINZ_TRACKID", + "MUSICBRAINZ_ARTIST_ID": "MUSICBRAINZ_ARTISTID", + "MUSICBRAINZ_RELEASE_ID": "MUSICBRAINZ_ALBUMID", + "MUSICBRAINZ_RELEASEGROUPID": "MUSICBRAINZ_RELEASEGROUPID", + "MUSICBRAINZ_ALBUMARTISTID": "MUSICBRAINZ_ALBUMARTISTID", + "MUSICBRAINZ_RELEASETRACKID": "MUSICBRAINZ_RELEASETRACKID", +} + +MP4_TAG_MAP = { + "MUSICBRAINZ_RECORDING_ID": "MusicBrainz Track Id", + "MUSICBRAINZ_ARTIST_ID": "MusicBrainz Artist Id", + "MUSICBRAINZ_RELEASE_ID": "MusicBrainz Album Id", + "MUSICBRAINZ_RELEASEGROUPID": "MusicBrainz Release Group Id", + "MUSICBRAINZ_ALBUMARTISTID": "MusicBrainz Album Artist Id", + "MUSICBRAINZ_RELEASETRACKID": "MusicBrainz Release Track Id", + "RELEASETYPE": "MusicBrainz Album Type", + "RELEASESTATUS": "MusicBrainz Album Status", + "RELEASECOUNTRY": "MusicBrainz Album Release Country", +} + + +def _tag_enabled(cfg, path: str) -> bool: + return cfg.get(path, True) is not False + + +def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: + if not a or not b: + return False + from difflib import SequenceMatcher + + norm = lambda s: re.sub(r"[^a-z0-9 ]", "", re.sub(r"\(.*?\)", "", s).lower()).strip() + return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold + + +def _collect_source_ids(metadata: dict, cfg) -> dict: + source_ids = {} + source = (metadata.get("source") or "").strip().lower() + if source: + source_tag_names = get_source_tag_names(source) + source_track_id = metadata.get("source_track_id") + source_artist_id = metadata.get("source_artist_id") + source_album_id = metadata.get("source_album_id") + if cfg.get(f"{source}.embed_tags", True) is not False: + if source_tag_names.get("track") and source_track_id: + source_ids[source_tag_names["track"]] = source_track_id + if source_tag_names.get("artist") and source_artist_id: + source_ids[source_tag_names["artist"]] = source_artist_id + if source_tag_names.get("album") and source_album_id: + source_ids[source_tag_names["album"]] = source_album_id + + if not source_ids: + if cfg.get("spotify.embed_tags", True) is not False: + if metadata.get("spotify_track_id"): + source_ids["SPOTIFY_TRACK_ID"] = metadata["spotify_track_id"] + if metadata.get("spotify_artist_id"): + source_ids["SPOTIFY_ARTIST_ID"] = metadata["spotify_artist_id"] + if metadata.get("spotify_album_id"): + source_ids["SPOTIFY_ALBUM_ID"] = metadata["spotify_album_id"] + if cfg.get("itunes.embed_tags", True) is not False: + if metadata.get("itunes_track_id"): + source_ids["ITUNES_TRACK_ID"] = metadata["itunes_track_id"] + if metadata.get("itunes_artist_id"): + source_ids["ITUNES_ARTIST_ID"] = metadata["itunes_artist_id"] + if metadata.get("itunes_album_id"): + source_ids["ITUNES_ALBUM_ID"] = metadata["itunes_album_id"] + + return source_ids + + +def _process_musicbrainz_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: + if cfg.get("musicbrainz.embed_tags", True) is False: + return + if not track_title or not artist_name: + return + + mb_worker = getattr(runtime, "mb_worker", None) + mb_service = mb_worker.mb_service if mb_worker else None + if not mb_service: + return + + try: + result = mb_service.match_recording(track_title, artist_name) + if result and result.get("mbid"): + pp["recording_mbid"] = result["mbid"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"] + details = mb_service.mb_client.get_recording(pp["recording_mbid"], includes=["isrcs", "genres"]) + if details: + isrcs = details.get("isrcs", []) + if isrcs: + pp["isrc"] = isrcs[0] + pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)] + + track_artist_name = metadata.get("artist", "") or artist_name + if ", " in track_artist_name: + track_artist_name = track_artist_name.split(", ")[0] + artist_result = mb_service.match_artist(track_artist_name) + if artist_result and artist_result.get("mbid"): + pp["artist_mbid"] = artist_result["mbid"] + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"] + + album_name_for_mb = metadata.get("album", "") + if album_name_for_mb: + artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() + rc_key_norm = (normalize_album_cache_key(album_name_for_mb), artist_key) + rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) + with mb_release_cache_lock: + cached = mb_release_cache.get(rc_key_norm) + if cached is None: + cached = mb_release_cache.get(rc_key_exact) + if cached is not None: + pp["release_mbid"] = cached + else: + try: + rc_result = mb_service.match_release(album_name_for_mb, artist_name) + pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" + except Exception: + pp["release_mbid"] = "" + mb_release_cache[rc_key_norm] = pp["release_mbid"] + mb_release_cache[rc_key_exact] = pp["release_mbid"] + if pp["release_mbid"]: + pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] + + if pp["release_mbid"]: + with mb_release_detail_cache_lock: + release_detail = mb_release_detail_cache.get(pp["release_mbid"]) + if release_detail is None: + release_detail = mb_service.mb_client.get_release( + pp["release_mbid"], + includes=["release-groups", "labels", "media", "artist-credits", "recordings"], + ) or {} + with mb_release_detail_cache_lock: + mb_release_detail_cache[pp["release_mbid"]] = release_detail + if release_detail: + rg = release_detail.get("release-group", {}) + if rg.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"] + ac = release_detail.get("artist-credit", []) + if ac and isinstance(ac[0], dict): + aa = ac[0].get("artist", {}) + if aa.get("id"): + pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"] + if rg.get("primary-type"): + pp["id_tags"]["RELEASETYPE"] = rg["primary-type"] + if rg.get("first-release-date"): + pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"] + if not pp["release_year"] and len(rg["first-release-date"]) >= 4: + year = rg["first-release-date"][:4] + if year.isdigit(): + pp["release_year"] = year + if release_detail.get("status"): + pp["id_tags"]["RELEASESTATUS"] = release_detail["status"] + if release_detail.get("country"): + pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"] + if release_detail.get("barcode"): + pp["id_tags"]["BARCODE"] = release_detail["barcode"] + media_list = release_detail.get("media", []) + if media_list: + fmt = media_list[0].get("format", "") + if fmt: + pp["id_tags"]["MEDIA"] = fmt + pp["id_tags"]["TOTALDISCS"] = str(len(media_list)) + label_info = release_detail.get("label-info", []) + if label_info and isinstance(label_info[0], dict): + cat = label_info[0].get("catalog-number", "") + if cat: + pp["id_tags"]["CATALOGNUMBER"] = cat + text_rep = release_detail.get("text-representation", {}) + if isinstance(text_rep, dict) and text_rep.get("script"): + pp["id_tags"]["SCRIPT"] = text_rep["script"] + if release_detail.get("asin"): + pp["id_tags"]["ASIN"] = release_detail["asin"] + track_num = metadata.get("track_number") + disc_num = metadata.get("disc_number") or 1 + if track_num and media_list: + try: + track_num_int = int(track_num) + disc_num_int = int(disc_num) + for medium in media_list: + if medium.get("position", 1) == disc_num_int: + for mtrack in (medium.get("tracks") or medium.get("track-list", [])): + if mtrack.get("position") == track_num_int: + if mtrack.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"] + release_recording = mtrack.get("recording", {}) + if release_recording.get("id"): + pp["recording_mbid"] = release_recording["id"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"] + break + break + except (ValueError, TypeError): + pass + except Exception as exc: + logger.error("MusicBrainz lookup failed (non-fatal): %s", exc) + + +def _process_deezer_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: + if cfg.get("deezer.embed_tags", True) is False: + return + if not track_title or not artist_name: + return + + try: + deezer_worker = getattr(runtime, "deezer_worker", None) + dz_client = deezer_worker.client if deezer_worker else None + if not dz_client: + return + dz_result = dz_client.search_track(artist_name, track_title) + if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name): + dz_track_id = dz_result["id"] + pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id) + dz_artist_id = dz_result.get("artist", {}).get("id") + if dz_artist_id: + pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id) + dz_details = dz_client.get_track_details(dz_track_id) + if dz_details: + bpm_val = dz_details.get("bpm") + if bpm_val and bpm_val > 0: + pp["deezer_bpm"] = bpm_val + dz_isrc = dz_details.get("isrc") + if dz_isrc: + pp["deezer_isrc"] = dz_isrc + if not pp["release_year"]: + dz_album = dz_result.get("album", {}) + dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or "" + if len(dz_release) >= 4 and dz_release[:4].isdigit(): + pp["release_year"] = dz_release[:4] + except Exception as exc: + logger.error("Deezer lookup failed (non-fatal): %s", exc) + + +def _process_audiodb_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: + if cfg.get("audiodb.embed_tags", True) is False: + return + if not track_title or not artist_name: + return + + try: + audiodb_worker = getattr(runtime, "audiodb_worker", None) + adb_client = audiodb_worker.client if audiodb_worker else None + if not adb_client: + return + adb_result = adb_client.search_track(artist_name, track_title) + if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name): + adb_track_id = adb_result.get("idTrack") + if adb_track_id: + pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id) + adb_mb_track = adb_result.get("strMusicBrainzID") + if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track + pp["recording_mbid"] = adb_mb_track + adb_mb_artist = adb_result.get("strMusicBrainzArtistID") + if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist + pp["artist_mbid"] = adb_mb_artist + pp["audiodb_mood"] = adb_result.get("strMood") or None + pp["audiodb_style"] = adb_result.get("strStyle") or None + pp["audiodb_genre"] = adb_result.get("strGenre") or None + except Exception as exc: + logger.error("AudioDB lookup failed (non-fatal): %s", exc) + + +def _process_tidal_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: + if cfg.get("tidal.embed_tags", True) is False: + return + if not track_title or not artist_name: + return + + try: + tidal_client = getattr(runtime, "tidal_client", None) + if not (tidal_client and tidal_client.is_authenticated()): + return + td_result = tidal_client.search_track(artist_name, track_title) + if td_result and _names_match(td_result.get("title", ""), track_title): + td_track_id = td_result.get("id") + if td_track_id: + pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id) + td_artist = td_result.get("artist", {}) + if isinstance(td_artist, dict) and td_artist.get("id"): + pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"]) + if td_track_id: + td_details = tidal_client.get_track(str(td_track_id)) + if td_details: + pp["tidal_isrc"] = td_details.get("isrc") + td_copyright = td_details.get("copyright") + if isinstance(td_copyright, dict): + td_copyright = td_copyright.get("text", td_copyright.get("name", "")) + pp["tidal_copyright"] = td_copyright or None + if not pp["release_year"]: + td_album = td_result.get("album", {}) + td_release = "" + if isinstance(td_album, dict): + td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "") + if len(td_release) >= 4 and td_release[:4].isdigit(): + pp["release_year"] = td_release[:4] + except Exception as exc: + logger.error("Tidal lookup failed (non-fatal): %s", exc) + + +def _process_qobuz_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: + if cfg.get("qobuz.embed_tags", True) is False: + return + if not track_title or not artist_name: + return + + try: + qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None) + qz_client = qobuz_worker.client if qobuz_worker else None + if not (qz_client and qz_client.is_authenticated()): + return + qz_result = qz_client.search_track(artist_name, track_title) + if qz_result: + qz_performer = qz_result.get("performer") or {} + if not isinstance(qz_performer, dict): + qz_performer = {} + qz_artist_name = qz_performer.get("name", "") + if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name): + qz_track_id = qz_result.get("id") + if qz_track_id: + pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id) + if qz_performer.get("id"): + pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"]) + qz_isrc = qz_result.get("isrc") + if isinstance(qz_isrc, dict): + qz_isrc = qz_isrc.get("value", qz_isrc.get("id", "")) + if qz_isrc: + pp["qobuz_isrc"] = qz_isrc + qz_copyright = qz_result.get("copyright") + if isinstance(qz_copyright, dict): + qz_copyright = qz_copyright.get("text", qz_copyright.get("name", "")) + if isinstance(qz_copyright, str): + pp["qobuz_copyright"] = qz_copyright + qz_album = qz_result.get("album", {}) + if isinstance(qz_album, dict): + qz_label_info = qz_album.get("label", {}) + if isinstance(qz_label_info, dict) and qz_label_info.get("name"): + pp["qobuz_label"] = qz_label_info["name"] + if not pp["release_year"]: + qz_release = str(qz_album.get("release_date_original", "") or "") + if not qz_release: + qz_ts = qz_album.get("released_at") + if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: + import datetime as _dt + + qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) + if len(qz_release) >= 4 and qz_release[:4].isdigit(): + pp["release_year"] = qz_release[:4] + except Exception as exc: + logger.error("Qobuz lookup failed (non-fatal): %s", exc) + + +def _process_lastfm_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: + if cfg.get("lastfm.embed_tags", True) is False: + return + if not track_title or not artist_name: + return + + try: + lastfm_worker = getattr(runtime, "lastfm_worker", None) + lf_client = lastfm_worker.client if lastfm_worker else None + if not lf_client: + return + lf_result = lf_client.get_track_info(artist_name, track_title) + if lf_result: + lf_url = lf_result.get("url") + if lf_url: + pp["lastfm_url"] = lf_url + lf_toptags = lf_result.get("toptags", {}) + if isinstance(lf_toptags, dict): + tag_list = lf_toptags.get("tag", []) + if isinstance(tag_list, list): + pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")] + elif isinstance(tag_list, dict) and tag_list.get("name"): + pp["lastfm_tags"] = [tag_list["name"]] + except Exception as exc: + logger.error("Last.fm lookup failed (non-fatal): %s", exc) + + +def _process_genius_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: + if cfg.get("genius.embed_tags", True) is False: + return + if not track_title or not artist_name: + return + + try: + import core.genius_client as _genius_module + + if time.time() < _genius_module._rate_limit_until: + logger.info("Genius rate-limited, skipping (non-blocking)") + return + genius_worker = getattr(runtime, "genius_worker", None) + g_client = genius_worker.client if genius_worker else None + if not g_client: + return + g_result = g_client.search_song(artist_name, track_title) + if g_result: + g_id = g_result.get("id") + if g_id: + pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id) + g_url = g_result.get("url") + if g_url: + pp["genius_url"] = g_url + except Exception as exc: + logger.error("Genius lookup failed (non-fatal): %s", exc) + + +def _process_source_enrichment(source_name: str, pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: + if source_name == "musicbrainz": + _process_musicbrainz_source(pp, metadata, cfg, runtime, track_title, artist_name) + elif source_name == "deezer": + _process_deezer_source(pp, metadata, cfg, runtime, track_title, artist_name) + elif source_name == "audiodb": + _process_audiodb_source(pp, metadata, cfg, runtime, track_title, artist_name) + elif source_name == "tidal": + _process_tidal_source(pp, metadata, cfg, runtime, track_title, artist_name) + elif source_name == "qobuz": + _process_qobuz_source(pp, metadata, cfg, runtime, track_title, artist_name) + elif source_name == "lastfm": + _process_lastfm_source(pp, metadata, cfg, runtime, track_title, artist_name) + elif source_name == "genius": + _process_genius_source(pp, metadata, cfg, runtime, track_title, artist_name) + + +def _write_embedded_metadata(audio_file, metadata: dict, pp: dict, cfg, symbols): + filtered_tags: Dict[str, str] = {} + for tag_name, value in pp["id_tags"].items(): + config_path = SOURCE_TAG_CONFIG.get(tag_name) + if config_path and not _tag_enabled(cfg, config_path): + continue + filtered_tags[tag_name] = value + + written = [] + release_year = pp["release_year"] + + if isinstance(audio_file.tags, symbols.ID3): + for tag_name, value in filtered_tags.items(): + spec = ID3_TAG_MAP.get(tag_name) + if spec: + frame_type, desc = spec + if frame_type == "UFID": + audio_file.tags.add(symbols.UFID(owner=desc, data=str(value).encode("ascii"))) + written.append(f"UFID:{desc}") + elif frame_type == "TDOR": + audio_file.tags.add(symbols.TDOR(encoding=3, text=[value])) + written.append("TDOR") + elif frame_type == "TMED": + audio_file.tags.add(symbols.TMED(encoding=3, text=[value])) + written.append("TMED") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=desc, text=[value])) + written.append(f"TXXX:{desc}") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=tag_name, text=[str(value)])) + written.append(f"TXXX:{tag_name}") + elif isinstance(audio_file, symbols.MP4): + for tag_name, value in filtered_tags.items(): + key = f"----:com.apple.iTunes:{MP4_TAG_MAP.get(tag_name, tag_name)}" + audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))] + written.append(key) + elif is_vorbis_like(audio_file, symbols): + for tag_name, value in filtered_tags.items(): + audio_file[VORBIS_TAG_MAP.get(tag_name, tag_name)] = [str(value)] + written.append(VORBIS_TAG_MAP.get(tag_name, tag_name)) + + if written: + logger.info("Embedded IDs: %s", ", ".join(written)) + + if release_year and not metadata.get("date"): + metadata["date"] = release_year + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year])) + elif is_vorbis_like(audio_file, symbols): + audio_file["date"] = [release_year] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9day"] = [release_year] + logger.info("Date tag: %s", release_year) + + if _tag_enabled(cfg, "deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0: + bpm_int = int(pp["deezer_bpm"]) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)])) + elif is_vorbis_like(audio_file, symbols): + audio_file["BPM"] = [str(bpm_int)] + elif isinstance(audio_file, symbols.MP4): + audio_file["tmpo"] = [bpm_int] + logger.info("BPM: %s", bpm_int) + + if _tag_enabled(cfg, "audiodb.tags.mood") and pp["audiodb_mood"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]])) + elif is_vorbis_like(audio_file, symbols): + audio_file["MOOD"] = [pp["audiodb_mood"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))] + + if _tag_enabled(cfg, "audiodb.tags.style") and pp["audiodb_style"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]])) + elif is_vorbis_like(audio_file, symbols): + audio_file["STYLE"] = [pp["audiodb_style"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))] + + if _tag_enabled(cfg, "metadata_enhancement.tags.genre_merge"): + enrichment_genres = [] + if _tag_enabled(cfg, "musicbrainz.tags.genres"): + enrichment_genres += pp["mb_genres"] + if pp["audiodb_genre"] and _tag_enabled(cfg, "audiodb.tags.genre"): + enrichment_genres.append(pp["audiodb_genre"]) + if _tag_enabled(cfg, "lastfm.tags.genres"): + enrichment_genres += pp["lastfm_tags"] + if enrichment_genres: + from core.genre_filter import filter_genres as _filter_genres + + enrichment_genres = _filter_genres(enrichment_genres, cfg) + source_genres = [g.strip() for g in str(metadata.get("genre", "")).split(",") if g.strip()] + seen = set() + merged = [] + for genre in source_genres + enrichment_genres: + key = genre.strip().lower() + if key and key not in seen: + seen.add(key) + merged.append(genre.strip().title()) + if len(merged) >= 5: + break + if merged: + genre_string = ", ".join(merged) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string])) + elif is_vorbis_like(audio_file, symbols): + audio_file["GENRE"] = [genre_string] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9gen"] = [genre_string] + logger.info("Genres merged: %s", genre_string) + + isrc_candidates = [] + if pp["isrc"] and _tag_enabled(cfg, "musicbrainz.tags.isrc"): + isrc_candidates.append(("MusicBrainz", pp["isrc"])) + if pp["deezer_isrc"] and _tag_enabled(cfg, "deezer.tags.isrc"): + isrc_candidates.append(("Deezer", pp["deezer_isrc"])) + if pp["tidal_isrc"] and _tag_enabled(cfg, "tidal.tags.isrc"): + isrc_candidates.append(("Tidal", pp["tidal_isrc"])) + if pp["qobuz_isrc"] and _tag_enabled(cfg, "qobuz.tags.isrc"): + isrc_candidates.append(("Qobuz", pp["qobuz_isrc"])) + if isrc_candidates: + isrc_source, final_isrc = isrc_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc])) + elif is_vorbis_like(audio_file, symbols): + audio_file["ISRC"] = [final_isrc] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))] + logger.info("ISRC (%s): %s", isrc_source, final_isrc) + + copyright_candidates = [] + if pp["tidal_copyright"] and _tag_enabled(cfg, "tidal.tags.copyright"): + copyright_candidates.append(("Tidal", pp["tidal_copyright"])) + if pp["qobuz_copyright"] and _tag_enabled(cfg, "qobuz.tags.copyright"): + copyright_candidates.append(("Qobuz", pp["qobuz_copyright"])) + if copyright_candidates: + copyright_source, final_copyright = copyright_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright])) + elif is_vorbis_like(audio_file, symbols): + audio_file["COPYRIGHT"] = [final_copyright] + elif isinstance(audio_file, symbols.MP4): + audio_file["cprt"] = [final_copyright] + logger.info("Copyright (%s): %s", copyright_source, final_copyright[:60]) + + if _tag_enabled(cfg, "qobuz.tags.label") and pp["qobuz_label"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]])) + elif is_vorbis_like(audio_file, symbols): + audio_file["LABEL"] = [pp["qobuz_label"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))] + + if _tag_enabled(cfg, "lastfm.tags.url") and pp["lastfm_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]])) + elif is_vorbis_like(audio_file, symbols): + audio_file["LASTFM_URL"] = [pp["lastfm_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))] + + if _tag_enabled(cfg, "genius.tags.url") and pp["genius_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]])) + elif is_vorbis_like(audio_file, symbols): + audio_file["GENIUS_URL"] = [pp["genius_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))] + + return release_year + + +def _update_album_year_in_database(db, metadata: dict, release_year) -> None: + if db is None: + return + + try: + album_name_for_db = metadata.get("album", "") + album_artist_for_db = metadata.get("album_artist", "") or metadata.get("artist", "") + if album_name_for_db and album_artist_for_db: + conn = db._get_connection() + try: + cursor = conn.cursor() + cursor.execute( + """ + UPDATE albums SET year = ? + WHERE (year IS NULL OR year = 0) + AND id IN ( + SELECT al.id FROM albums al + JOIN artists ar ON ar.id = al.artist_id + WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) + ) + """, + (int(release_year), album_name_for_db, album_artist_for_db), + ) + if cursor.rowcount > 0: + conn.commit() + logger.info("Updated album year to %s in database", release_year) + else: + conn.rollback() + finally: + conn.close() + except Exception as exc: + logger.error("Could not update album year in DB: %s", exc) + + def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> dict: if album_info is None: album_info = {} cfg = get_config_manager() - logger_ = get_logger() context = normalize_import_context(context) original_search = get_import_original_search(context) album_ctx = get_import_context_album(context) @@ -91,11 +770,11 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di metadata["title"] = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "")) if original_search.get("clean_title"): - logger_.info("Metadata: Using clean title: '%s'", metadata["title"]) + logger.info("Metadata: Using clean title: '%s'", metadata["title"]) elif album_info.get("clean_track_name"): - logger_.info("Metadata: Using album info clean name: '%s'", metadata["title"]) + logger.info("Metadata: Using album info clean name: '%s'", metadata["title"]) else: - logger_.warning("Metadata: Using original title as fallback: '%s'", metadata["title"]) + logger.warning("Metadata: Using original title as fallback: '%s'", metadata["title"]) artists = original_search.get("artists") if isinstance(artists, list) and artists: @@ -108,10 +787,10 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di else: all_artists.append(str(artist_item)) metadata["artist"] = ", ".join(all_artists) - logger_.info("Metadata: Using all artists: '%s'", metadata["artist"]) + logger.info("Metadata: Using all artists: '%s'", metadata["artist"]) else: metadata["artist"] = artist_dict.get("name", "") or get_import_clean_artist(context) - logger_.info("Metadata: Using primary artist: '%s'", metadata["artist"]) + logger.info("Metadata: Using primary artist: '%s'", metadata["artist"]) raw_album_artist = artist_dict.get("name", "") or metadata["artist"] track_info_ctx = track_info or {} @@ -157,10 +836,10 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di metadata["album"] = album_info.get("album_name", "Unknown Album") metadata["track_number"] = album_info.get("track_number", 1) metadata["total_tracks"] = album_ctx.get("total_tracks", 1) if album_ctx else 1 - logger_.info("[METADATA] Album track - track_number: %s, album: %s", metadata["track_number"], metadata["album"]) + logger.info("[METADATA] Album track - track_number: %s, album: %s", metadata["track_number"], metadata["album"]) else: if album_ctx and album_ctx.get("name"): - logger_.info("[SAFEGUARD] Using album context name instead of track title for album metadata") + logger.info("[SAFEGUARD] Using album context name instead of track title for album metadata") metadata["album"] = album_ctx["name"] metadata["track_number"] = album_info.get("track_number", 1) if album_info else 1 metadata["total_tracks"] = album_ctx.get("total_tracks", 1) @@ -193,7 +872,7 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di album_image = first_image.get("url") if isinstance(first_image, dict) else None metadata["album_art_url"] = album_image - logger_.info( + logger.info( "[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | track=%s/%s | disc=%s", metadata.get("title"), metadata.get("artist"), @@ -209,87 +888,13 @@ def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> di def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=None): cfg = get_config_manager() - logger_ = get_logger() symbols = get_mutagen_symbols() if not symbols: return try: - tag_config = { - "SPOTIFY_TRACK_ID": "spotify.tags.track_id", - "SPOTIFY_ARTIST_ID": "spotify.tags.artist_id", - "SPOTIFY_ALBUM_ID": "spotify.tags.album_id", - "ITUNES_TRACK_ID": "itunes.tags.track_id", - "ITUNES_ARTIST_ID": "itunes.tags.artist_id", - "ITUNES_ALBUM_ID": "itunes.tags.album_id", - "MUSICBRAINZ_RECORDING_ID": "musicbrainz.tags.recording_id", - "MUSICBRAINZ_ARTIST_ID": "musicbrainz.tags.artist_id", - "MUSICBRAINZ_RELEASE_ID": "musicbrainz.tags.release_id", - "MUSICBRAINZ_RELEASEGROUPID": "musicbrainz.tags.release_group_id", - "MUSICBRAINZ_ALBUMARTISTID": "musicbrainz.tags.album_artist_id", - "MUSICBRAINZ_RELEASETRACKID": "musicbrainz.tags.release_track_id", - "RELEASETYPE": "musicbrainz.tags.release_type", - "ORIGINALDATE": "musicbrainz.tags.original_date", - "RELEASESTATUS": "musicbrainz.tags.release_status", - "RELEASECOUNTRY": "musicbrainz.tags.release_country", - "BARCODE": "musicbrainz.tags.barcode", - "MEDIA": "musicbrainz.tags.media", - "TOTALDISCS": "musicbrainz.tags.total_discs", - "CATALOGNUMBER": "musicbrainz.tags.catalog_number", - "SCRIPT": "musicbrainz.tags.script", - "ASIN": "musicbrainz.tags.asin", - "DEEZER_TRACK_ID": "deezer.tags.track_id", - "DEEZER_ARTIST_ID": "deezer.tags.artist_id", - "AUDIODB_TRACK_ID": "audiodb.tags.track_id", - "TIDAL_TRACK_ID": "tidal.tags.track_id", - "TIDAL_ARTIST_ID": "tidal.tags.artist_id", - "QOBUZ_TRACK_ID": "qobuz.tags.track_id", - "QOBUZ_ARTIST_ID": "qobuz.tags.artist_id", - "GENIUS_TRACK_ID": "genius.tags.track_id", - } - - def _tag_enabled(path: str) -> bool: - return cfg.get(path, True) is not False - - def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: - if not a or not b: - return False - from difflib import SequenceMatcher - - norm = lambda s: re.sub(r"[^a-z0-9 ]", "", re.sub(r"\(.*?\)", "", s).lower()).strip() - return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold - context = normalize_import_context(context) - source = (metadata.get("source") or "").strip().lower() - source_ids = {} - if source: - source_tag_names = get_source_tag_names(source) - source_track_id = metadata.get("source_track_id") - source_artist_id = metadata.get("source_artist_id") - source_album_id = metadata.get("source_album_id") - if cfg.get(f"{source}.embed_tags", True) is not False: - if source_tag_names.get("track") and source_track_id: - source_ids[source_tag_names["track"]] = source_track_id - if source_tag_names.get("artist") and source_artist_id: - source_ids[source_tag_names["artist"]] = source_artist_id - if source_tag_names.get("album") and source_album_id: - source_ids[source_tag_names["album"]] = source_album_id - - if not source_ids: - if cfg.get("spotify.embed_tags", True) is not False: - if metadata.get("spotify_track_id"): - source_ids["SPOTIFY_TRACK_ID"] = metadata["spotify_track_id"] - if metadata.get("spotify_artist_id"): - source_ids["SPOTIFY_ARTIST_ID"] = metadata["spotify_artist_id"] - if metadata.get("spotify_album_id"): - source_ids["SPOTIFY_ALBUM_ID"] = metadata["spotify_album_id"] - if cfg.get("itunes.embed_tags", True) is not False: - if metadata.get("itunes_track_id"): - source_ids["ITUNES_TRACK_ID"] = metadata["itunes_track_id"] - if metadata.get("itunes_artist_id"): - source_ids["ITUNES_ARTIST_ID"] = metadata["itunes_artist_id"] - if metadata.get("itunes_album_id"): - source_ids["ITUNES_ALBUM_ID"] = metadata["itunes_album_id"] + source_ids = _collect_source_ids(metadata, cfg) track_title = metadata.get("title", "") artist_name = metadata.get("album_artist", "") or metadata.get("artist", "") @@ -330,576 +935,21 @@ def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=N source_order = cfg.get("metadata_enhancement.post_process_order", None) if not isinstance(source_order, list) or not source_order: - source_order = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"] + source_order = DEFAULT_SOURCE_ORDER db = get_database() for source_name in source_order: - if source_name == "musicbrainz": - if cfg.get("musicbrainz.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - mb_worker = getattr(runtime, "mb_worker", None) - mb_service = mb_worker.mb_service if mb_worker else None - if not mb_service: - continue - try: - result = mb_service.match_recording(track_title, artist_name) - if result and result.get("mbid"): - pp["recording_mbid"] = result["mbid"] - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"] - details = mb_service.mb_client.get_recording(pp["recording_mbid"], includes=["isrcs", "genres"]) - if details: - isrcs = details.get("isrcs", []) - if isrcs: - pp["isrc"] = isrcs[0] - pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)] - - track_artist_name = metadata.get("artist", "") or artist_name - if ", " in track_artist_name: - track_artist_name = track_artist_name.split(", ")[0] - artist_result = mb_service.match_artist(track_artist_name) - if artist_result and artist_result.get("mbid"): - pp["artist_mbid"] = artist_result["mbid"] - pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"] - - album_name_for_mb = metadata.get("album", "") - if album_name_for_mb: - artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() - rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), artist_key) - rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) - with mb_release_cache_lock: - cached = mb_release_cache.get(rc_key_norm) - if cached is None: - cached = mb_release_cache.get(rc_key_exact) - if cached is not None: - pp["release_mbid"] = cached - else: - try: - rc_result = mb_service.match_release(album_name_for_mb, artist_name) - pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" - except Exception: - pp["release_mbid"] = "" - mb_release_cache[rc_key_norm] = pp["release_mbid"] - mb_release_cache[rc_key_exact] = pp["release_mbid"] - if pp["release_mbid"]: - pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] - - if pp["release_mbid"]: - with mb_release_detail_cache_lock: - release_detail = mb_release_detail_cache.get(pp["release_mbid"]) - if release_detail is None: - release_detail = mb_service.mb_client.get_release( - pp["release_mbid"], - includes=["release-groups", "labels", "media", "artist-credits", "recordings"], - ) or {} - with mb_release_detail_cache_lock: - mb_release_detail_cache[pp["release_mbid"]] = release_detail - if release_detail: - rg = release_detail.get("release-group", {}) - if rg.get("id"): - pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"] - ac = release_detail.get("artist-credit", []) - if ac and isinstance(ac[0], dict): - aa = ac[0].get("artist", {}) - if aa.get("id"): - pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"] - if rg.get("primary-type"): - pp["id_tags"]["RELEASETYPE"] = rg["primary-type"] - if rg.get("first-release-date"): - pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"] - if not pp["release_year"] and len(rg["first-release-date"]) >= 4: - year = rg["first-release-date"][:4] - if year.isdigit(): - pp["release_year"] = year - if release_detail.get("status"): - pp["id_tags"]["RELEASESTATUS"] = release_detail["status"] - if release_detail.get("country"): - pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"] - if release_detail.get("barcode"): - pp["id_tags"]["BARCODE"] = release_detail["barcode"] - media_list = release_detail.get("media", []) - if media_list: - fmt = media_list[0].get("format", "") - if fmt: - pp["id_tags"]["MEDIA"] = fmt - pp["id_tags"]["TOTALDISCS"] = str(len(media_list)) - label_info = release_detail.get("label-info", []) - if label_info and isinstance(label_info[0], dict): - cat = label_info[0].get("catalog-number", "") - if cat: - pp["id_tags"]["CATALOGNUMBER"] = cat - text_rep = release_detail.get("text-representation", {}) - if isinstance(text_rep, dict) and text_rep.get("script"): - pp["id_tags"]["SCRIPT"] = text_rep["script"] - if release_detail.get("asin"): - pp["id_tags"]["ASIN"] = release_detail["asin"] - track_num = metadata.get("track_number") - disc_num = metadata.get("disc_number") or 1 - if track_num and media_list: - try: - track_num_int = int(track_num) - disc_num_int = int(disc_num) - for medium in media_list: - if medium.get("position", 1) == disc_num_int: - for mtrack in (medium.get("tracks") or medium.get("track-list", [])): - if mtrack.get("position") == track_num_int: - if mtrack.get("id"): - pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"] - release_recording = mtrack.get("recording", {}) - if release_recording.get("id"): - pp["recording_mbid"] = release_recording["id"] - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"] - break - break - except (ValueError, TypeError): - pass - except Exception as exc: - logger_.error("MusicBrainz lookup failed (non-fatal): %s", exc) - continue - - if source_name == "deezer": - if cfg.get("deezer.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - deezer_worker = getattr(runtime, "deezer_worker", None) - dz_client = deezer_worker.client if deezer_worker else None - if not dz_client: - continue - dz_result = dz_client.search_track(artist_name, track_title) - if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name): - dz_track_id = dz_result["id"] - pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id) - dz_artist_id = dz_result.get("artist", {}).get("id") - if dz_artist_id: - pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id) - dz_details = dz_client.get_track_details(dz_track_id) - if dz_details: - bpm_val = dz_details.get("bpm") - if bpm_val and bpm_val > 0: - pp["deezer_bpm"] = bpm_val - dz_isrc = dz_details.get("isrc") - if dz_isrc: - pp["deezer_isrc"] = dz_isrc - if not pp["release_year"]: - dz_album = dz_result.get("album", {}) - dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or "" - if len(dz_release) >= 4 and dz_release[:4].isdigit(): - pp["release_year"] = dz_release[:4] - except Exception as exc: - logger_.error("Deezer lookup failed (non-fatal): %s", exc) - continue - - if source_name == "audiodb": - if cfg.get("audiodb.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - audiodb_worker = getattr(runtime, "audiodb_worker", None) - adb_client = audiodb_worker.client if audiodb_worker else None - if not adb_client: - continue - adb_result = adb_client.search_track(artist_name, track_title) - if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name): - adb_track_id = adb_result.get("idTrack") - if adb_track_id: - pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id) - adb_mb_track = adb_result.get("strMusicBrainzID") - if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]: - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track - pp["recording_mbid"] = adb_mb_track - adb_mb_artist = adb_result.get("strMusicBrainzArtistID") - if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]: - pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist - pp["artist_mbid"] = adb_mb_artist - pp["audiodb_mood"] = adb_result.get("strMood") or None - pp["audiodb_style"] = adb_result.get("strStyle") or None - pp["audiodb_genre"] = adb_result.get("strGenre") or None - except Exception as exc: - logger_.error("AudioDB lookup failed (non-fatal): %s", exc) - continue - - if source_name == "tidal": - if cfg.get("tidal.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - tidal_client = getattr(runtime, "tidal_client", None) - if not (tidal_client and tidal_client.is_authenticated()): - continue - td_result = tidal_client.search_track(artist_name, track_title) - if td_result and _names_match(td_result.get("title", ""), track_title): - td_track_id = td_result.get("id") - if td_track_id: - pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id) - td_artist = td_result.get("artist", {}) - if isinstance(td_artist, dict) and td_artist.get("id"): - pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"]) - if td_track_id: - td_details = tidal_client.get_track(str(td_track_id)) - if td_details: - pp["tidal_isrc"] = td_details.get("isrc") - td_copyright = td_details.get("copyright") - if isinstance(td_copyright, dict): - td_copyright = td_copyright.get("text", td_copyright.get("name", "")) - pp["tidal_copyright"] = td_copyright or None - if not pp["release_year"]: - td_album = td_result.get("album", {}) - td_release = "" - if isinstance(td_album, dict): - td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "") - if len(td_release) >= 4 and td_release[:4].isdigit(): - pp["release_year"] = td_release[:4] - except Exception as exc: - logger_.error("Tidal lookup failed (non-fatal): %s", exc) - continue - - if source_name == "qobuz": - if cfg.get("qobuz.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None) - qz_client = qobuz_worker.client if qobuz_worker else None - if not (qz_client and qz_client.is_authenticated()): - continue - qz_result = qz_client.search_track(artist_name, track_title) - if qz_result: - qz_performer = qz_result.get("performer") or {} - if not isinstance(qz_performer, dict): - qz_performer = {} - qz_artist_name = qz_performer.get("name", "") - if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name): - qz_track_id = qz_result.get("id") - if qz_track_id: - pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id) - if qz_performer.get("id"): - pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"]) - qz_isrc = qz_result.get("isrc") - if isinstance(qz_isrc, dict): - qz_isrc = qz_isrc.get("value", qz_isrc.get("id", "")) - if qz_isrc: - pp["qobuz_isrc"] = qz_isrc - qz_copyright = qz_result.get("copyright") - if isinstance(qz_copyright, dict): - qz_copyright = qz_copyright.get("text", qz_copyright.get("name", "")) - if isinstance(qz_copyright, str): - pp["qobuz_copyright"] = qz_copyright - qz_album = qz_result.get("album", {}) - if isinstance(qz_album, dict): - qz_label_info = qz_album.get("label", {}) - if isinstance(qz_label_info, dict) and qz_label_info.get("name"): - pp["qobuz_label"] = qz_label_info["name"] - if not pp["release_year"]: - qz_release = str(qz_album.get("release_date_original", "") or "") - if not qz_release: - qz_ts = qz_album.get("released_at") - if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: - import datetime as _dt - qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) - if len(qz_release) >= 4 and qz_release[:4].isdigit(): - pp["release_year"] = qz_release[:4] - except Exception as exc: - logger_.error("Qobuz lookup failed (non-fatal): %s", exc) - continue - - if source_name == "lastfm": - if cfg.get("lastfm.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - lastfm_worker = getattr(runtime, "lastfm_worker", None) - lf_client = lastfm_worker.client if lastfm_worker else None - if not lf_client: - continue - lf_result = lf_client.get_track_info(artist_name, track_title) - if lf_result: - lf_url = lf_result.get("url") - if lf_url: - pp["lastfm_url"] = lf_url - lf_toptags = lf_result.get("toptags", {}) - if isinstance(lf_toptags, dict): - tag_list = lf_toptags.get("tag", []) - if isinstance(tag_list, list): - pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")] - elif isinstance(tag_list, dict) and tag_list.get("name"): - pp["lastfm_tags"] = [tag_list["name"]] - except Exception as exc: - logger_.error("Last.fm lookup failed (non-fatal): %s", exc) - continue - - if source_name == "genius": - if cfg.get("genius.embed_tags", True) is False: - continue - if not track_title or not artist_name: - continue - try: - import core.genius_client as _genius_module - - if time.time() < _genius_module._rate_limit_until: - logger_.info("Genius rate-limited, skipping (non-blocking)") - continue - genius_worker = getattr(runtime, "genius_worker", None) - g_client = genius_worker.client if genius_worker else None - if not g_client: - continue - g_result = g_client.search_song(artist_name, track_title) - if g_result: - g_id = g_result.get("id") - if g_id: - pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id) - g_url = g_result.get("url") - if g_url: - pp["genius_url"] = g_url - except Exception as exc: - logger_.error("Genius lookup failed (non-fatal): %s", exc) - continue + _process_source_enrichment(source_name, pp, metadata, cfg, runtime, track_title, artist_name) if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["audiodb_mood"] and not pp["audiodb_style"]: return - filtered_tags: Dict[str, str] = {} - for tag_name, value in pp["id_tags"].items(): - config_path = tag_config.get(tag_name) - if config_path and not _tag_enabled(config_path): - continue - filtered_tags[tag_name] = value - - written = [] - id3_tag_map = { - "MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"), - "MUSICBRAINZ_ARTIST_ID": ("TXXX", "MusicBrainz Artist Id"), - "MUSICBRAINZ_RELEASE_ID": ("TXXX", "MusicBrainz Album Id"), - "MUSICBRAINZ_RELEASEGROUPID": ("TXXX", "MusicBrainz Release Group Id"), - "MUSICBRAINZ_ALBUMARTISTID": ("TXXX", "MusicBrainz Album Artist Id"), - "MUSICBRAINZ_RELEASETRACKID": ("TXXX", "MusicBrainz Release Track Id"), - "RELEASETYPE": ("TXXX", "MusicBrainz Album Type"), - "RELEASESTATUS": ("TXXX", "MusicBrainz Album Status"), - "RELEASECOUNTRY": ("TXXX", "MusicBrainz Album Release Country"), - "ORIGINALDATE": ("TDOR", None), - "MEDIA": ("TMED", None), - } - vorbis_tag_map = { - "MUSICBRAINZ_RECORDING_ID": "MUSICBRAINZ_TRACKID", - "MUSICBRAINZ_ARTIST_ID": "MUSICBRAINZ_ARTISTID", - "MUSICBRAINZ_RELEASE_ID": "MUSICBRAINZ_ALBUMID", - "MUSICBRAINZ_RELEASEGROUPID": "MUSICBRAINZ_RELEASEGROUPID", - "MUSICBRAINZ_ALBUMARTISTID": "MUSICBRAINZ_ALBUMARTISTID", - "MUSICBRAINZ_RELEASETRACKID": "MUSICBRAINZ_RELEASETRACKID", - } - mp4_tag_map = { - "MUSICBRAINZ_RECORDING_ID": "MusicBrainz Track Id", - "MUSICBRAINZ_ARTIST_ID": "MusicBrainz Artist Id", - "MUSICBRAINZ_RELEASE_ID": "MusicBrainz Album Id", - "MUSICBRAINZ_RELEASEGROUPID": "MusicBrainz Release Group Id", - "MUSICBRAINZ_ALBUMARTISTID": "MusicBrainz Album Artist Id", - "MUSICBRAINZ_RELEASETRACKID": "MusicBrainz Release Track Id", - "RELEASETYPE": "MusicBrainz Album Type", - "RELEASESTATUS": "MusicBrainz Album Status", - "RELEASECOUNTRY": "MusicBrainz Album Release Country", - } - - if isinstance(audio_file.tags, symbols.ID3): - for tag_name, value in filtered_tags.items(): - spec = id3_tag_map.get(tag_name) - if spec: - frame_type, desc = spec - if frame_type == "UFID": - audio_file.tags.add(symbols.UFID(owner=desc, data=str(value).encode("ascii"))) - written.append(f"UFID:{desc}") - elif frame_type == "TDOR": - audio_file.tags.add(symbols.TDOR(encoding=3, text=[value])) - written.append("TDOR") - elif frame_type == "TMED": - audio_file.tags.add(symbols.TMED(encoding=3, text=[value])) - written.append("TMED") - else: - audio_file.tags.add(symbols.TXXX(encoding=3, desc=desc, text=[value])) - written.append(f"TXXX:{desc}") - else: - audio_file.tags.add(symbols.TXXX(encoding=3, desc=tag_name, text=[str(value)])) - written.append(f"TXXX:{tag_name}") - elif isinstance(audio_file, symbols.MP4): - # Keep the dedicated MP4 path last so the same tag maps can be reused. - for tag_name, value in filtered_tags.items(): - key = f"----:com.apple.iTunes:{mp4_tag_map.get(tag_name, tag_name)}" - audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))] - written.append(key) - elif is_vorbis_like(audio_file, symbols): - for tag_name, value in filtered_tags.items(): - audio_file[vorbis_tag_map.get(tag_name, tag_name)] = [str(value)] - written.append(vorbis_tag_map.get(tag_name, tag_name)) - - if written: - logger_.info("Embedded IDs: %s", ", ".join(written)) - - release_year = pp["release_year"] - needs_date_tag = bool(release_year and not metadata.get("date")) - if needs_date_tag: - metadata["date"] = release_year - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year])) - elif is_vorbis_like(audio_file, symbols): - audio_file["date"] = [release_year] - elif isinstance(audio_file, symbols.MP4): - audio_file["\xa9day"] = [release_year] - logger_.info("Date tag: %s", release_year) - - if _tag_enabled("deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0: - bpm_int = int(pp["deezer_bpm"]) - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)])) - elif is_vorbis_like(audio_file, symbols): - audio_file["BPM"] = [str(bpm_int)] - elif isinstance(audio_file, symbols.MP4): - audio_file["tmpo"] = [bpm_int] - logger_.info("BPM: %s", bpm_int) - - if _tag_enabled("audiodb.tags.mood") and pp["audiodb_mood"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]])) - elif is_vorbis_like(audio_file, symbols): - audio_file["MOOD"] = [pp["audiodb_mood"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))] - - if _tag_enabled("audiodb.tags.style") and pp["audiodb_style"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]])) - elif is_vorbis_like(audio_file, symbols): - audio_file["STYLE"] = [pp["audiodb_style"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))] - - if _tag_enabled("metadata_enhancement.tags.genre_merge"): - enrichment_genres = [] - if _tag_enabled("musicbrainz.tags.genres"): - enrichment_genres += pp["mb_genres"] - if pp["audiodb_genre"] and _tag_enabled("audiodb.tags.genre"): - enrichment_genres.append(pp["audiodb_genre"]) - if _tag_enabled("lastfm.tags.genres"): - enrichment_genres += pp["lastfm_tags"] - if enrichment_genres: - from core.genre_filter import filter_genres as _filter_genres - - enrichment_genres = _filter_genres(enrichment_genres, cfg) - source_genres = [g.strip() for g in str(metadata.get("genre", "")).split(",") if g.strip()] - seen = set() - merged = [] - for genre in source_genres + enrichment_genres: - key = genre.strip().lower() - if key and key not in seen: - seen.add(key) - merged.append(genre.strip().title()) - if len(merged) >= 5: - break - if merged: - genre_string = ", ".join(merged) - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string])) - elif is_vorbis_like(audio_file, symbols): - audio_file["GENRE"] = [genre_string] - elif isinstance(audio_file, symbols.MP4): - audio_file["\xa9gen"] = [genre_string] - logger_.info("Genres merged: %s", genre_string) - - isrc_candidates = [] - if pp["isrc"] and _tag_enabled("musicbrainz.tags.isrc"): - isrc_candidates.append(("MusicBrainz", pp["isrc"])) - if pp["deezer_isrc"] and _tag_enabled("deezer.tags.isrc"): - isrc_candidates.append(("Deezer", pp["deezer_isrc"])) - if pp["tidal_isrc"] and _tag_enabled("tidal.tags.isrc"): - isrc_candidates.append(("Tidal", pp["tidal_isrc"])) - if pp["qobuz_isrc"] and _tag_enabled("qobuz.tags.isrc"): - isrc_candidates.append(("Qobuz", pp["qobuz_isrc"])) - if isrc_candidates: - isrc_source, final_isrc = isrc_candidates[0] - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc])) - elif is_vorbis_like(audio_file, symbols): - audio_file["ISRC"] = [final_isrc] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))] - logger_.info("ISRC (%s): %s", isrc_source, final_isrc) - - copyright_candidates = [] - if pp["tidal_copyright"] and _tag_enabled("tidal.tags.copyright"): - copyright_candidates.append(("Tidal", pp["tidal_copyright"])) - if pp["qobuz_copyright"] and _tag_enabled("qobuz.tags.copyright"): - copyright_candidates.append(("Qobuz", pp["qobuz_copyright"])) - if copyright_candidates: - copyright_source, final_copyright = copyright_candidates[0] - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright])) - elif is_vorbis_like(audio_file, symbols): - audio_file["COPYRIGHT"] = [final_copyright] - elif isinstance(audio_file, symbols.MP4): - audio_file["cprt"] = [final_copyright] - logger_.info("Copyright (%s): %s", copyright_source, final_copyright[:60]) - - if _tag_enabled("qobuz.tags.label") and pp["qobuz_label"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]])) - elif is_vorbis_like(audio_file, symbols): - audio_file["LABEL"] = [pp["qobuz_label"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))] - - if _tag_enabled("lastfm.tags.url") and pp["lastfm_url"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]])) - elif is_vorbis_like(audio_file, symbols): - audio_file["LASTFM_URL"] = [pp["lastfm_url"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))] - - if _tag_enabled("genius.tags.url") and pp["genius_url"]: - if isinstance(audio_file.tags, symbols.ID3): - audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]])) - elif is_vorbis_like(audio_file, symbols): - audio_file["GENIUS_URL"] = [pp["genius_url"]] - elif isinstance(audio_file, symbols.MP4): - audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))] - + release_year = _write_embedded_metadata(audio_file, metadata, pp, cfg, symbols) release_id = pp["release_mbid"] if release_id: metadata["musicbrainz_release_id"] = release_id - if db is not None: - try: - album_name_for_db = metadata.get("album", "") - album_artist_for_db = metadata.get("album_artist", "") or metadata.get("artist", "") - if album_name_for_db and album_artist_for_db: - conn = db._get_connection() - try: - cursor = conn.cursor() - cursor.execute( - """ - UPDATE albums SET year = ? - WHERE (year IS NULL OR year = 0) - AND id IN ( - SELECT al.id FROM albums al - JOIN artists ar ON ar.id = al.artist_id - WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) - ) - """, - (int(release_year), album_name_for_db, album_artist_for_db), - ) - if cursor.rowcount > 0: - conn.commit() - logger_.info("Updated album year to %s in database", release_year) - else: - conn.rollback() - finally: - conn.close() - except Exception as exc: - logger_.error("Could not update album year in DB: %s", exc) + _update_album_year_in_database(db, metadata, release_year) except Exception as exc: - logger_.error("Error embedding source IDs (non-fatal): %s", exc) + logger.error("Error embedding source IDs (non-fatal): %s", exc) diff --git a/tests/test_metadata_enrichment.py b/tests/test_metadata_enrichment.py index 35ffcd4b..24c5a302 100644 --- a/tests/test_metadata_enrichment.py +++ b/tests/test_metadata_enrichment.py @@ -208,6 +208,60 @@ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatc assert "ITUNES_ALBUM_ID" in legacy_descs +def test_enhance_file_metadata_forwards_runtime_to_source_embedding(monkeypatch): + audio = _FakeAudio() + symbols = _fake_symbols(audio) + seen = {} + runtime = types.SimpleNamespace(marker="runtime") + + monkeypatch.setattr(me, "get_config_manager", lambda: _Config( + { + "metadata_enhancement.enabled": True, + "metadata_enhancement.embed_album_art": False, + "metadata_enhancement.tags.write_multi_artist": False, + } + )) + monkeypatch.setattr(me, "get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(me, "strip_all_non_audio_tags", lambda file_path: {"apev2_stripped": False, "apev2_tag_count": 0}) + monkeypatch.setattr( + me, + "extract_source_metadata", + lambda context, artist, album_info: { + "source": "deezer", + "source_track_id": "dz-track", + "source_artist_id": "dz-artist", + "source_album_id": "dz-album", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + "track_number": 3, + "total_tracks": 12, + "disc_number": 2, + "date": "2024", + "genre": "Rock", + }, + ) + monkeypatch.setattr( + me, + "embed_source_ids", + lambda audio_file, metadata, context, runtime=None: seen.setdefault("runtime", runtime), + ) + monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None) + monkeypatch.setattr(me, "verify_metadata_written", lambda file_path: True) + + result = me.enhance_file_metadata( + "song.flac", + {"_audio_quality": ""}, + {"name": "Artist One"}, + {}, + runtime=runtime, + ) + + assert result is True + assert seen["runtime"] is runtime + + def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch): audio = _FakeAudio() symbols = _fake_symbols(audio) diff --git a/web_server.py b/web_server.py index bf7c7d4a..bd92c325 100644 --- a/web_server.py +++ b/web_server.py @@ -19678,8 +19678,14 @@ import urllib.request def _wipe_source_tags(file_path: str) -> bool: return metadata_enrichment.wipe_source_tags(file_path) -def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: - return metadata_enrichment.enhance_file_metadata(file_path, context, artist, album_info) +def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: + return metadata_enrichment.enhance_file_metadata( + file_path, + context, + artist, + album_info, + runtime=runtime or _build_import_pipeline_runtime(), + ) def _download_cover_art(album_info: dict, target_dir: str, context: dict = None): @@ -19781,6 +19787,15 @@ def _build_import_pipeline_runtime(): on_download_completed=_on_download_completed, web_scan_manager=web_scan_manager, repair_worker=repair_worker, + mb_worker=mb_worker, + deezer_worker=deezer_worker, + audiodb_worker=audiodb_worker, + tidal_client=tidal_client, + qobuz_enrichment_worker=qobuz_enrichment_worker, + lastfm_worker=lastfm_worker, + genius_worker=genius_worker, + spotify_enrichment_worker=spotify_enrichment_worker, + itunes_enrichment_worker=itunes_enrichment_worker, ) @@ -19897,18 +19912,28 @@ def _build_import_pipeline_runtime(): on_download_completed=_on_download_completed, web_scan_manager=web_scan_manager, repair_worker=repair_worker, + mb_worker=mb_worker, + deezer_worker=deezer_worker, + audiodb_worker=audiodb_worker, + tidal_client=tidal_client, + qobuz_enrichment_worker=qobuz_enrichment_worker, + lastfm_worker=lastfm_worker, + genius_worker=genius_worker, + spotify_enrichment_worker=spotify_enrichment_worker, + itunes_enrichment_worker=itunes_enrichment_worker, ) def _wipe_source_tags(file_path: str) -> bool: return metadata_enrichment.wipe_source_tags(file_path) -def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: +def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: return metadata_enrichment.enhance_file_metadata( file_path, context, artist, album_info, + runtime=runtime or _build_import_pipeline_runtime(), ) From 9e496397daed656912e571492ad20c0066815022 Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sat, 25 Apr 2026 14:32:17 +0300 Subject: [PATCH 09/19] Move shared metadata helpers into package - Relocate the shared metadata helper module from core/metadata_common.py into core/metadata/common.py. - Update the new metadata package, the import pipeline, and the web entrypoint to use the package-scoped helper. - Keep the shared config, mutagen, file-lock, and tag-writing helpers centralized without touching unrelated files. --- core/imports/pipeline.py | 2 +- core/metadata/artwork.py | 27 +++++++------- .../common.py} | 2 +- core/metadata/enrichment.py | 36 ++++++++++--------- core/metadata/lyrics.py | 13 ++++--- core/metadata/source.py | 2 +- web_server.py | 2 +- 7 files changed, 45 insertions(+), 39 deletions(-) rename core/{metadata_common.py => metadata/common.py} (99%) diff --git a/core/imports/pipeline.py b/core/imports/pipeline.py index b915042f..a94445b8 100644 --- a/core/imports/pipeline.py +++ b/core/imports/pipeline.py @@ -52,7 +52,7 @@ from core.runtime_state import ( tasks_lock, ) from core.metadata.artwork import download_cover_art -from core.metadata_common import wipe_source_tags +from core.metadata.common import wipe_source_tags from core.metadata.enrichment import enhance_file_metadata from core.imports.paths import ( build_final_path_for_track, diff --git a/core/metadata/artwork.py b/core/metadata/artwork.py index 8d43b0af..ca268e52 100644 --- a/core/metadata/artwork.py +++ b/core/metadata/artwork.py @@ -7,12 +7,12 @@ import re import urllib.request from core.imports.context import get_import_context_album -from core.metadata_common import ( +from core.metadata.common import ( get_config_manager, get_image_dimensions, - get_logger, get_mutagen_symbols, ) +from utils.logging_config import get_logger as _create_logger __all__ = [ "embed_album_art_metadata", @@ -20,9 +20,11 @@ __all__ = [ ] +logger = _create_logger("metadata.artwork") + + def embed_album_art_metadata(audio_file, metadata: dict): cfg = get_config_manager() - logger_ = get_logger() symbols = get_mutagen_symbols() if not symbols: return @@ -47,14 +49,14 @@ def embed_album_art_metadata(audio_file, metadata: dict): if not image_data: art_url = metadata.get("album_art_url") if not art_url: - logger_.warning("No album art URL available for embedding.") + logger.warning("No album art URL available for embedding.") return with urllib.request.urlopen(art_url, timeout=10) as response: image_data = response.read() mime_type = response.info().get_content_type() or "image/jpeg" if not image_data: - logger_.error("Failed to download album art data.") + logger.error("Failed to download album art data.") return if isinstance(audio_file.tags, symbols.ID3): @@ -73,14 +75,13 @@ def embed_album_art_metadata(audio_file, metadata: dict): fmt = symbols.MP4Cover.FORMAT_JPEG if "jpeg" in mime_type else symbols.MP4Cover.FORMAT_PNG audio_file["covr"] = [symbols.MP4Cover(image_data, imageformat=fmt)] - logger_.info("Album art successfully embedded.") + logger.info("Album art successfully embedded.") except Exception as exc: - logger_.error("Error embedding album art: %s", exc) + logger.error("Error embedding album art: %s", exc) def download_cover_art(album_info: dict, target_dir: str, context: dict = None): cfg = get_config_manager() - logger_ = get_logger() if cfg.get("metadata_enhancement.cover_art_download", True) is False: return @@ -117,7 +118,7 @@ def download_cover_art(album_info: dict, target_dir: str, context: dict = None): image_data = None if is_upgrade and not image_data: - logger_.error("CAA upgrade failed - keeping existing cover.jpg") + logger.error("CAA upgrade failed - keeping existing cover.jpg") return if not image_data: @@ -130,7 +131,7 @@ def download_cover_art(album_info: dict, target_dir: str, context: dict = None): if images and isinstance(images[0], dict): art_url = images[0].get("url", "") if art_url: - logger_.info("Using cover art URL from album context") + logger.info("Using cover art URL from album context") if art_url and "i.scdn.co" in art_url: try: from core.spotify_client import _upgrade_spotify_image_url @@ -141,7 +142,7 @@ def download_cover_art(album_info: dict, target_dir: str, context: dict = None): elif art_url and "mzstatic.com" in art_url: art_url = re.sub(r"\d+x\d+bb", "3000x3000bb", art_url) if not art_url: - logger_.warning("No cover art URL available for download.") + logger.warning("No cover art URL available for download.") return with urllib.request.urlopen(art_url, timeout=10) as response: image_data = response.read() @@ -151,6 +152,6 @@ def download_cover_art(album_info: dict, target_dir: str, context: dict = None): with open(cover_path, "wb") as handle: handle.write(image_data) - logger_.info("Cover art downloaded to: %s", cover_path) + logger.info("Cover art downloaded to: %s", cover_path) except Exception as exc: - logger_.error("Error downloading cover.jpg: %s", exc) + logger.error("Error downloading cover.jpg: %s", exc) diff --git a/core/metadata_common.py b/core/metadata/common.py similarity index 99% rename from core/metadata_common.py rename to core/metadata/common.py index 5e27bc16..62408352 100644 --- a/core/metadata_common.py +++ b/core/metadata/common.py @@ -10,7 +10,7 @@ from typing import Any, Dict from utils.logging_config import get_logger as _create_logger -logger = _create_logger("metadata_common") +logger = _create_logger("metadata.common") __all__ = [ "get_logger", diff --git a/core/metadata/enrichment.py b/core/metadata/enrichment.py index f44d706a..962a6bcf 100644 --- a/core/metadata/enrichment.py +++ b/core/metadata/enrichment.py @@ -5,10 +5,9 @@ from __future__ import annotations import os from core.metadata.artwork import embed_album_art_metadata -from core.metadata_common import ( +from core.metadata.common import ( get_config_manager, get_file_lock, - get_logger, get_mutagen_symbols, is_vorbis_like, save_audio_file, @@ -16,6 +15,7 @@ from core.metadata_common import ( verify_metadata_written, ) from core.metadata.source import embed_source_ids, extract_source_metadata +from utils.logging_config import get_logger as _create_logger __all__ = [ @@ -25,11 +25,13 @@ __all__ = [ ] +logger = _create_logger("metadata.enrichment") + + def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: cfg = get_config_manager() - logger_ = get_logger() if cfg.get("metadata_enhancement.enabled", True) is False: - logger_.warning("Metadata enhancement disabled in config.") + logger.warning("Metadata enhancement disabled in config.") return True if album_info is None: @@ -37,17 +39,17 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf symbols = get_mutagen_symbols() if not symbols: - logger_.error("Mutagen is unavailable, cannot enhance metadata.") + logger.error("Mutagen is unavailable, cannot enhance metadata.") return False file_lock = get_file_lock(file_path) with file_lock: - logger_.info("Enhancing metadata for: %s", os.path.basename(file_path)) + logger.info("Enhancing metadata for: %s", os.path.basename(file_path)) try: strip_all_non_audio_tags(file_path) audio_file = symbols.File(file_path) if audio_file is None: - logger_.error("Could not load audio file with Mutagen: %s", file_path) + logger.error("Could not load audio file with Mutagen: %s", file_path) return False if hasattr(audio_file, "clear_pictures"): @@ -56,7 +58,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf if audio_file.tags is not None: if len(audio_file.tags) > 0: tag_keys = list(audio_file.tags.keys())[:15] - logger_.info("Clearing %s existing tags: %s", len(audio_file.tags), ", ".join(str(k) for k in tag_keys)) + logger.info("Clearing %s existing tags: %s", len(audio_file.tags), ", ".join(str(k) for k in tag_keys)) audio_file.tags.clear() else: audio_file.add_tags() @@ -65,7 +67,7 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf metadata = extract_source_metadata(context, artist, album_info) if not metadata: - logger_.error("Could not extract source metadata, saving with cleared tags.") + logger.error("Could not extract source metadata, saving with cleared tags.") save_audio_file(audio_file, symbols) return True @@ -147,17 +149,17 @@ def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_inf verified = verify_metadata_written(file_path) if verified: - logger_.info("Metadata enhanced successfully.") + logger.info("Metadata enhanced successfully.") else: - logger_.info("Metadata saved but verification found issues (see above).") + logger.info("Metadata saved but verification found issues (see above).") return True except Exception as exc: import traceback - logger_.error("Error enhancing metadata for %s: %s", file_path, exc) - logger_.error("[Metadata Debug] Exception type: %s", type(exc).__name__) - logger_.info("[Metadata Debug] File exists: %s", os.path.exists(file_path)) - logger_.warning("[Metadata Debug] Artist: %s", artist.get("name", "MISSING") if artist else "None") - logger_.warning("[Metadata Debug] Album info: %s", album_info.get("album_name", "MISSING") if album_info else "None") - logger_.error("[Metadata Debug] Traceback:\n%s", traceback.format_exc()) + logger.error("Error enhancing metadata for %s: %s", file_path, exc) + logger.error("[Metadata Debug] Exception type: %s", type(exc).__name__) + logger.info("[Metadata Debug] File exists: %s", os.path.exists(file_path)) + logger.warning("[Metadata Debug] Artist: %s", artist.get("name", "MISSING") if artist else "None") + logger.warning("[Metadata Debug] Album info: %s", album_info.get("album_name", "MISSING") if album_info else "None") + logger.error("[Metadata Debug] Traceback:\n%s", traceback.format_exc()) return False diff --git a/core/metadata/lyrics.py b/core/metadata/lyrics.py index 00719c5c..bbfde752 100644 --- a/core/metadata/lyrics.py +++ b/core/metadata/lyrics.py @@ -9,16 +9,19 @@ from core.imports.context import ( get_import_original_search, normalize_import_context, ) -from core.metadata_common import get_config_manager, get_logger +from core.metadata.common import get_config_manager +from utils.logging_config import get_logger as _create_logger __all__ = [ "generate_lrc_file", ] +logger = _create_logger("metadata.lyrics") + + def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: cfg = get_config_manager() - logger_ = get_logger() if cfg.get("metadata_enhancement.lrclib_enabled", True) is False: return False @@ -58,10 +61,10 @@ def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: d ) if success: - logger_.info("LRC file generated for: %s", track_name) + logger.info("LRC file generated for: %s", track_name) else: - logger_.warning("No lyrics found for: %s", track_name) + logger.warning("No lyrics found for: %s", track_name) return success except Exception as exc: - logger_.error("Error generating LRC file for %s: %s", file_path, exc) + logger.error("Error generating LRC file for %s: %s", file_path, exc) return False diff --git a/core/metadata/source.py b/core/metadata/source.py index 0661d11a..5bd790ba 100644 --- a/core/metadata/source.py +++ b/core/metadata/source.py @@ -21,7 +21,7 @@ from core.imports.context import ( ) from core.metadata_service import get_itunes_client from database.music_database import get_database -from core.metadata_common import ( +from core.metadata.common import ( get_config_manager, get_mutagen_symbols, is_vorbis_like, diff --git a/web_server.py b/web_server.py index bd92c325..a002458e 100644 --- a/web_server.py +++ b/web_server.py @@ -132,7 +132,7 @@ from core.imports.staging import ( start_import_suggestions_cache, ) from core.imports.paths import build_final_path_for_track as _build_final_path_for_track -from core.metadata_common import get_file_lock +from core.metadata.common import get_file_lock from core.metadata.source import ( mb_release_cache, mb_release_cache_lock, From bcab54095ee457e9d03bb09ad047c68b8eed582d Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sat, 25 Apr 2026 14:44:54 +0300 Subject: [PATCH 10/19] Group metadata tests under tests/metadata - Move the metadata and MusicBrainz-related tests into a dedicated tests/metadata subfolder. - Keep the rest of the suite flat for now. - Preserve the existing test filenames so the change stays organizational rather than behavioral. --- tests/{ => metadata}/test_artist_source_detail.py | 0 tests/{ => metadata}/test_artist_source_lookup.py | 0 tests/{ => metadata}/test_enrichment_events.py | 0 tests/{ => metadata}/test_metadata_cache_batch_lookup.py | 0 tests/{ => metadata}/test_metadata_enrichment.py | 0 tests/{ => metadata}/test_metadata_gap_filler.py | 0 tests/{ => metadata}/test_metadata_service_album_tracks.py | 0 tests/{ => metadata}/test_metadata_service_artist_image.py | 0 tests/{ => metadata}/test_metadata_service_cache.py | 0 tests/{ => metadata}/test_metadata_service_discography.py | 0 tests/{ => metadata}/test_metadata_service_musicmap.py | 0 .../{ => metadata}/test_metadata_service_single_import_context.py | 0 tests/{ => metadata}/test_musicbrainz_search.py | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename tests/{ => metadata}/test_artist_source_detail.py (100%) rename tests/{ => metadata}/test_artist_source_lookup.py (100%) rename tests/{ => metadata}/test_enrichment_events.py (100%) rename tests/{ => metadata}/test_metadata_cache_batch_lookup.py (100%) rename tests/{ => metadata}/test_metadata_enrichment.py (100%) rename tests/{ => metadata}/test_metadata_gap_filler.py (100%) rename tests/{ => metadata}/test_metadata_service_album_tracks.py (100%) rename tests/{ => metadata}/test_metadata_service_artist_image.py (100%) rename tests/{ => metadata}/test_metadata_service_cache.py (100%) rename tests/{ => metadata}/test_metadata_service_discography.py (100%) rename tests/{ => metadata}/test_metadata_service_musicmap.py (100%) rename tests/{ => metadata}/test_metadata_service_single_import_context.py (100%) rename tests/{ => metadata}/test_musicbrainz_search.py (100%) diff --git a/tests/test_artist_source_detail.py b/tests/metadata/test_artist_source_detail.py similarity index 100% rename from tests/test_artist_source_detail.py rename to tests/metadata/test_artist_source_detail.py diff --git a/tests/test_artist_source_lookup.py b/tests/metadata/test_artist_source_lookup.py similarity index 100% rename from tests/test_artist_source_lookup.py rename to tests/metadata/test_artist_source_lookup.py diff --git a/tests/test_enrichment_events.py b/tests/metadata/test_enrichment_events.py similarity index 100% rename from tests/test_enrichment_events.py rename to tests/metadata/test_enrichment_events.py diff --git a/tests/test_metadata_cache_batch_lookup.py b/tests/metadata/test_metadata_cache_batch_lookup.py similarity index 100% rename from tests/test_metadata_cache_batch_lookup.py rename to tests/metadata/test_metadata_cache_batch_lookup.py diff --git a/tests/test_metadata_enrichment.py b/tests/metadata/test_metadata_enrichment.py similarity index 100% rename from tests/test_metadata_enrichment.py rename to tests/metadata/test_metadata_enrichment.py diff --git a/tests/test_metadata_gap_filler.py b/tests/metadata/test_metadata_gap_filler.py similarity index 100% rename from tests/test_metadata_gap_filler.py rename to tests/metadata/test_metadata_gap_filler.py diff --git a/tests/test_metadata_service_album_tracks.py b/tests/metadata/test_metadata_service_album_tracks.py similarity index 100% rename from tests/test_metadata_service_album_tracks.py rename to tests/metadata/test_metadata_service_album_tracks.py diff --git a/tests/test_metadata_service_artist_image.py b/tests/metadata/test_metadata_service_artist_image.py similarity index 100% rename from tests/test_metadata_service_artist_image.py rename to tests/metadata/test_metadata_service_artist_image.py diff --git a/tests/test_metadata_service_cache.py b/tests/metadata/test_metadata_service_cache.py similarity index 100% rename from tests/test_metadata_service_cache.py rename to tests/metadata/test_metadata_service_cache.py diff --git a/tests/test_metadata_service_discography.py b/tests/metadata/test_metadata_service_discography.py similarity index 100% rename from tests/test_metadata_service_discography.py rename to tests/metadata/test_metadata_service_discography.py diff --git a/tests/test_metadata_service_musicmap.py b/tests/metadata/test_metadata_service_musicmap.py similarity index 100% rename from tests/test_metadata_service_musicmap.py rename to tests/metadata/test_metadata_service_musicmap.py diff --git a/tests/test_metadata_service_single_import_context.py b/tests/metadata/test_metadata_service_single_import_context.py similarity index 100% rename from tests/test_metadata_service_single_import_context.py rename to tests/metadata/test_metadata_service_single_import_context.py diff --git a/tests/test_musicbrainz_search.py b/tests/metadata/test_musicbrainz_search.py similarity index 100% rename from tests/test_musicbrainz_search.py rename to tests/metadata/test_musicbrainz_search.py From 9b2b6d856febaf6939e003c27cf8fd698f8893a5 Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sat, 25 Apr 2026 16:43:17 +0300 Subject: [PATCH 11/19] Split runtime builders into owning modules - Move the import pipeline runtime factory into core.imports.pipeline - Move the metadata runtime factory into core.metadata.enrichment - Keep the web server wiring thin and drop the shared glue module - Add contract tests that keep the two runtime bundles separate --- core/imports/pipeline.py | 35 ++++++++++-- core/metadata/enrichment.py | 29 ++++++++++ tests/imports/test_import_pipeline.py | 81 +++++++++++++++++++++++++++ tests/metadata/test_runtime_bundle.py | 56 ++++++++++++++++++ web_server.py | 68 ++++------------------ 5 files changed, 206 insertions(+), 63 deletions(-) create mode 100644 tests/metadata/test_runtime_bundle.py diff --git a/core/imports/pipeline.py b/core/imports/pipeline.py index a94445b8..5d202a2a 100644 --- a/core/imports/pipeline.py +++ b/core/imports/pipeline.py @@ -6,6 +6,8 @@ import json import os import threading import time +from types import SimpleNamespace +from typing import Any from config.settings import config_manager from core.imports.file_ops import ( @@ -68,12 +70,35 @@ from utils.logging_config import get_logger logger = get_logger("imports.pipeline") pp_logger = get_logger("post_processing") +__all__ = [ + "build_import_pipeline_runtime", + "post_process_matched_download", + "post_process_matched_download_with_verification", +] -def post_process_matched_download(context_key, context, file_path, runtime): + +def build_import_pipeline_runtime( + *, + automation_engine: Any | None = None, + on_download_completed: Any | None = None, + web_scan_manager: Any | None = None, + repair_worker: Any | None = None, +) -> SimpleNamespace: + """Build the runtime object consumed by core.imports.pipeline.""" + return SimpleNamespace( + automation_engine=automation_engine, + on_download_completed=on_download_completed, + web_scan_manager=web_scan_manager, + repair_worker=repair_worker, + ) + + +def post_process_matched_download(context_key, context, file_path, runtime, metadata_runtime=None): on_download_completed = getattr(runtime, "on_download_completed", None) automation_engine = getattr(runtime, "automation_engine", None) web_scan_manager = getattr(runtime, "web_scan_manager", None) repair_worker = getattr(runtime, "repair_worker", None) + metadata_runtime = metadata_runtime or runtime def _notify_download_completed(batch_id, task_id, success=True): if on_download_completed: @@ -364,7 +389,7 @@ def post_process_matched_download(context_key, context, file_path, runtime): f"[Metadata Input] Playlist mode - artist: '{artist_context.get('name', 'MISSING')}' " f"(id: {artist_context.get('id', 'MISSING')})" ) - enhance_file_metadata(file_path, context, artist_context, None, runtime=runtime) + enhance_file_metadata(file_path, context, artist_context, None, runtime=metadata_runtime) except Exception as meta_err: import traceback pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") @@ -529,7 +554,7 @@ def post_process_matched_download(context_key, context, file_path, runtime): ) else: logger.info("[Metadata Input] album_info: None (single track)") - enhance_file_metadata(file_path, context, artist_context, album_info, runtime=runtime) + enhance_file_metadata(file_path, context, artist_context, album_info, runtime=metadata_runtime) except Exception as meta_err: import traceback pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") @@ -762,7 +787,7 @@ def post_process_matched_download(context_key, context, file_path, runtime): post_process_locks.pop(context_key, None) -def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime): +def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime, metadata_runtime=None): on_download_completed = getattr(runtime, "on_download_completed", None) def _notify_download_completed(batch_id, task_id, success=True): @@ -773,7 +798,7 @@ def post_process_matched_download_with_verification(context_key, context, file_p try: original_task_id = context.pop('task_id', None) original_batch_id = context.pop('batch_id', None) - post_process_matched_download(context_key, context, file_path, runtime) + post_process_matched_download(context_key, context, file_path, runtime, metadata_runtime=metadata_runtime) if original_task_id: context['task_id'] = original_task_id if original_batch_id: diff --git a/core/metadata/enrichment.py b/core/metadata/enrichment.py index 962a6bcf..82f462d6 100644 --- a/core/metadata/enrichment.py +++ b/core/metadata/enrichment.py @@ -3,6 +3,8 @@ from __future__ import annotations import os +from types import SimpleNamespace +from typing import Any from core.metadata.artwork import embed_album_art_metadata from core.metadata.common import ( @@ -19,6 +21,7 @@ from utils.logging_config import get_logger as _create_logger __all__ = [ + "build_metadata_enrichment_runtime", "enhance_file_metadata", "extract_source_metadata", "embed_source_ids", @@ -28,6 +31,32 @@ __all__ = [ logger = _create_logger("metadata.enrichment") +def build_metadata_enrichment_runtime( + *, + mb_worker: Any | None = None, + deezer_worker: Any | None = None, + audiodb_worker: Any | None = None, + tidal_client: Any | None = None, + qobuz_enrichment_worker: Any | None = None, + lastfm_worker: Any | None = None, + genius_worker: Any | None = None, + spotify_enrichment_worker: Any | None = None, + itunes_enrichment_worker: Any | None = None, +) -> SimpleNamespace: + """Build the runtime object consumed by core.metadata.enrichment/source.""" + return SimpleNamespace( + mb_worker=mb_worker, + deezer_worker=deezer_worker, + audiodb_worker=audiodb_worker, + tidal_client=tidal_client, + qobuz_enrichment_worker=qobuz_enrichment_worker, + lastfm_worker=lastfm_worker, + genius_worker=genius_worker, + spotify_enrichment_worker=spotify_enrichment_worker, + itunes_enrichment_worker=itunes_enrichment_worker, + ) + + def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: cfg = get_config_manager() if cfg.get("metadata_enhancement.enabled", True) is False: diff --git a/tests/imports/test_import_pipeline.py b/tests/imports/test_import_pipeline.py index 20b71692..231a0437 100644 --- a/tests/imports/test_import_pipeline.py +++ b/tests/imports/test_import_pipeline.py @@ -126,3 +126,84 @@ def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch): runtime_state.processed_download_ids.update(original_processed_ids) runtime_state.post_process_locks.clear() runtime_state.post_process_locks.update(original_post_locks) + + +def test_post_process_matched_download_forwards_separate_metadata_runtime(tmp_path, monkeypatch): + source_path = tmp_path / "source.flac" + source_path.write_bytes(b"audio") + target_path = tmp_path / "Album Folder" / "track.flac" + + runtime = types.SimpleNamespace( + automation_engine=None, + on_download_completed=None, + web_scan_manager=None, + repair_worker=None, + ) + metadata_runtime = types.SimpleNamespace(marker="metadata-runtime") + seen = {} + + monkeypatch.setattr(import_pipeline, "config_manager", types.SimpleNamespace( + get=lambda key, default=None: { + "post_processing.replaygain_enabled": False, + "lossy_copy.enabled": False, + "lossy_copy.delete_original": False, + "import.replace_lower_quality": False, + "soulseek.download_path": str(tmp_path / "downloads"), + }.get(key, default) + )) + monkeypatch.setattr(import_pipeline, "normalize_import_context", lambda context: context) + monkeypatch.setattr(import_pipeline, "get_import_track_info", lambda context: {"_playlist_folder_mode": True, "_playlist_name": "Playlist"}) + monkeypatch.setattr(import_pipeline, "get_import_original_search", lambda context: {"title": "Track", "album": "Album"}) + monkeypatch.setattr(import_pipeline, "get_import_context_artist", lambda context: {"name": "Artist"}) + monkeypatch.setattr(import_pipeline, "get_import_has_clean_metadata", lambda context: True) + monkeypatch.setattr( + import_pipeline, + "build_import_album_info", + lambda context, force_album=False: { + "is_album": True, + "album_name": "Album", + "track_number": 1, + "disc_number": 1, + "clean_track_name": "Track", + "source": "spotify", + }, + ) + monkeypatch.setattr(import_pipeline, "resolve_album_group", lambda artist_context, album_info, original_album: album_info["album_name"]) + monkeypatch.setattr(import_pipeline, "get_import_clean_title", lambda *args, **kwargs: "Track") + monkeypatch.setattr(import_pipeline, "get_audio_quality_string", lambda file_path: "") + monkeypatch.setattr(import_pipeline, "check_flac_bit_depth", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "build_final_path_for_track", lambda *args, **kwargs: (str(target_path), None)) + + def _capture_enhance(file_path, context, artist, album_info, runtime=None): + seen["runtime"] = runtime + return True + + monkeypatch.setattr(import_pipeline, "enhance_file_metadata", _capture_enhance) + monkeypatch.setattr(import_pipeline, "safe_move_file", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "download_cover_art", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "generate_lrc_file", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "downsample_hires_flac", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "create_lossy_copy", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "cleanup_empty_directories", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "record_library_history_download", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "record_download_provenance", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "record_soulsync_library_entry", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "check_and_remove_from_wishlist", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "record_retag_download", lambda *args, **kwargs: None) + + context = { + "track_info": {"_playlist_folder_mode": True, "_playlist_name": "Playlist"}, + "original_search_result": {"title": "Track", "album": "Album"}, + "is_album_download": False, + } + + import_pipeline.post_process_matched_download( + "ctx-1", + context, + str(source_path), + runtime, + metadata_runtime=metadata_runtime, + ) + + assert seen["runtime"] is metadata_runtime diff --git a/tests/metadata/test_runtime_bundle.py b/tests/metadata/test_runtime_bundle.py new file mode 100644 index 00000000..415944af --- /dev/null +++ b/tests/metadata/test_runtime_bundle.py @@ -0,0 +1,56 @@ +import types + +from core.imports.pipeline import build_import_pipeline_runtime +from core.metadata.enrichment import build_metadata_enrichment_runtime + + +def test_build_import_pipeline_runtime_exposes_expected_contract(): + import_fields = { + "automation_engine": object(), + "on_download_completed": object(), + "web_scan_manager": object(), + "repair_worker": object(), + } + runtime = build_import_pipeline_runtime(**import_fields) + + assert isinstance(runtime, types.SimpleNamespace) + for name, value in import_fields.items(): + assert hasattr(runtime, name) + assert getattr(runtime, name) is value + + for name in ( + "mb_worker", + "deezer_worker", + "audiodb_worker", + "tidal_client", + "qobuz_enrichment_worker", + "lastfm_worker", + "genius_worker", + "spotify_enrichment_worker", + "itunes_enrichment_worker", + ): + assert not hasattr(runtime, name) + + +def test_build_metadata_enrichment_runtime_exposes_expected_contract(): + metadata_fields = { + "mb_worker": object(), + "deezer_worker": object(), + "audiodb_worker": object(), + "tidal_client": object(), + "qobuz_enrichment_worker": object(), + "lastfm_worker": object(), + "genius_worker": object(), + "spotify_enrichment_worker": object(), + "itunes_enrichment_worker": object(), + } + + runtime = build_metadata_enrichment_runtime(**metadata_fields) + + assert isinstance(runtime, types.SimpleNamespace) + for name, value in metadata_fields.items(): + assert hasattr(runtime, name) + assert getattr(runtime, name) is value + + for name in ("automation_engine", "on_download_completed", "web_scan_manager", "repair_worker"): + assert not hasattr(runtime, name) diff --git a/web_server.py b/web_server.py index a002458e..d564c911 100644 --- a/web_server.py +++ b/web_server.py @@ -132,7 +132,9 @@ from core.imports.staging import ( start_import_suggestions_cache, ) from core.imports.paths import build_final_path_for_track as _build_final_path_for_track +from core.imports.pipeline import build_import_pipeline_runtime as _build_import_pipeline_runtime from core.metadata.common import get_file_lock +from core.metadata.enrichment import build_metadata_enrichment_runtime as _build_metadata_enrichment_runtime from core.metadata.source import ( mb_release_cache, mb_release_cache_lock, @@ -19678,13 +19680,13 @@ import urllib.request def _wipe_source_tags(file_path: str) -> bool: return metadata_enrichment.wipe_source_tags(file_path) -def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: +def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, metadata_runtime=None) -> bool: return metadata_enrichment.enhance_file_metadata( file_path, context, artist, album_info, - runtime=runtime or _build_import_pipeline_runtime(), + runtime=metadata_runtime or _build_metadata_enrichment_runtime(), ) @@ -19778,24 +19780,7 @@ def _post_process_matched_download_with_verification(context_key, context, file_ task_id, batch_id, _build_import_pipeline_runtime(), - ) - -def _build_import_pipeline_runtime(): - """Collect live controller dependencies for the shared import pipeline.""" - return types.SimpleNamespace( - automation_engine=automation_engine, - on_download_completed=_on_download_completed, - web_scan_manager=web_scan_manager, - repair_worker=repair_worker, - mb_worker=mb_worker, - deezer_worker=deezer_worker, - audiodb_worker=audiodb_worker, - tidal_client=tidal_client, - qobuz_enrichment_worker=qobuz_enrichment_worker, - lastfm_worker=lastfm_worker, - genius_worker=genius_worker, - spotify_enrichment_worker=spotify_enrichment_worker, - itunes_enrichment_worker=itunes_enrichment_worker, + _build_metadata_enrichment_runtime(), ) @@ -19903,45 +19888,12 @@ def _post_process_matched_download(context_key, context, file_path): just move files to /Transfer without metadata enhancement. """ from core.imports.pipeline import post_process_matched_download - return post_process_matched_download(context_key, context, file_path, _build_import_pipeline_runtime()) - -def _build_import_pipeline_runtime(): - """Collect the live controller dependencies needed by core.imports.pipeline.""" - return types.SimpleNamespace( - automation_engine=automation_engine, - on_download_completed=_on_download_completed, - web_scan_manager=web_scan_manager, - repair_worker=repair_worker, - mb_worker=mb_worker, - deezer_worker=deezer_worker, - audiodb_worker=audiodb_worker, - tidal_client=tidal_client, - qobuz_enrichment_worker=qobuz_enrichment_worker, - lastfm_worker=lastfm_worker, - genius_worker=genius_worker, - spotify_enrichment_worker=spotify_enrichment_worker, - itunes_enrichment_worker=itunes_enrichment_worker, - ) - -def _wipe_source_tags(file_path: str) -> bool: - return metadata_enrichment.wipe_source_tags(file_path) - - -def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: - return metadata_enrichment.enhance_file_metadata( + return post_process_matched_download( + context_key, + context, file_path, - context, - artist, - album_info, - runtime=runtime or _build_import_pipeline_runtime(), - ) - - -def _download_cover_art(album_info: dict, target_dir: str, context: dict = None): - return metadata_enrichment.download_cover_art( - album_info, - target_dir, - context, + _build_import_pipeline_runtime(), + metadata_runtime=_build_metadata_enrichment_runtime(), ) # Track stale transfer keys (completed in slskd but no context — e.g., from before app restart) From 594c8c1b935e56900e99f916f81b1d68974898f6 Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sat, 25 Apr 2026 19:19:35 +0300 Subject: [PATCH 12/19] Cleanup duplicated code Leftovers from the earlier recovery mission --- web_server.py | 1549 ------------------------------------------------- 1 file changed, 1549 deletions(-) diff --git a/web_server.py b/web_server.py index d564c911..1e036ef6 100644 --- a/web_server.py +++ b/web_server.py @@ -799,218 +799,6 @@ transfer_data_cache = { } -def get_cached_transfer_data(): - """ - Return live Soulseek transfer data with a short cache window. - - The download modal, batch status endpoint, and socket emit loop all call - this helper, so we keep the API hit rate low while still refreshing often. - """ - current_time = time.time() - - with transfer_data_cache['update_lock']: - if (current_time - transfer_data_cache['last_update']) < transfer_data_cache['cache_duration']: - return transfer_data_cache['data'] - - if not soulseek_client: - return {} - - live_transfers_lookup = {} - try: - transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) - if transfers_data: - for user_data in transfers_data: - if not isinstance(user_data, dict): - continue - username = user_data.get('username', 'Unknown') - for directory in user_data.get('directories', []) or []: - if not isinstance(directory, dict): - continue - for file_info in directory.get('files', []) or []: - if not isinstance(file_info, dict): - continue - transfer = dict(file_info) - transfer['username'] = username - lookup_key = _make_context_key(username, transfer.get('filename', '')) - live_transfers_lookup[lookup_key] = transfer - - transfer_data_cache['data'] = live_transfers_lookup - transfer_data_cache['last_update'] = current_time - except Exception as e: - logger.warning("Could not fetch live transfers (cached): %s", e) - return {} - - return live_transfers_lookup - - -def _regenerate_batch_m3u(batch, tracks): - """Regenerate an exported playlist M3U after downloads finish.""" - try: - from collections import defaultdict - from difflib import SequenceMatcher - - try: - from unidecode import unidecode as _unidecode - except ImportError: - _unidecode = lambda x: x - - from core.imports.paths import sanitize_filename as _sanitize_filename - - def _norm(text): - return _unidecode(text).lower().strip() if text else '' - - def _track_name(track): - return track.get('name') or track.get('title') or 'Unknown' - - def _track_artist(track): - return track.get('artist') or track.get('artist_name') or 'Unknown' - - playlist_name = batch.get('playlist_name', 'Playlist') - db = get_database() - active_server = config_manager.get_active_media_server() - raw_base = config_manager.get('m3u_export.entry_base_path', '') or '' - entry_base_path = raw_base.rstrip('/\\') - - artist_groups = defaultdict(list) - for idx, track in enumerate(tracks): - artist_groups[_track_artist(track)].append((idx, track)) - - file_path_map = {} - for artist, group in artist_groups.items(): - if not artist or artist == 'Unknown': - for idx, _ in group: - file_path_map[idx] = None - continue - - try: - db_tracks = db.search_tracks(artist=artist, limit=500, server_source=active_server) - except Exception as search_err: - logger.debug("[M3U] Track lookup failed for %s: %s", artist, search_err) - db_tracks = [] - - if not db_tracks: - for idx, _ in group: - file_path_map[idx] = None - continue - - db_entries = [] - for db_track in db_tracks: - db_title = getattr(db_track, 'title', '') or '' - db_entries.append((_norm(db_title), db_track)) - - for idx, track in group: - name = _track_name(track) - if not name: - file_path_map[idx] = None - continue - - s_norm = _norm(name) - matched = None - for db_n, db_track in db_entries: - if not db_n: - continue - if s_norm == db_n or SequenceMatcher(None, s_norm, db_n).ratio() >= 0.7: - matched = db_track - break - - if matched is None: - file_path_map[idx] = None - else: - file_path = getattr(matched, 'file_path', None) - if file_path is None and isinstance(matched, dict): - file_path = matched.get('file_path') - file_path_map[idx] = file_path or None - - lines = ['#EXTM3U', f'#PLAYLIST:{playlist_name}', f'#GENERATED:{datetime.utcnow().isoformat()}Z', ''] - found = 0 - missing = 0 - for idx, track in enumerate(tracks): - duration_ms = track.get('duration_ms', 0) or 0 - artist = _track_artist(track) - name = _track_name(track) - lines.append(f'#EXTINF:{int(duration_ms / 1000)},{artist} - {name}') - file_path = file_path_map.get(idx) - if file_path: - path = f'{entry_base_path}/{file_path}' if entry_base_path else file_path - lines.append(path.replace('\\', '/')) - found += 1 - else: - lines.append(f'# MISSING: {artist} - {name}') - missing += 1 - lines.append('') - - if found == 0: - logger.info("[M3U] Skipping regeneration for %s: no library paths resolved", playlist_name) - return - - transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) - playlist_dir = _compute_m3u_folder(transfer_dir, 'playlist', playlist_name, '', '', '') - os.makedirs(playlist_dir, exist_ok=True) - m3u_path = os.path.join(playlist_dir, f'{_sanitize_filename(playlist_name)}.m3u') - - with open(m3u_path, 'w', encoding='utf-8') as f: - f.write('\n'.join(lines)) - - logger.info( - "[M3U] Regenerated M3U on batch complete: %s (%s/%s resolved, %s missing)", - m3u_path, - found, - len(tracks), - missing, - ) - except Exception as e: - logger.error("[M3U] Error in _regenerate_batch_m3u: %s", e) - - -def _sanitize_filename(filename: str) -> str: - """Sanitize a filename for filesystem use.""" - sanitized = re.sub(r'[<>:"/\\|?*]', '_', filename or '') - sanitized = re.sub(r'\s+', ' ', sanitized).strip() - sanitized = sanitized.rstrip('. ') or '_' - if re.match(r'^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)', sanitized, re.IGNORECASE): - sanitized = '_' + sanitized - return sanitized[:200] - - -def _compute_m3u_folder(transfer_dir, context_type, playlist_name, artist_name='', album_name='', year=''): - """Compute the target folder for an M3U file using the template system.""" - from core.imports.paths import get_file_path_from_template - - if context_type == 'album' and artist_name and album_name: - template_context = { - 'artist': artist_name, - 'albumartist': artist_name, - 'album': album_name, - 'title': 'placeholder', - 'track_number': 1, - 'disc_number': 1, - 'year': year, - 'quality': '' - } - folder_path, _ = get_file_path_from_template(template_context, 'album_path') - if folder_path: - return os.path.join(transfer_dir, folder_path) - artist_sanitized = _sanitize_filename(artist_name) - album_sanitized = _sanitize_filename(album_name) - return os.path.join(transfer_dir, artist_sanitized, f"{artist_sanitized} - {album_sanitized}") - - template_context = { - 'artist': 'placeholder', - 'albumartist': 'placeholder', - 'album': 'placeholder', - 'title': 'placeholder', - 'playlist_name': playlist_name, - 'track_number': 1, - 'disc_number': 1, - 'year': '', - 'quality': '' - } - folder_path, _ = get_file_path_from_template(template_context, 'playlist_path') - if folder_path: - return os.path.join(transfer_dir, folder_path) - playlist_sanitized = _sanitize_filename(playlist_name) - return os.path.join(transfer_dir, playlist_sanitized) - # --- Restored Web UI Helper State --- session_completed_downloads = 0 session_stats_lock = threading.Lock() @@ -1039,1179 +827,6 @@ _hydrabase_ws = None _hydrabase_lock = threading.Lock() -def get_cached_beatport_data(section_type, data_key, genre_slug=None): - current_time = time.time() - with beatport_data_cache['cache_lock']: - try: - if section_type == 'homepage': - cache_entry = beatport_data_cache['homepage'].get(data_key) - elif section_type == 'genre' and genre_slug: - cache_entry = beatport_data_cache['genre'].get(genre_slug, {}).get(data_key) - else: - return None - - if not cache_entry: - return None - - age = current_time - cache_entry['timestamp'] - if age < cache_entry['ttl'] and cache_entry['data'] is not None: - return cache_entry['data'] - return None - except Exception as e: - print(f"Cache lookup error for {section_type}/{data_key}: {e}") - return None - - -def set_cached_beatport_data(section_type, data_key, data, genre_slug=None): - current_time = time.time() - with beatport_data_cache['cache_lock']: - try: - if section_type == 'homepage': - if data_key in beatport_data_cache['homepage']: - beatport_data_cache['homepage'][data_key]['data'] = data - beatport_data_cache['homepage'][data_key]['timestamp'] = current_time - elif section_type == 'genre' and genre_slug: - if genre_slug not in beatport_data_cache['genre']: - beatport_data_cache['genre'][genre_slug] = {} - if data_key not in beatport_data_cache['genre'][genre_slug]: - beatport_data_cache['genre'][genre_slug][data_key] = { - 'data': None, - 'timestamp': 0, - 'ttl': 600, - } - beatport_data_cache['genre'][genre_slug][data_key]['data'] = data - beatport_data_cache['genre'][genre_slug][data_key]['timestamp'] = current_time - except Exception as e: - print(f"Cache storage error for {section_type}/{data_key}: {e}") - - -def add_cache_headers(response, cache_duration=300): - response.headers['Cache-Control'] = f'public, max-age={cache_duration}' - response.headers['Pragma'] = 'cache' - return response - - -def _get_max_concurrent(): - return config_manager.get('download_source.max_concurrent', 3) - - -def _get_batch_max_concurrent(is_album=False, source=None): - if is_album and source in ('soulseek', None): - if source == 'soulseek': - return 1 - mode = config_manager.get('download_source.mode', 'soulseek') - if mode == 'soulseek': - return 1 - return _get_max_concurrent() - - -def _update_task_status(task_id, new_status): - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = new_status - download_tasks[task_id]['status_change_time'] = time.time() - - -class WebUIDownloadMonitor: - def __init__(self): - self.monitoring = False - self.monitor_thread = None - self.monitored_batches = set() - self._lock = threading.Lock() - - def start_monitoring(self, batch_id): - with self._lock: - self.monitored_batches.add(batch_id) - if not self.monitoring: - self.monitoring = True - self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True) - self.monitor_thread.start() - - def stop_monitoring(self, batch_id): - with self._lock: - self.monitored_batches.discard(batch_id) - if not self.monitored_batches: - self.monitoring = False - - def shutdown(self): - with self._lock: - self.monitoring = False - self.monitored_batches.clear() - self.monitor_thread = None - - def _monitor_loop(self): - while self.monitoring and self.monitored_batches: - try: - if globals().get('IS_SHUTTING_DOWN', False): - self.monitoring = False - break - self._check_all_downloads() - time.sleep(1) - except Exception as e: - if 'interpreter shutdown' in str(e) or 'cannot schedule new futures' in str(e): - self.monitoring = False - break - print(f"Download monitor error: {e}") - - def _check_all_downloads(self): - current_time = time.time() - live_transfers_lookup = self._get_live_transfers() - exhausted_tasks = [] - completed_tasks = [] - deferred_ops = [] - - with tasks_lock: - for batch_id in list(self.monitored_batches): - if batch_id not in download_batches: - self.monitored_batches.discard(batch_id) - continue - - for task_id in download_batches[batch_id].get('queue', []): - task = download_tasks.get(task_id) - if not task or task['status'] not in ['downloading', 'queued']: - continue - - retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) - if retry_exhausted: - exhausted_tasks.append((batch_id, task_id)) - - task_filename = task.get('filename') or task.get('track_info', {}).get('filename') - task_username = task.get('username') or task.get('track_info', {}).get('username') - if task_filename and task_username: - lookup_key = _make_context_key(task_username, task_filename) - live_info = live_transfers_lookup.get(lookup_key) - if live_info: - state = live_info.get('state', '') - has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) - has_completion = ('Completed' in state or 'Succeeded' in state) - if has_completion and not has_error: - expected_size = live_info.get('size', 0) - transferred = live_info.get('bytesTransferred', 0) - if expected_size > 0 and transferred < expected_size: - if not task.get('_incomplete_warned'): - task['_incomplete_warned'] = True - continue - if has_completion and not has_error and task['status'] == 'downloading': - task.pop('_incomplete_warned', None) - task['status'] = 'post_processing' - task['status_change_time'] = current_time - completed_tasks.append((batch_id, task_id)) - - for op in deferred_ops: - try: - if op[0] == 'cancel_download': - _, download_id, username = op - run_async(soulseek_client.cancel_download(download_id, username, remove=True)) - elif op[0] == 'cleanup_orphan': - _, context_key = op - with matched_context_lock: - matched_downloads_context.pop(context_key, None) - elif op[0] == 'restart_worker': - _, task_id, batch_id = op - missing_download_executor.submit(_download_track_worker, task_id, batch_id) - except Exception as e: - print(f"[Deferred] Error executing deferred operation {op[0]}: {e}") - - for batch_id, task_id in completed_tasks: - try: - missing_download_executor.submit(_run_post_processing_worker, task_id, batch_id) - _on_download_completed(batch_id, task_id, success=True) - except Exception as e: - print(f"[Monitor] Error handling completed task {task_id}: {e}") - - for batch_id, task_id in exhausted_tasks: - try: - _on_download_completed(batch_id, task_id, success=False) - except Exception as e: - print(f"[Monitor] Error handling exhausted task {task_id}: {e}") - - self._validate_worker_counts() - - def _get_live_transfers(self): - try: - if not self.monitoring or not soulseek_client: - return {} - - live_transfers = {} - transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) - if transfers_data: - for user_data in transfers_data: - username = user_data.get('username', 'Unknown') - if 'directories' in user_data: - for directory in user_data['directories']: - if 'files' in directory: - for file_info in directory['files']: - key = _make_context_key(username, file_info.get('filename', '')) - live_transfers[key] = file_info - - try: - all_downloads = run_async(soulseek_client.get_all_downloads()) - for download in all_downloads: - if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): - key = _make_context_key(download.username, download.filename) - live_transfers[key] = { - 'id': download.id, - 'filename': download.filename, - 'username': download.username, - 'state': download.state, - 'percentComplete': download.progress, - 'size': download.size, - 'bytesTransferred': download.transferred, - 'averageSpeed': download.speed, - } - except Exception as yt_error: - print(f"Monitor: Could not fetch streaming source downloads: {yt_error}") - - return live_transfers - except Exception as e: - if ('interpreter shutdown' in str(e) or 'cannot schedule new futures' in str(e) or 'Event loop is closed' in str(e)): - self.monitoring = False - return {} - print(f"Monitor: Could not fetch live transfers: {e}") - return {} - - def _should_retry_task(self, task_id, task, live_transfers_lookup, current_time, deferred_ops): - ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} - task_filename = task.get('filename') or ti.get('filename') - task_username = task.get('username') or ti.get('username') - - if not task_filename or not task_username: - return False - - lookup_key = _make_context_key(task_username, task_filename) - live_info = live_transfers_lookup.get(lookup_key) - - if not live_info: - if current_time - task.get('status_change_time', current_time) > 90: - retry_count = task.get('stuck_retry_count', 0) - last_retry = task.get('last_retry_time', 0) - if retry_count < 3 and (current_time - last_retry) > 30: - task['stuck_retry_count'] = retry_count + 1 - task['last_retry_time'] = current_time - download_id = task.get('download_id') - if task_username and download_id: - deferred_ops.append(('cancel_download', download_id, task_username)) - if task_username and task_filename: - used_sources = task.get('used_sources', set()) - source_key = f"{task_username}_{task_filename}" - used_sources.add(source_key) - task['used_sources'] = used_sources - if task_username and task_filename: - _orphaned_download_keys.add(lookup_key) - deferred_ops.append(('cleanup_orphan', lookup_key)) - task.pop('download_id', None) - task.pop('username', None) - task.pop('filename', None) - task['status'] = 'searching' - task.pop('queued_start_time', None) - task.pop('downloading_start_time', None) - task['status_change_time'] = current_time - batch_id = task.get('batch_id') - if task_id and batch_id: - deferred_ops.append(('restart_worker', task_id, batch_id)) - return False - elif retry_count < 3: - return False - else: - track_label = task.get('track_info', {}).get('name', 'Unknown') - tried_sources = task.get('used_sources', set()) - sources_str = f" (tried {len(tried_sources)} source{'s' if len(tried_sources) != 1 else ''})" if tried_sources else '' - task['status'] = 'failed' - task['error_message'] = f'Download disappeared from transfer list 3 times for "{track_label}"{sources_str} — source may be unavailable' - return bool(task.get('batch_id')) - return False - - state_str = live_info.get('state', '') - progress = live_info.get('percentComplete', 0) - - if any(token in state_str for token in ('Errored', 'Failed', 'Rejected', 'TimedOut')): - retry_count = task.get('error_retry_count', 0) - last_retry = task.get('last_error_retry_time', 0) - if retry_count < 3 and (current_time - last_retry) > 5: - task['error_retry_count'] = retry_count + 1 - task['last_error_retry_time'] = current_time - _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} - username = task.get('username') or _ti.get('username') - filename = task.get('filename') or _ti.get('filename') - download_id = task.get('download_id') - if username and download_id: - deferred_ops.append(('cancel_download', download_id, username)) - if username and filename: - used_sources = task.get('used_sources', set()) - used_sources.add(f"{username}_{filename}") - task['used_sources'] = used_sources - if username and filename: - old_context_key = _make_context_key(username, filename) - _orphaned_download_keys.add(old_context_key) - deferred_ops.append(('cleanup_orphan', old_context_key)) - task.pop('download_id', None) - task.pop('username', None) - task.pop('filename', None) - task['status'] = 'searching' - task.pop('queued_start_time', None) - task.pop('downloading_start_time', None) - task['status_change_time'] = current_time - batch_id = task.get('batch_id') - if task_id and batch_id: - deferred_ops.append(('restart_worker', task_id, batch_id)) - return False - elif retry_count < 3: - return False - task['status'] = 'failed' - task['error_message'] = f'Soulseek transfer errored 3 times for "{task.get("track_info", {}).get("name", "Unknown")}"' - return bool(task.get('batch_id')) - - elif 'Queued' in state_str or task['status'] == 'queued': - if 'queued_start_time' not in task: - task['queued_start_time'] = current_time - return False - queue_time = current_time - task['queued_start_time'] - timeout_threshold = 15.0 if task.get('track_info', {}).get('is_album_download', False) else 90.0 - if queue_time > timeout_threshold: - retry_count = task.get('stuck_retry_count', 0) - last_retry = task.get('last_retry_time', 0) - if retry_count < 3 and (current_time - last_retry) > 30: - task['stuck_retry_count'] = retry_count + 1 - task['last_retry_time'] = current_time - _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} - username = task.get('username') or _ti.get('username') - filename = task.get('filename') or _ti.get('filename') - download_id = task.get('download_id') - if username and download_id: - deferred_ops.append(('cancel_download', download_id, username)) - if username and filename: - used_sources = task.get('used_sources', set()) - used_sources.add(f"{username}_{filename}") - task['used_sources'] = used_sources - if username and filename: - old_context_key = _make_context_key(username, filename) - _orphaned_download_keys.add(old_context_key) - deferred_ops.append(('cleanup_orphan', old_context_key)) - task.pop('download_id', None) - task.pop('username', None) - task.pop('filename', None) - task['status'] = 'searching' - task.pop('queued_start_time', None) - task.pop('downloading_start_time', None) - task['status_change_time'] = current_time - batch_id = task.get('batch_id') - if task_id and batch_id: - deferred_ops.append(('restart_worker', task_id, batch_id)) - return False - elif retry_count < 3: - return False - task['status'] = 'failed' - task['error_message'] = f'Download stayed queued too long 3 times for "{task.get("track_info", {}).get("name", "Unknown")}"' - task.pop('queued_start_time', None) - task.pop('downloading_start_time', None) - return bool(task.get('batch_id')) - - elif 'InProgress' in state_str and progress < 1: - if 'downloading_start_time' not in task: - task['downloading_start_time'] = current_time - return False - download_time = current_time - task['downloading_start_time'] - timeout_threshold = 15.0 if task.get('track_info', {}).get('is_album_download', False) else 90.0 - if download_time > timeout_threshold: - retry_count = task.get('stuck_retry_count', 0) - last_retry = task.get('last_retry_time', 0) - if retry_count < 3 and (current_time - last_retry) > 30: - task['stuck_retry_count'] = retry_count + 1 - task['last_retry_time'] = current_time - _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} - username = task.get('username') or _ti.get('username') - filename = task.get('filename') or _ti.get('filename') - download_id = task.get('download_id') - if username and download_id: - deferred_ops.append(('cancel_download', download_id, username)) - if username and filename: - used_sources = task.get('used_sources', set()) - used_sources.add(f"{username}_{filename}") - task['used_sources'] = used_sources - if username and filename: - old_context_key = _make_context_key(username, filename) - _orphaned_download_keys.add(old_context_key) - deferred_ops.append(('cleanup_orphan', old_context_key)) - task.pop('download_id', None) - task.pop('username', None) - task.pop('filename', None) - task['status'] = 'searching' - task.pop('queued_start_time', None) - task.pop('downloading_start_time', None) - task['status_change_time'] = current_time - batch_id = task.get('batch_id') - if task_id and batch_id: - deferred_ops.append(('restart_worker', task_id, batch_id)) - return False - elif retry_count < 3: - return False - task['status'] = 'failed' - task['error_message'] = f'Download stuck at 0% three times for "{task.get("track_info", {}).get("name", "Unknown")}"' - task.pop('queued_start_time', None) - task.pop('downloading_start_time', None) - return bool(task.get('batch_id')) - else: - bytes_transferred = live_info.get('bytesTransferred', 0) - if progress >= 1 or bytes_transferred > 0: - task.pop('queued_start_time', None) - task.pop('downloading_start_time', None) - task.pop('stuck_retry_count', None) - else: - if 'downloading_start_time' not in task: - task['downloading_start_time'] = current_time - download_time = current_time - task['downloading_start_time'] - timeout_threshold = 15.0 if task.get('track_info', {}).get('is_album_download', False) else 90.0 - if download_time > timeout_threshold: - retry_count = task.get('stuck_retry_count', 0) - last_retry = task.get('last_retry_time', 0) - if retry_count < 3 and (current_time - last_retry) > 30: - task['stuck_retry_count'] = retry_count + 1 - task['last_retry_time'] = current_time - _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} - username = task.get('username') or _ti.get('username') - filename = task.get('filename') or _ti.get('filename') - download_id = task.get('download_id') - if username and download_id: - deferred_ops.append(('cancel_download', download_id, username)) - if username and filename: - used_sources = task.get('used_sources', set()) - used_sources.add(f"{username}_{filename}") - task['used_sources'] = used_sources - if username and filename: - old_context_key = _make_context_key(username, filename) - _orphaned_download_keys.add(old_context_key) - deferred_ops.append(('cleanup_orphan', old_context_key)) - task.pop('download_id', None) - task.pop('username', None) - task.pop('filename', None) - task['status'] = 'searching' - task.pop('queued_start_time', None) - task.pop('downloading_start_time', None) - task['status_change_time'] = current_time - batch_id = task.get('batch_id') - if task_id and batch_id: - deferred_ops.append(('restart_worker', task_id, batch_id)) - return False - elif retry_count >= 3: - task['status'] = 'failed' - task['error_message'] = f'Download stuck in "{state_str}" state 3 times for "{task.get("track_info", {}).get("name", "Unknown")}"' - task.pop('queued_start_time', None) - task.pop('downloading_start_time', None) - return bool(task.get('batch_id')) - - return False - - def _validate_worker_counts(self): - try: - batches_needing_workers = [] - with tasks_lock: - for batch_id in list(self.monitored_batches): - if batch_id not in download_batches: - continue - batch = download_batches[batch_id] - reported_active = batch['active_count'] - max_concurrent = batch['max_concurrent'] - queue = batch.get('queue', []) - queue_index = batch.get('queue_index', 0) - actually_active = 0 - orphaned_tasks = [] - completed_task_ids = batch.get('_completed_task_ids', set()) - for task_id in queue: - if task_id in download_tasks: - task_status = download_tasks[task_id]['status'] - if task_status in ['searching', 'downloading', 'queued', 'post_processing']: - if task_id not in completed_task_ids: - actually_active += 1 - elif task_status in ['failed', 'completed', 'cancelled', 'not_found'] and task_id in queue[queue_index:]: - orphaned_tasks.append(task_id) - if reported_active != actually_active or orphaned_tasks: - if reported_active != actually_active: - batch['active_count'] = actually_active - if actually_active < max_concurrent and queue_index < len(queue): - batches_needing_workers.append(batch_id) - for batch_id in batches_needing_workers: - try: - _start_next_batch_of_downloads(batch_id) - except Exception as e: - print(f"[Worker Validation] Error starting workers for {batch_id}: {e}") - except Exception as validation_error: - print(f"Error in worker count validation: {validation_error}") - - -download_monitor = WebUIDownloadMonitor() - - -def _is_explicit_blocked(track_data): - if config_manager.get('content_filter.allow_explicit', True): - return False - if track_data.get('explicit', False): - return True - sp_data = track_data.get('spotify_data', {}) - if isinstance(sp_data, str): - try: - sp_data = json.loads(sp_data) - except Exception: - sp_data = {} - return sp_data.get('explicit', False) - - -def fix_artist_image_url(thumb_url): - if not thumb_url: - return None - - try: - needs_fixing = ( - thumb_url.startswith('http://localhost:') or - thumb_url.startswith('https://localhost:') or - thumb_url.startswith('/library/') or - thumb_url.startswith('/Items/') or - thumb_url.startswith('/api/') or - thumb_url.startswith('/rest/') - ) - - if needs_fixing: - active_server = config_manager.get_active_media_server() - if active_server == 'plex': - plex_config = config_manager.get_plex_config() - plex_base_url = plex_config.get('base_url', '') - plex_token = plex_config.get('token', '') - if plex_base_url and plex_token: - if thumb_url.startswith('/library/'): - path = thumb_url - else: - from urllib.parse import urlparse - path = urlparse(thumb_url).path - return f"{plex_base_url.rstrip('/')}{path}?X-Plex-Token={plex_token}" - - elif active_server == 'jellyfin': - jellyfin_config = config_manager.get_jellyfin_config() - jellyfin_base_url = jellyfin_config.get('base_url', '') - jellyfin_token = jellyfin_config.get('api_key', '') - if jellyfin_base_url: - if thumb_url.startswith('/Items/') or thumb_url.startswith('/api/'): - path = thumb_url - else: - from urllib.parse import urlparse - path = urlparse(thumb_url).path - if jellyfin_token: - separator = '&' if '?' in path else '?' - return f"{jellyfin_base_url.rstrip('/')}{path}{separator}X-Emby-Token={jellyfin_token}" - return f"{jellyfin_base_url.rstrip('/')}{path}" - - elif active_server == 'navidrome': - navidrome_config = config_manager.get_navidrome_config() - navidrome_base_url = navidrome_config.get('base_url', '') - navidrome_username = navidrome_config.get('username', '') - navidrome_password = navidrome_config.get('password', '') - if navidrome_base_url and navidrome_username and navidrome_password: - if thumb_url.startswith('/rest/'): - path = thumb_url - else: - from urllib.parse import urlparse - path = urlparse(thumb_url).path - import hashlib - import secrets - salt = secrets.token_hex(6) - token = hashlib.md5((navidrome_password + salt).encode()).hexdigest() - separator = '&' if '?' in path else '?' - auth_params = f"u={navidrome_username}&t={token}&s={salt}&v=1.16.1&c=SoulSync&f=json" - return f"{navidrome_base_url.rstrip('/')}{path}{separator}{auth_params}" - - return thumb_url - except Exception as e: - print(f"Error fixing image URL '{thumb_url}': {e}") - return thumb_url - - -def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): - import os - import re - from difflib import SequenceMatcher - from unidecode import unidecode - - if '||' in api_filename: - _, title = api_filename.split('||', 1) - api_filename = title - - def normalize_for_finding(text: str) -> str: - if not text: - return "" - text = unidecode(text).lower() - text = re.sub(r'[._/]', ' ', text) - text = re.sub(r'[\[\(].*?[\]\)]', '', text) - text = re.sub(r'[^a-z0-9\s-]', '', text) - return ' '.join(text.split()).strip() - - def _path_matches_api_dirs(file_path): - path_parts = set(p.lower() for p in file_path.replace('\\', '/').split('/')) - return all(d in path_parts for d in api_dir_parts) - - def search_in_directory(search_dir, location_name): - best_fuzzy_path = None - highest_fuzzy_similarity = 0.0 - exact_matches = [] - for root, dirs, files in os.walk(search_dir): - dirs[:] = [d for d in dirs if d != 'ss_quarantine'] - for file in files: - if os.path.basename(file) == target_basename: - file_path = os.path.join(root, file) - if api_dir_parts and _path_matches_api_dirs(file_path): - return file_path, 1.0 - if not api_dir_parts: - return file_path, 1.0 - exact_matches.append(file_path) - continue - - file_stem, file_ext_part = os.path.splitext(file) - stripped_stem = re.sub(r'_\d{10,}$', '', file_stem) - if stripped_stem != file_stem and stripped_stem + file_ext_part == target_basename: - file_path = os.path.join(root, file) - if api_dir_parts and _path_matches_api_dirs(file_path): - return file_path, 1.0 - if not api_dir_parts: - return file_path, 1.0 - exact_matches.append(file_path) - continue - - normalized_file = normalize_for_finding(file) - similarity = SequenceMatcher(None, normalized_target, normalized_file).ratio() - if similarity > highest_fuzzy_similarity: - highest_fuzzy_similarity = similarity - best_fuzzy_path = os.path.join(root, file) - - if exact_matches: - if len(exact_matches) == 1: - return exact_matches[0], 1.0 - best = exact_matches[0] - best_score = -1 - for m in exact_matches: - m_parts = set(p.lower() for p in m.replace('\\', '/').split('/')) - score = sum(1 for d in api_dir_parts if d in m_parts) - if score > best_score: - best_score = score - best = m - return best, 1.0 - - return best_fuzzy_path, highest_fuzzy_similarity - - target_basename = extract_filename(api_filename) - normalized_target = normalize_for_finding(target_basename) - api_path_normalized = api_filename.replace('\\', '/') if api_filename else '' - api_dir_parts = [p.lower() for p in api_path_normalized.split('/')[:-1] if p] - - best_downloads_path, downloads_similarity = search_in_directory(download_dir, 'downloads') - if downloads_similarity > 0.85: - return (best_downloads_path, 'downloads') - - transfer_similarity = 0.0 - if transfer_dir and os.path.exists(transfer_dir): - best_transfer_path, transfer_similarity = search_in_directory(transfer_dir, 'transfer') - if transfer_similarity > 0.85: - return (best_transfer_path, 'transfer') - - return (None, None) - - -def _cleanup_empty_directories(download_path, moved_file_path): - try: - current_dir = os.path.dirname(moved_file_path) - while current_dir != download_path and current_dir.startswith(download_path): - is_empty = not any(not f.startswith('.') for f in os.listdir(current_dir)) - if is_empty: - os.rmdir(current_dir) - current_dir = os.path.dirname(current_dir) - else: - break - except Exception as e: - print(f"Warning: An error occurred during directory cleanup: {e}") - - -def _sweep_empty_download_directories(): - try: - download_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - if not os.path.isdir(download_path): - return 0 - - removed = 0 - for dirpath, _dirnames, _filenames in os.walk(download_path, topdown=False): - if os.path.normpath(dirpath) == os.path.normpath(download_path): - continue - try: - entries = os.listdir(dirpath) - except OSError: - continue - visible = [e for e in entries if not e.startswith('.')] - if not visible: - try: - for hidden in entries: - try: - os.remove(os.path.join(dirpath, hidden)) - except Exception: - pass - os.rmdir(dirpath) - removed += 1 - except OSError: - pass - - return removed - except Exception as e: - print(f"[Folder Cleanup] Error sweeping empty directories: {e}") - return 0 - - -def _get_audio_quality_string(file_path): - try: - ext = os.path.splitext(file_path)[1].lower() - if ext == '.flac': - audio = FLAC(file_path) - return f"FLAC {audio.info.bits_per_sample}bit" - if ext == '.mp3': - from mutagen.mp3 import MP3, BitrateMode - audio = MP3(file_path) - bitrate_kbps = audio.info.bitrate // 1000 - if audio.info.bitrate_mode == BitrateMode.VBR: - return "MP3-VBR" - return f"MP3-{bitrate_kbps}" - if ext in ('.m4a', '.aac', '.mp4'): - audio = MP4(file_path) - return f"M4A-{audio.info.bitrate // 1000}" - if ext == '.ogg': - audio = OggVorbis(file_path) - return f"OGG-{audio.info.bitrate // 1000}" - if ext == '.opus': - from mutagen.oggopus import OggOpus - audio = OggOpus(file_path) - return f"OPUS-{audio.info.bitrate // 1000}" - return '' - except Exception as e: - logger.debug(f"Could not determine audio quality for {file_path}: {e}") - return '' - - -def parse_youtube_playlist(url): - try: - import re - - def clean_youtube_artist(artist_string): - if not artist_string: - return artist_string - original_artist = artist_string - artist_string = artist_string.replace('"', '').replace("'", '') - artist_string = re.sub(r'\s*\([^)]*\)', '', artist_string).strip() - artist_string = re.sub(r'\s*\[[^\]]*\]', '', artist_string).strip() - for suffix in [ - r'\s*-\s*Topic\s*$', - r'\s*VEVO\s*$', - r'\s*Music\s*$', - r'\s*Official\s*$', - r'\s*Records\s*$', - r'\s*Entertainment\s*$', - r'\s*TV\s*$', - r'\s*Channel\s*$', - ]: - artist_string = re.sub(suffix, '', artist_string, flags=re.IGNORECASE).strip() - for sep in [',', '&', ' and ', ' x ', ' X ', ' feat.', ' ft.', ' featuring', ' with', ' vs ', ' vs.']: - if sep in artist_string: - artist_string = artist_string.split(sep)[0].strip() - break - artist_string = re.sub(r'\s+', ' ', artist_string).strip() - artist_string = re.sub(r'^\-\s*|\s*\-$', '', artist_string).strip() - artist_string = re.sub(r'^,\s*|\s*,$', '', artist_string).strip() - return artist_string or original_artist - - def clean_youtube_track_title(title, artist_name=None): - if not title: - return title - original_title = title - artist_removed = False - if artist_name and '-' in title: - artist_pattern = r'^' + re.escape(artist_name.strip()) + r'(?:\s*[&,x]\s*[^-]+)?\s*[-–—]\s*' - cleaned_title = re.sub(artist_pattern, '', title, flags=re.IGNORECASE).strip() - if cleaned_title != title: - title = cleaned_title - artist_removed = True - else: - artist_end_pattern = r'\s*[-–—]\s*' + re.escape(artist_name.strip()) + r'(?:\s*[&,x]\s*[^-]+)?\s*$' - cleaned_title = re.sub(artist_end_pattern, '', title, flags=re.IGNORECASE).strip() - if cleaned_title != title: - title = cleaned_title - artist_removed = True - title = re.sub(r'【[^】]*】', '', title) - title = re.sub(r'\s*\([^)]*\)', '', title) - title = re.sub(r'\s*\(.*$', '', title) - title = re.sub(r'\[[^\]]*\]', '', title) - title = re.sub(r'\{[^}]*\}', '', title) - title = re.sub(r'<[^>]*>', '', title) - if artist_removed: - title = re.sub(r'\s*-\s*.*$', '', title) - title = re.split(r'\s*\|\s*', title)[0].strip() - for pattern in [ - r'\bapple\s+music\b', - r'\bfull\s+video\b', - r'\bmusic\s+video\b', - r'\bofficial\s+video\b', - r'\bofficial\s+music\s+video\b', - r'\bofficial\b', - r'\bcensored\s+version\b', - r'\buncensored\s+version\b', - r'\bexplicit\s+version\b', - r'\blive\s+version\b', - r'\bversion\b', - r'\btopic\b', - r'\baudio\b', - r'\blyrics?\b', - r'\blyric\s+video\b', - r'\bwith\s+lyrics?\b', - r'\bvisuali[sz]er\b', - r'\bmv\b', - r'\bdirectors?\s+cut\b', - r'\bremaster(ed)?\b', - r'\bremix\b', - ]: - title = re.sub(pattern, '', title, flags=re.IGNORECASE) - if artist_name: - collab_pattern = rf'\b{re.escape(artist_name)}\s*[&,]\s*\w+|[\w\s]+[&,]\s*{re.escape(artist_name)}\b' - if not re.search(collab_pattern, title, flags=re.IGNORECASE): - title = re.sub(rf'\b{re.escape(artist_name)}\b', '', title, flags=re.IGNORECASE) - title = re.sub(rf'\b{re.escape(artist_name)}\s*[-–—:]\s*', '', title, flags=re.IGNORECASE) - title = re.sub(rf'^{re.escape(artist_name)}\s*[-–—:]\s*', '', title, flags=re.IGNORECASE) - title = re.sub(r'\s+prod\.?\s+\S+', '', title, flags=re.IGNORECASE) - title = re.sub(r'["\']', '', title) - for pattern in [ - r'\s+feat\.?\s+.+$', - r'\s+ft\.?\s+.+$', - r'\s+featuring\s+.+$', - r'\s+with\s+.+$', - ]: - title = re.sub(pattern, '', title, flags=re.IGNORECASE).strip() - title = re.sub(r'\s+', ' ', title).strip() - title = re.sub(r'^[-–—:,.\s]+|[-–—:,.\s]+$', '', title).strip() - return title or original_title - - ydl_opts = { - 'quiet': True, - 'no_warnings': True, - 'extract_flat': 'in_playlist', - 'skip_download': True, - 'lazy_playlist': False, - } - - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - playlist_info = ydl.extract_info(url, download=False) - if not playlist_info: - return None - - playlist_name = playlist_info.get('title', 'Unknown Playlist') - playlist_id = playlist_info.get('id', 'unknown_id') - entries = list(playlist_info.get('entries', []) or []) - tracks = [] - - for entry in entries: - if not entry: - continue - raw_title = entry.get('title', 'Unknown Track') - raw_uploader = entry.get('uploader', 'Unknown Artist') - duration = entry.get('duration', 0) - video_id = entry.get('id', '') - cleaned_artist = clean_youtube_artist(raw_uploader) - cleaned_title = clean_youtube_track_title(raw_title, cleaned_artist) - tracks.append({ - 'id': video_id, - 'name': cleaned_title, - 'artists': [cleaned_artist], - 'duration_ms': duration * 1000 if duration else 0, - 'raw_title': raw_title, - 'raw_artist': raw_uploader, - 'url': f"https://www.youtube.com/watch?v={video_id}", - }) - - return { - 'id': playlist_id, - 'name': playlist_name, - 'tracks': tracks, - 'track_count': len(tracks), - 'url': url, - 'source': 'youtube', - } - except Exception as e: - print(f"Error parsing YouTube playlist: {e}") - return None - - -def get_download_status(): - if not soulseek_client: - return jsonify({"transfers": []}) - - try: - global processed_download_ids - transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) - all_transfers = [] - completed_matched_downloads = [] - _files_claimed_this_cycle = set() - - if transfers_data: - for user_data in transfers_data: - username = user_data.get('username', 'Unknown') - if 'directories' in user_data: - for directory in user_data['directories']: - if 'files' in directory: - for file_info in directory['files']: - file_info['username'] = username - all_transfers.append(file_info) - state = file_info.get('state', '').lower() - if ('succeeded' in state or 'completed' in state) and 'errored' not in state and 'rejected' not in state: - _fi_size = file_info.get('size', 0) - _fi_transferred = file_info.get('bytesTransferred', 0) - if _fi_size > 0 and _fi_transferred < _fi_size: - continue - filename_from_api = file_info.get('filename') - if not filename_from_api: - continue - - context_key = _make_context_key(username, filename_from_api) - if context_key in _orphaned_download_keys: - with matched_context_lock: - has_active_context = context_key in matched_downloads_context - if has_active_context: - _orphaned_download_keys.discard(context_key) - else: - download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - found_result = _find_completed_file_robust(download_dir, filename_from_api) - found_path = found_result[0] if found_result and found_result[0] else None - orphan_cleaned = False - if found_path: - try: - os.remove(found_path) - orphan_cleaned = True - except Exception as e: - print(f"Failed to delete orphaned file (will retry next poll): {e}") - else: - orphan_cleaned = True - if orphan_cleaned: - transfer_id = file_info.get('id') - if transfer_id: - try: - run_async(soulseek_client.cancel_download(str(transfer_id), username, remove=True)) - except Exception: - pass - _orphaned_download_keys.discard(context_key) - continue - - if context_key in processed_download_ids: - continue - - with matched_context_lock: - context = matched_downloads_context.get(context_key) - available_keys = list(matched_downloads_context.keys())[:5] if not context else None - - if context and context_key not in _stale_transfer_keys: - pass - - if context: - download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - found_result = _find_completed_file_robust(download_dir, filename_from_api) - found_path = found_result[0] if found_result and found_result[0] else None - - if found_path: - _norm_path = os.path.normpath(found_path) - if _norm_path not in _files_claimed_this_cycle: - _files_claimed_this_cycle.add(_norm_path) - completed_matched_downloads.append((context_key, context, found_path)) - with _download_retry_lock: - if context_key in _download_retry_attempts: - del _download_retry_attempts[context_key] - else: - with _download_retry_lock: - if context_key not in _download_retry_attempts: - _download_retry_attempts[context_key] = {'count': 1, 'first_attempt': time.time()} - else: - _download_retry_attempts[context_key]['count'] += 1 - retry_count = _download_retry_attempts[context_key]['count'] - if retry_count >= _download_retry_max: - processed_download_ids.add(context_key) - del _download_retry_attempts[context_key] - if completed_matched_downloads: - def process_completed_downloads(): - for context_key, context, found_path in completed_matched_downloads: - try: - _pp_task_id = context.get('task_id') - _pp_batch_id = context.get('batch_id') - if _pp_task_id and _pp_batch_id: - _pp_target = _post_process_matched_download_with_verification - _pp_args = (context_key, context, found_path, _pp_task_id, _pp_batch_id) - else: - _pp_target = _post_process_matched_download - _pp_args = (context_key, context, found_path) - thread = threading.Thread(target=_pp_target, args=_pp_args) - thread.daemon = True - thread.start() - processed_download_ids.add(context_key) - except Exception as e: - print(f"Error starting post-processing thread for {context_key}: {e}") - processing_thread = threading.Thread(target=process_completed_downloads) - processing_thread.daemon = True - processing_thread.start() - - return jsonify({"transfers": all_transfers}) - except Exception as e: - logger.error(f"Error building download status: {e}") - return jsonify({"error": str(e)}), 500 - - -def _get_windowed_calls(key, current_total): - now = time.time() - history = _enrichment_activity_log.setdefault(key, collections.deque(maxlen=17300)) - history.append((now, current_total)) - - cutoff_1h = now - 3600 - cutoff_24h = now - 86400 - oldest_1h_total = current_total - oldest_24h_total = current_total - found_24h = False - for ts, total in history: - if not found_24h and ts >= cutoff_24h: - oldest_24h_total = total - found_24h = True - if ts >= cutoff_1h: - oldest_1h_total = total - break - - return max(0, current_total - oldest_1h_total), max(0, current_total - oldest_24h_total) - - -def _get_enrichment_status(): - services = {} - workers_info = [ - ('musicbrainz', 'MusicBrainz', lambda: mb_worker), - ('spotify_enrichment', 'Spotify', lambda: spotify_enrichment_worker), - ('itunes_enrichment', 'iTunes', lambda: itunes_enrichment_worker), - ('deezer_enrichment', 'Deezer', lambda: deezer_worker), - ('tidal_enrichment', 'Tidal', lambda: tidal_enrichment_worker), - ('qobuz_enrichment', 'Qobuz', lambda: qobuz_enrichment_worker), - ('lastfm', 'Last.fm', lambda: lastfm_worker), - ('genius', 'Genius', lambda: genius_worker), - ('audiodb', 'AudioDB', lambda: audiodb_worker), - ('discogs', 'Discogs', lambda: discogs_worker), - ] - configured_checks = { - 'spotify_enrichment': lambda: bool(config_manager.get('spotify.client_id') and config_manager.get('spotify.client_secret')), - 'tidal_enrichment': lambda: bool(tidal_client and getattr(tidal_client, 'access_token', None)), - 'qobuz_enrichment': lambda: bool(qobuz_enrichment_worker and qobuz_enrichment_worker.client and qobuz_enrichment_worker.client.user_auth_token), - 'lastfm': lambda: bool(config_manager.get('lastfm.api_key', '')), - 'genius': lambda: bool(config_manager.get('genius.access_token', '')), - } - - for key, name, get_worker in workers_info: - worker = get_worker() - if worker is not None: - is_alive = worker.thread is not None and worker.thread.is_alive() - try: - configured = configured_checks.get(key, lambda: True)() - except Exception: - configured = False - - stats = worker.stats - total_processed = stats.get('matched', 0) + stats.get('not_found', 0) + stats.get('errors', 0) - calls_1h, calls_24h = _get_windowed_calls(key, total_processed) - - has_item = getattr(worker, 'current_item', None) is not None - if has_item: - _idle_since.pop(key, None) - is_idle = False - else: - if key not in _idle_since: - _idle_since[key] = time.time() - is_idle = (time.time() - _idle_since[key]) >= _IDLE_GRACE_SECONDS - - svc_data = { - 'name': name, - 'configured': configured, - 'running': worker.running and is_alive and not worker.paused, - 'paused': worker.paused, - 'idle': is_alive and not worker.paused and is_idle, - 'calls_1h': calls_1h, - 'calls_24h': calls_24h, - } - if key == 'spotify_enrichment': - try: - svc_data['daily_budget'] = worker._get_daily_budget_info() - except Exception: - pass - services[key] = svc_data - else: - services[key] = { - 'name': name, - 'configured': False, - 'running': False, - 'paused': False, - 'idle': False, - 'calls_1h': 0, - 'calls_24h': 0, - } - - services['acoustid'] = { - 'name': 'AcoustID', - 'configured': bool(config_manager.get('acoustid.api_key', '')), - } - services['listenbrainz'] = { - 'name': 'ListenBrainz', - 'configured': bool(config_manager.get('listenbrainz.token', '')), - } - return services - - -def _build_system_stats(): - try: - import psutil - except Exception: - psutil = None - - from datetime import timedelta - - start_time = getattr(app, 'start_time', time.time()) - uptime = str(timedelta(seconds=int(time.time() - start_time))) - memory_usage = '0%' - if psutil is not None: - memory_usage = f"{psutil.virtual_memory().percent}%" - - active_downloads = len([batch_id for batch_id, batch_data in download_batches.items() if batch_data.get('phase') == 'downloading']) - with session_stats_lock: - finished_downloads = session_completed_downloads - - total_download_speed = 0.0 - try: - transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) - if transfers_data: - for user_data in transfers_data: - if 'directories' in user_data: - for directory in user_data['directories']: - if 'files' in directory: - for file_info in directory['files']: - state = file_info.get('state', '').lower() - if 'inprogress' in state or 'downloading' in state or 'transferring' in state: - speed = file_info.get('averageSpeed', 0) - if isinstance(speed, (int, float)) and speed > 0: - total_download_speed += float(speed) - except Exception: - pass - - return { - 'uptime': uptime, - 'memory_usage': memory_usage, - 'active_downloads': active_downloads, - 'finished_downloads': finished_downloads, - 'download_speed': f"{total_download_speed / (1024 * 1024):.1f} MB/s" if total_download_speed > 1024 * 1024 else f"{total_download_speed / 1024:.1f} KB/s", - } - - -def _is_hydrabase_active(): - try: - if hydrabase_client is None or not hydrabase_client.is_connected(): - return False - return dev_mode_enabled - except (NameError, Exception): - return False - # --- Automation Engine --- try: automation_engine = AutomationEngine(get_database()) @@ -2221,46 +836,8 @@ except Exception as e: automation_engine = None # --- Automation Progress Tracking --- -automation_progress_states = {} # automation_id (int) -> state dict -automation_progress_lock = threading.Lock() _scan_library_automation_id = None -def _init_automation_progress(automation_id, automation_name, action_type): - """Initialize progress state when an automation starts running.""" - with automation_progress_lock: - automation_progress_states[automation_id] = { - 'status': 'running', - 'action_type': action_type, - 'progress': 0, 'phase': 'Starting...', 'current_item': '', - 'processed': 0, 'total': 0, - 'log': [{'type': 'info', 'text': f'Starting {automation_name}'}], - 'started_at': datetime.now().isoformat(), - 'finished_at': None, - } - - -def _update_automation_progress(automation_id, **kwargs): - """Update progress state from handler threads. Thread-safe.""" - if automation_id is None: - return - with automation_progress_lock: - state = automation_progress_states.get(automation_id) - if not state: - return - for k, v in kwargs.items(): - if k == 'log_line': - state['log'].append({'type': kwargs.get('log_type', 'info'), 'text': v}) - if len(state['log']) > 50: - state['log'] = state['log'][-50:] - elif k != 'log_type': - state[k] = v - # Immediate emit on finish so frontend gets final state without waiting for loop - if kwargs.get('status') in ('finished', 'error'): - state['finished_at'] = datetime.now().isoformat() - try: - socketio.emit('automation:progress', {str(automation_id): dict(state)}) - except Exception: - pass def _register_automation_handlers(): """Register real SoulSync action handlers with the automation engine.""" @@ -3842,92 +2419,18 @@ except Exception as e: # --- Global Streaming State Management --- # Thread-safe state tracking for streaming functionality -stream_state = { - "status": "stopped", # States: stopped, loading, queued, ready, error - "progress": 0, - "track_info": None, - "file_path": None, # Path to the audio file in the 'Stream' folder - "error_message": None -} -stream_lock = threading.Lock() # Prevent race conditions -stream_background_task = None -stream_executor = ThreadPoolExecutor(max_workers=1) # Only one stream at a time # --- Global OAuth State Management --- # Store PKCE values for Tidal OAuth flow -tidal_oauth_state = { - "code_verifier": None, - "code_challenge": None -} -tidal_oauth_lock = threading.Lock() -db_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DBUpdate") -db_update_worker = None -db_update_state = { - "status": "idle", # idle, running, finished, error - "phase": "Idle", - "progress": 0, - "current_item": "", - "processed": 0, - "total": 0, - "error_message": "", - "removed_artists": 0, - "removed_albums": 0, - "removed_tracks": 0 -} -_db_update_automation_id = None # Set when automation triggers DB update, used by callbacks -_scan_library_automation_id = None # Set when automation triggers library scan, used for tracking # Quality Scanner state -quality_scanner_state = { - "status": "idle", # idle, running, finished, error - "phase": "Ready to scan", - "progress": 0, - "processed": 0, - "total": 0, - "quality_met": 0, - "low_quality": 0, - "matched": 0, - "error_message": "", - "results": [] # List of low quality tracks with match status -} -quality_scanner_lock = threading.Lock() -quality_scanner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="QualityScanner") # Duplicate Cleaner state -duplicate_cleaner_state = { - "status": "idle", # idle, running, finished, error - "phase": "Ready to scan", - "progress": 0, - "files_scanned": 0, - "total_files": 0, - "duplicates_found": 0, - "deleted": 0, - "space_freed": 0, # in bytes - "error_message": "" -} -duplicate_cleaner_lock = threading.Lock() -duplicate_cleaner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DuplicateCleaner") # --- Retag Tool Globals --- -retag_state = { - "status": "idle", - "phase": "Ready", - "progress": 0, - "current_track": "", - "total_tracks": 0, - "processed": 0, - "error_message": "" -} -retag_lock = threading.Lock() -retag_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="RetagWorker") # --- Sync Page Globals --- -sync_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="SyncWorker") -active_sync_workers = {} # Key: playlist_id, Value: Future object -sync_states = {} # Key: playlist_id, Value: dict with progress info -sync_lock = threading.Lock() -db_update_lock = threading.Lock() # --- Automation Progress Tracking --- automation_progress_states = {} # automation_id (int) -> state dict @@ -3972,12 +2475,9 @@ def _update_automation_progress(automation_id, **kwargs): # --- Global Matched Downloads Context Management --- # Shared with core.runtime_state so the refactored pipeline and web # server operate on the same context registry. -_orphaned_download_keys = set() # Context keys of downloads abandoned during retry # --- Download Missing Tracks Modal State Management --- # Thread-safe state tracking for modal download functionality with batch management -missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker") -batch_locks = {} # batch_id -> Lock() for atomic batch operations def _get_max_concurrent(): """Get configured max concurrent downloads. Default 3.""" @@ -3998,8 +2498,6 @@ def _get_batch_max_concurrent(is_album=False, source=None): # --- Session Download Statistics --- # Track individual download completions (matches dashboard.py behavior) -session_completed_downloads = 0 -session_stats_lock = threading.Lock() def _mark_task_completed(task_id, track_info=None): """ @@ -4018,22 +2516,10 @@ def _mark_task_completed(task_id, track_info=None): # Server-side timer system for automatic wishlist processing (replaces client-side JavaScript timers) # --- Automatic Wishlist/Watchlist Processing Flags --- # Processing state flags (guards/recovery — timers are now managed by AutomationEngine) -wishlist_auto_processing = False # Flag to prevent concurrent auto-processing -wishlist_auto_processing_timestamp = 0 # Timestamp when processing started (for stuck detection) -wishlist_timer_lock = threading.Lock() # Thread safety for flag operations -watchlist_auto_scanning = False # Flag to prevent concurrent auto-scanning -watchlist_auto_scanning_timestamp = 0 # Timestamp when scanning started (for stuck detection) -watchlist_timer_lock = threading.Lock() # Thread safety for flag operations # --- Shared Transfer Data Cache --- # Cache transfer data to avoid hammering the Soulseek API with multiple concurrent modals -transfer_data_cache = { - 'data': {}, - 'last_update': 0, - 'update_lock': threading.Lock(), - 'cache_duration': 0.75 # Cache for 0.75 seconds for faster updates -} def get_cached_transfer_data(): """ @@ -4115,23 +2601,6 @@ def get_cached_transfer_data(): # --- Beatport Data Cache --- # Cache Beatport scraping data to reduce load times and avoid hammering Beatport.com -beatport_data_cache = { - 'homepage': { - 'hero_tracks': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours - 'top_10_lists': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours - 'top_10_releases': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours - 'new_releases': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours - 'hype_picks': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours - 'featured_charts': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours - 'dj_charts': {'data': None, 'timestamp': 0, 'ttl': 86400} # 24 hours - }, - 'genre': { - # Future expansion for genre-specific caching - # 'house': {'top_10': {...}, 'releases': {...}}, - # 'techno': {'top_10': {...}, 'releases': {...}} - }, - 'cache_lock': threading.Lock() -} @@ -5242,7 +3711,6 @@ def cleanup_monitor(): logger.error(f"Error cleaning up batch locks: {e}") # Global shutdown flag -IS_SHUTTING_DOWN = False def _shutdown_executor(executor, name): """Shut down a ThreadPoolExecutor without waiting for long-running tasks.""" @@ -6536,7 +5004,6 @@ def spa_catch_all(page): # Tracks cumulative item-processed totals over time for windowed counting. # Each entry: (timestamp, cumulative_total). Polled every ~5s, 24h = ~17280 entries. -_enrichment_activity_log = {} # key -> deque of (timestamp, total) def _get_windowed_calls(key, current_total): """Record current cumulative total and return (calls_1h, calls_24h). @@ -6564,8 +5031,6 @@ def _get_windowed_calls(key, current_total): return max(0, current_total - oldest_1h_total), max(0, current_total - oldest_24h_total) -_idle_since = {} # worker_key -> timestamp when current_item first became None -_IDLE_GRACE_SECONDS = 5 # Don't report idle until current_item has been None this long def _get_enrichment_status(): """Get lightweight status for all enrichment services (no DB queries). @@ -6662,17 +5127,6 @@ def _get_enrichment_status(): # Status check caching to reduce unnecessary API calls -_status_cache = { - 'spotify': {'connected': False, 'response_time': 0, 'source': 'itunes'}, - 'media_server': {'connected': False, 'response_time': 0, 'type': None}, - 'soulseek': {'connected': False, 'response_time': 0} -} -_status_cache_timestamps = { - 'spotify': 0, - 'media_server': 0, - 'soulseek': 0 -} -STATUS_CACHE_TTL = 120 # Cache for 2 minutes (reduces API calls while staying fresh) @app.route('/status') def get_status(): @@ -7698,7 +6152,6 @@ def handle_settings(): except Exception as e: return jsonify({"error": str(e)}), 500 -dev_mode_enabled = False @app.route('/api/dev-mode', methods=['GET', 'POST']) def handle_dev_mode(): @@ -7713,8 +6166,6 @@ def handle_dev_mode(): return jsonify({"enabled": dev_mode_enabled}) # ── Hydrabase WebSocket Connection ── -_hydrabase_ws = None -_hydrabase_lock = threading.Lock() # ── Hydrabase Comparison Store ── import collections as _collections From d04573f397e24ddd07af8f2e0ff667a9a3bb85ce Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sat, 25 Apr 2026 20:16:29 +0300 Subject: [PATCH 13/19] Fix single import source handling - pass the selected manual match through singles import - keep the import context source-aware so artist and album stay correct - avoid treating non-Spotify IDs as wishlist Spotify IDs - make wishlist logging and local variable names source-neutral --- core/imports/resolution.py | 8 ++--- core/imports/side_effects.py | 53 +++++++++++++++++-------------- core/imports/staging.py | 4 +-- web_server.py | 43 +++++++++++++++---------- webui/static/stats-automations.js | 10 +++--- 5 files changed, 69 insertions(+), 49 deletions(-) diff --git a/core/imports/resolution.py b/core/imports/resolution.py index 40e4149c..be55c1f9 100644 --- a/core/imports/resolution.py +++ b/core/imports/resolution.py @@ -119,7 +119,7 @@ def _pick_best_track_match(search_results: List[Any], title: str, artist: str = return search_results[0] -def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]: +def search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]: if not client or not hasattr(client, 'search_tracks'): return [] @@ -366,11 +366,11 @@ def get_single_track_import_context( if not search_query: continue - search_results = _search_tracks_for_source(source, client, search_query, limit=5) + search_results = search_tracks_for_source(source, client, search_query, limit=5) if not search_results and search_query != title: - search_results = _search_tracks_for_source(source, client, title, limit=5) + search_results = search_tracks_for_source(source, client, title, limit=5) if not search_results and artist and search_query != artist: - search_results = _search_tracks_for_source(source, client, artist, limit=5) + search_results = search_tracks_for_source(source, client, artist, limit=5) if not search_results: continue diff --git a/core/imports/side_effects.py b/core/imports/side_effects.py index dbc5bbf7..b44bfe61 100644 --- a/core/imports/side_effects.py +++ b/core/imports/side_effects.py @@ -484,10 +484,10 @@ def record_retag_download(context: Dict[str, Any], artist_context: Dict[str, Any ) file_format = os.path.splitext(str(final_path))[1].lstrip(".").lower() - spotify_track_id = None + source_track_id = None itunes_track_id = None if source == "spotify": - spotify_track_id = source_ids.get("track_id", "") or None + source_track_id = source_ids.get("track_id", "") or None elif source == "itunes": itunes_track_id = source_ids.get("track_id", "") or None @@ -499,7 +499,7 @@ def record_retag_download(context: Dict[str, Any], artist_context: Dict[str, Any title=title, file_path=str(final_path), file_format=file_format, - spotify_track_id=spotify_track_id, + spotify_track_id=source_track_id, itunes_track_id=itunes_track_id, ) logger.info("[Retag] Recorded track for retag: '%s' in '%s'", title, album_name) @@ -513,31 +513,38 @@ def check_and_remove_from_wishlist(context: Dict[str, Any]) -> None: """Check whether a successful download should be removed from the wishlist.""" try: wishlist_service = get_wishlist_service() - spotify_track_id = None - + source = get_import_source(context) + source_ids = get_import_source_ids(context) + source_label = { + "spotify": "Spotify", + "itunes": "iTunes", + "deezer": "Deezer", + "discogs": "Discogs", + "hydrabase": "Hydrabase", + }.get(source, "Source") track_info = context.get("track_info", {}) - if track_info.get("id"): - spotify_track_id = track_info["id"] - logger.info("[Wishlist] Found Spotify ID from track_info: %s", spotify_track_id) - elif context.get("original_search_result", {}).get("id"): - spotify_track_id = context["original_search_result"]["id"] - logger.info("[Wishlist] Found Spotify ID from original_search_result: %s", spotify_track_id) + track_id = None + + if source == "spotify": + track_id = source_ids.get("track_id") or None + if track_id: + logger.info("[Wishlist] Found %s track ID from source_ids: %s", source_label, track_id) elif "wishlist_id" in track_info: wishlist_id = track_info["wishlist_id"] logger.info("[Wishlist] Found wishlist_id in context: %s", wishlist_id) wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service) for wishlist_track in wishlist_tracks: if wishlist_track.get("wishlist_id") == wishlist_id: - spotify_track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id") - logger.info("[Wishlist] Found Spotify ID from wishlist entry: %s", spotify_track_id) + track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id") + logger.info("[Wishlist] Found track ID from wishlist entry: %s", track_id) break - if not spotify_track_id: + if not track_id: track_name = track_info.get("name") or context.get("original_search_result", {}).get("title", "") artist_name = _primary_track_artist_name(track_info) or _primary_track_artist_name(context.get("original_search_result", {})) if track_name and artist_name: - logger.warning("[Wishlist] No Spotify ID found, checking for fuzzy match: '%s' by '%s'", track_name, artist_name) + logger.warning("[Wishlist] No track ID found, checking for fuzzy match: '%s' by '%s'", track_name, artist_name) wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service) for wishlist_track in wishlist_tracks: @@ -550,18 +557,18 @@ def check_and_remove_from_wishlist(context: Dict[str, Any]) -> None: else: wl_artist_name = str(wl_artists[0]).lower() if wl_name == track_name.lower() and wl_artist_name == artist_name.lower(): - spotify_track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id") - logger.info("[Wishlist] Found fuzzy match - Spotify ID: %s", spotify_track_id) + track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id") + logger.info("[Wishlist] Found fuzzy match - track ID: %s", track_id) break - if spotify_track_id: - logger.info("[Wishlist] Attempting to remove track from wishlist: %s", spotify_track_id) - removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) + if track_id: + logger.info("[Wishlist] Attempting to remove track from wishlist: %s", track_id) + removed = wishlist_service.mark_track_download_result(track_id, success=True) if removed: - logger.info("[Wishlist] Successfully removed track from wishlist: %s", spotify_track_id) + logger.info("[Wishlist] Successfully removed track from wishlist: %s", track_id) else: - logger.warning("ℹ️ [Wishlist] Track not found in wishlist or already removed: %s", spotify_track_id) + logger.warning("ℹ️ [Wishlist] Track not found in wishlist or already removed: %s", track_id) else: - logger.warning("ℹ️ [Wishlist] No Spotify track ID found for wishlist removal check") + logger.warning("ℹ️ [Wishlist] No track ID found for wishlist removal check") except Exception as exc: logger.error("[Wishlist] Error in wishlist removal check: %s", exc) diff --git a/core/imports/staging.py b/core/imports/staging.py index 44edb0bf..5843d144 100644 --- a/core/imports/staging.py +++ b/core/imports/staging.py @@ -137,9 +137,9 @@ def _search_albums_for_source(source: str, client: Any, query: str, limit: int = def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 5): - from core.metadata_service import _search_tracks_for_source as _metadata_search_tracks_for_source + from core.imports.resolution import search_tracks_for_source - return _metadata_search_tracks_for_source(source, client, query, limit=limit) + return search_tracks_for_source(source, client, query, limit=limit) def _extract_value(value: Any, *names: str, default: Any = None) -> Any: diff --git a/web_server.py b/web_server.py index 1e036ef6..c2880dfb 100644 --- a/web_server.py +++ b/web_server.py @@ -18625,32 +18625,43 @@ def _check_and_remove_from_wishlist(context): """ try: from core.wishlist_service import get_wishlist_service + from core.imports.context import get_import_source, get_import_source_ids wishlist_service = get_wishlist_service() - # Try to extract Spotify track ID from various sources in the context + # Try to extract a source-aware track ID from the context spotify_track_id = None # Populated lazily by Method 3 or Method 4. Initialized here so Method 4's # `if not wishlist_tracks` guard doesn't UnboundLocalError when Methods 1/2 # found nothing and Method 3 never ran (no wishlist_id in track_info). wishlist_tracks = [] - # Method 1: Direct track_info with id + # Method 1: Source-specific track lookup from track_info / source_ids track_info = context.get('track_info', {}) - if track_info.get('id'): - spotify_track_id = track_info['id'] - logger.info(f"[Wishlist] Found Spotify ID from track_info: {spotify_track_id}") + source = get_import_source(context) + source_ids = get_import_source_ids(context) + source_label = { + 'spotify': 'Spotify', + 'itunes': 'iTunes', + 'deezer': 'Deezer', + 'discogs': 'Discogs', + 'hydrabase': 'Hydrabase', + }.get(source, 'Source') + + if source == 'spotify' and source_ids.get('track_id'): + spotify_track_id = source_ids['track_id'] + logger.info(f"[Wishlist] Found {source_label} track ID from source_ids: {spotify_track_id}") - # Method 2: From original search result - elif context.get('original_search_result', {}).get('id'): + # Method 2: Fallback to the original search result for source-specific IDs + elif source == 'spotify' and context.get('original_search_result', {}).get('id'): spotify_track_id = context['original_search_result']['id'] - logger.info(f"[Wishlist] Found Spotify ID from original_search_result: {spotify_track_id}") + logger.info(f"[Wishlist] Found {source_label} track ID from original_search_result: {spotify_track_id}") # Method 3: Check if this is a wishlist download (context has wishlist_id) elif 'wishlist_id' in track_info: wishlist_id = track_info['wishlist_id'] logger.info(f"[Wishlist] Found wishlist_id in context: {wishlist_id}") - # Get the Spotify track ID from the wishlist entry (search all profiles) + # Get the track ID from the wishlist entry (search all profiles) database = get_database() all_profiles = database.get_all_profiles() wishlist_tracks = [] @@ -18659,16 +18670,16 @@ def _check_and_remove_from_wishlist(context): for wl_track in wishlist_tracks: if wl_track.get('wishlist_id') == wishlist_id: spotify_track_id = wl_track.get('spotify_track_id') or wl_track.get('id') - logger.info(f"[Wishlist] Found Spotify ID from wishlist entry: {spotify_track_id}") + logger.info(f"[Wishlist] Found track ID from wishlist entry: {spotify_track_id}") break - # Method 4: Try to construct ID from track metadata for fuzzy matching + # Method 4: Try to construct a track ID from metadata for fuzzy matching if not spotify_track_id: track_name = track_info.get('name') or context.get('original_search_result', {}).get('title', '') artist_name = _get_track_artist_name(track_info) or _get_track_artist_name(context.get('original_search_result', {})) if track_name and artist_name: - logger.warning(f"[Wishlist] No Spotify ID found, checking for fuzzy match: '{track_name}' by '{artist_name}'") + logger.warning(f"[Wishlist] No track ID found, checking for fuzzy match: '{track_name}' by '{artist_name}'") # Get all wishlist tracks and find potential matches (search all profiles) if not wishlist_tracks: @@ -18692,10 +18703,10 @@ def _check_and_remove_from_wishlist(context): # Simple fuzzy matching if (wl_name == track_name.lower() and wl_artist_name == artist_name.lower()): spotify_track_id = wl_track.get('spotify_track_id') or wl_track.get('id') - logger.info(f"[Wishlist] Found fuzzy match - Spotify ID: {spotify_track_id}") + logger.info(f"[Wishlist] Found fuzzy match - track ID: {spotify_track_id}") break - # If we found a Spotify track ID, remove it from wishlist + # If we found a track ID, remove it from wishlist if spotify_track_id: logger.info(f"[Wishlist] Attempting to remove track from wishlist: {spotify_track_id}") removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) @@ -18704,7 +18715,7 @@ def _check_and_remove_from_wishlist(context): else: logger.warning(f"ℹ️ [Wishlist] Track not found in wishlist or already removed: {spotify_track_id}") else: - logger.warning("ℹ️ [Wishlist] No Spotify track ID found for wishlist removal check") + logger.warning("ℹ️ [Wishlist] No track ID found for wishlist removal check") except Exception as e: logger.error(f"[Wishlist] Error in wishlist removal check: {e}") @@ -48259,7 +48270,7 @@ def import_singles_process(): title = file_info.get('title', '') artist = file_info.get('artist', '') - manual_match = file_info.get('manual_match') + manual_match = file_info.get('manual_match') or file_info.get('spotify_override') if manual_match is not None and not isinstance(manual_match, dict): manual_match = None diff --git a/webui/static/stats-automations.js b/webui/static/stats-automations.js index 7b48c34c..fe6d1a98 100644 --- a/webui/static/stats-automations.js +++ b/webui/static/stats-automations.js @@ -1214,6 +1214,9 @@ function importPageRenderMatchList() { // Build rows let matchedCount = 0; const rows = data.matches.map((m, idx) => { + const track = m.track || m.spotify_track || {}; + const trackNumber = track.track_number ?? track.trackNumber; + const displayTrackNumber = trackNumber ? trackNumber : (idx + 1); let file = null; let confidence = m.confidence; let isOverride = false; @@ -1253,8 +1256,8 @@ function importPageRenderMatchList() { <div class="import-page-match-row ${file ? 'matched' : ''}" ondragover="importPageHandleDragOver(event)" ondragleave="this.classList.remove('drag-over')" ondrop="importPageHandleDrop(event, ${idx})" onclick="importPageTapAssign(${idx})"> - <span class="import-page-match-num">${m.spotify_track.track_number}</span> - <span class="import-page-match-track">${_esc(m.spotify_track.name)}</span> + <span class="import-page-match-num">${displayTrackNumber}</span> + <span class="import-page-match-track">${_esc(track.name || track.title || 'Unknown Track')}</span> <span class="import-page-match-file ${file ? 'has-file' : ''}"> ${file ? `<span class="import-page-match-file-name">${_esc(file.filename)}</span> @@ -1622,7 +1625,7 @@ function importPageProcessSingles() { const f = importPageState.stagingFiles[i]; const manualMatch = importPageState.singlesManualMatches[i]; if (manualMatch) { - return { ...f, spotify_override: manualMatch }; + return { ...f, manual_match: manualMatch }; } return f; }); @@ -7608,4 +7611,3 @@ window.updateEnhanceSelectedCount = updateEnhanceSelectedCount; window.submitEnhanceQuality = submitEnhanceQuality; // ===== END ENHANCE QUALITY MODAL ===== - From 9321fc4ad2b2536c64579bd1cdac10b6fef2141c Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sat, 25 Apr 2026 20:28:37 +0300 Subject: [PATCH 14/19] Small cleanup --- web_server.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/web_server.py b/web_server.py index c2880dfb..751266bd 100644 --- a/web_server.py +++ b/web_server.py @@ -95,23 +95,14 @@ from core.tidal_client import TidalClient # Added import for Tidal from core.matching_engine import MusicMatchingEngine from core.database_update_worker import DatabaseUpdateWorker from core.web_scan_manager import WebScanManager -from core.lyrics_client import lyrics_client from core.metadata_cache import get_metadata_cache from core.imports.context import ( - build_import_album_info, get_import_clean_album, - get_import_clean_artist, get_import_clean_title, get_import_context_album, get_import_context_artist, - get_import_has_clean_metadata, - get_import_has_full_metadata, get_import_original_search, - get_import_source, - get_import_source_ids, get_import_track_info, - get_library_source_id_columns, - get_source_tag_names, normalize_import_context, ) from core.imports.album import ( @@ -2417,20 +2408,6 @@ try: except Exception as e: logger.error(f"Public REST API v1 failed to register: {e}") -# --- Global Streaming State Management --- -# Thread-safe state tracking for streaming functionality - -# --- Global OAuth State Management --- -# Store PKCE values for Tidal OAuth flow - - -# Quality Scanner state - -# Duplicate Cleaner state - -# --- Retag Tool Globals --- - -# --- Sync Page Globals --- # --- Automation Progress Tracking --- automation_progress_states = {} # automation_id (int) -> state dict @@ -48270,7 +48247,7 @@ def import_singles_process(): title = file_info.get('title', '') artist = file_info.get('artist', '') - manual_match = file_info.get('manual_match') or file_info.get('spotify_override') + manual_match = file_info.get('manual_match') if manual_match is not None and not isinstance(manual_match, dict): manual_match = None From 4c819681a1b67bd7814eb064919af347dd663a46 Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sat, 25 Apr 2026 21:09:11 +0300 Subject: [PATCH 15/19] Move single-track resolver; fix wishlist cleanup - keep single-track import lookup in imports/resolution.py - normalize simple-download search_result data before wishlist matching - run wishlist cleanup for simple-download post-processing - keep source-only artist detail on resolved names and MB short-circuit --- core/imports/context.py | 16 ++++++- core/imports/pipeline.py | 4 ++ core/imports/side_effects.py | 8 ++-- core/metadata_service.py | 19 -------- tests/imports/test_import_pipeline.py | 3 ++ tests/imports/test_import_side_effects.py | 43 +++++++++++++++++++ ..._metadata_service_single_import_context.py | 9 ++-- web_server.py | 2 +- 8 files changed, 76 insertions(+), 28 deletions(-) diff --git a/core/imports/context.py b/core/imports/context.py index c19aa180..e24b8663 100644 --- a/core/imports/context.py +++ b/core/imports/context.py @@ -49,11 +49,12 @@ def normalize_import_context(context: Optional[Dict[str, Any]]) -> Dict[str, Any album = _as_dict(context.get("album") or context.get("spotify_album")) track_info = _as_dict(context.get("track_info")) original_search = _as_dict(context.get("original_search_result")) + search_result = _as_dict(context.get("search_result")) context["artist"] = artist context["album"] = album context["track_info"] = track_info - context["original_search_result"] = original_search + context["original_search_result"] = original_search or search_result context.pop("spotify_artist", None) context.pop("spotify_album", None) @@ -102,6 +103,12 @@ def get_import_original_search(context: Optional[Dict[str, Any]]) -> Dict[str, A return _as_dict(context.get("original_search_result")) +def get_import_search_result(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("search_result")) + + def get_import_source(context: Optional[Dict[str, Any]]) -> str: if not isinstance(context, dict): return "" @@ -199,6 +206,7 @@ def get_import_has_full_metadata(context: Optional[Dict[str, Any]]) -> bool: def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]: track_info = get_import_track_info(context) original_search = get_import_original_search(context) + search_result = get_import_search_result(context) artist = get_import_context_artist(context) album = get_import_context_album(context) @@ -208,12 +216,16 @@ def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]: _first_value(track_info, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""), _first_value(original_search, "id", "track_id", "source_track_id", default=""), _first_value(original_search, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""), + _first_value(search_result, "id", "track_id", "source_track_id", default=""), + _first_value(search_result, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""), ), "artist_id": _first_id_value( _first_value(artist, "id", "artist_id", "source_artist_id", default=""), _first_value(artist, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""), _first_value(original_search, "artist_id", "source_artist_id", default=""), _first_value(original_search, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""), + _first_value(search_result, "artist_id", "source_artist_id", default=""), + _first_value(search_result, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""), ), "album_id": _first_id_value( _first_value(album, "id", "album_id", "collectionId", "source_album_id", default=""), @@ -222,6 +234,8 @@ def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]: _first_value(original_search, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), _first_value(track_info, "album_id", "source_album_id", default=""), _first_value(track_info, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), + _first_value(search_result, "album_id", "source_album_id", default=""), + _first_value(search_result, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), ), } diff --git a/core/imports/pipeline.py b/core/imports/pipeline.py index 5d202a2a..7eaacc5d 100644 --- a/core/imports/pipeline.py +++ b/core/imports/pipeline.py @@ -257,6 +257,10 @@ def post_process_matched_download(context_key, context, file_path, runtime, meta emit_track_downloaded(context, automation_engine) record_library_history_download(context) record_download_provenance(context) + try: + check_and_remove_from_wishlist(context) + except Exception as wishlist_error: + logger.error(f"[Simple Download] Error checking wishlist removal: {wishlist_error}") return logger.info(f"Starting robust post-processing for: {context_key}") diff --git a/core/imports/side_effects.py b/core/imports/side_effects.py index b44bfe61..9c7c6636 100644 --- a/core/imports/side_effects.py +++ b/core/imports/side_effects.py @@ -15,6 +15,7 @@ from core.imports.context import ( get_import_context_album, get_import_context_artist, get_import_original_search, + get_import_search_result, get_import_source, get_import_source_ids, get_import_track_info, @@ -522,7 +523,8 @@ def check_and_remove_from_wishlist(context: Dict[str, Any]) -> None: "discogs": "Discogs", "hydrabase": "Hydrabase", }.get(source, "Source") - track_info = context.get("track_info", {}) + track_info = get_import_track_info(context) or get_import_search_result(context) + search_result = get_import_original_search(context) or get_import_search_result(context) track_id = None if source == "spotify": @@ -540,8 +542,8 @@ def check_and_remove_from_wishlist(context: Dict[str, Any]) -> None: break if not track_id: - track_name = track_info.get("name") or context.get("original_search_result", {}).get("title", "") - artist_name = _primary_track_artist_name(track_info) or _primary_track_artist_name(context.get("original_search_result", {})) + track_name = track_info.get("name") or search_result.get("title", "") + artist_name = _primary_track_artist_name(track_info) or _primary_track_artist_name(search_result) if track_name and artist_name: logger.warning("[Wishlist] No track ID found, checking for fuzzy match: '%s' by '%s'", track_name, artist_name) diff --git a/core/metadata_service.py b/core/metadata_service.py index 66487d60..44ded0d2 100644 --- a/core/metadata_service.py +++ b/core/metadata_service.py @@ -841,25 +841,6 @@ def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]: return normalized -def get_single_track_import_context( - title: str, - artist: str = '', - override_id: Optional[str] = None, - override_source: str = 'spotify', - source_override: Optional[str] = None, -) -> Dict[str, Any]: - """Compatibility wrapper for the single-track import resolver.""" - from core.imports.resolution import get_single_track_import_context as _get_single_track_import_context - - return _get_single_track_import_context( - title, - artist=artist, - override_id=override_id, - override_source=override_source, - source_override=source_override, - ) - - def resolve_album_reference( album_id: str, preferred_source: Optional[str] = None, diff --git a/tests/imports/test_import_pipeline.py b/tests/imports/test_import_pipeline.py index 231a0437..51b34361 100644 --- a/tests/imports/test_import_pipeline.py +++ b/tests/imports/test_import_pipeline.py @@ -59,6 +59,7 @@ def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch): completion_calls = [] scan_calls = [] activity_calls = [] + wishlist_calls = [] original_matched_context = dict(runtime_state.matched_downloads_context) original_download_tasks = dict(runtime_state.download_tasks) @@ -89,6 +90,7 @@ def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch): monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None) monkeypatch.setattr(import_pipeline, "record_library_history_download", lambda *args, **kwargs: None) monkeypatch.setattr(import_pipeline, "record_download_provenance", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "check_and_remove_from_wishlist", lambda context: wishlist_calls.append(dict(context))) monkeypatch.setattr(import_pipeline, "_mark_task_completed", lambda task, track_info: mark_calls.append((task, track_info))) monkeypatch.setattr(import_pipeline.threading, "Thread", _ImmediateThread) @@ -114,6 +116,7 @@ def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch): assert completion_calls == [(batch_id, task_id, True)] assert context_key not in runtime_state.matched_downloads_context assert scan_calls == ["Simple download completed"] + assert wishlist_calls and wishlist_calls[0]["search_result"]["is_simple_download"] is True assert activity_calls finally: runtime_state.matched_downloads_context.clear() diff --git a/tests/imports/test_import_side_effects.py b/tests/imports/test_import_side_effects.py index 697d845b..d1dc744a 100644 --- a/tests/imports/test_import_side_effects.py +++ b/tests/imports/test_import_side_effects.py @@ -12,6 +12,19 @@ class _FakeDB: return self._conn +class _FakeWishlistService: + def __init__(self, tracks): + self.tracks = tracks + self.removed = [] + + def get_wishlist_tracks_for_download(self, profile_id=1): + return list(self.tracks) + + def mark_track_download_result(self, spotify_track_id, success, error_message=None, profile_id=1): + self.removed.append((spotify_track_id, success, error_message, profile_id)) + return True + + def _make_soulsync_db(): conn = sqlite3.connect(":memory:") conn.row_factory = sqlite3.Row @@ -142,3 +155,33 @@ def test_record_soulsync_library_entry_writes_artist_album_and_track(tmp_path, m assert track_row["track_artist"] == "Guest Artist" assert track_row["album_id"] == album_row["id"] assert track_row["file_path"] == str(final_path) + + +def test_check_and_remove_from_wishlist_uses_search_result_fallback(monkeypatch): + fake_db = SimpleNamespace(get_all_profiles=lambda: [{"id": 1}]) + wishlist_service = _FakeWishlistService([ + { + "wishlist_id": 11, + "spotify_track_id": "sp-track-1", + "id": "sp-track-1", + "name": "Song One", + "artists": [{"name": "Artist One"}], + } + ]) + + monkeypatch.setattr(side_effects, "get_database", lambda: fake_db) + monkeypatch.setattr(side_effects, "get_wishlist_service", lambda: wishlist_service) + + context = { + "search_result": { + "title": "Song One", + "artist": "Artist One", + "album": "Album One", + }, + "track_info": {}, + "original_search_result": {}, + } + + side_effects.check_and_remove_from_wishlist(context) + + assert wishlist_service.removed == [("sp-track-1", True, None, 1)] diff --git a/tests/metadata/test_metadata_service_single_import_context.py b/tests/metadata/test_metadata_service_single_import_context.py index ed7ae567..0d09d668 100644 --- a/tests/metadata/test_metadata_service_single_import_context.py +++ b/tests/metadata/test_metadata_service_single_import_context.py @@ -1,6 +1,7 @@ from types import SimpleNamespace from core import metadata_service +from core.imports import resolution class FakeClient: @@ -74,7 +75,7 @@ def test_get_single_track_import_context_uses_primary_source_priority(monkeypatc lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), ) - result = metadata_service.get_single_track_import_context("Song One", "Artist One") + result = resolution.get_single_track_import_context("Song One", "Artist One") assert result["success"] is True assert result["source"] == "deezer" @@ -108,7 +109,7 @@ def test_get_single_track_import_context_falls_back_to_next_source(monkeypatch): lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), ) - result = metadata_service.get_single_track_import_context("Song Two", "Artist Two") + result = resolution.get_single_track_import_context("Song Two", "Artist Two") assert result["success"] is True assert result["source"] == "spotify" @@ -141,7 +142,7 @@ def test_get_single_track_import_context_uses_explicit_override_first(monkeypatc lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), ) - result = metadata_service.get_single_track_import_context( + result = resolution.get_single_track_import_context( "Ignored Title", "Ignored Artist", override_id="override-track-1", @@ -174,7 +175,7 @@ def test_get_single_track_import_context_uses_explicit_override_source(monkeypat lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": itunes_client}.get(source), ) - result = metadata_service.get_single_track_import_context( + result = resolution.get_single_track_import_context( "Ignored Title", "Ignored Artist", override_id="override-track-2", diff --git a/web_server.py b/web_server.py index 751266bd..b548b08f 100644 --- a/web_server.py +++ b/web_server.py @@ -48267,7 +48267,7 @@ def import_singles_process(): if not artist: artist = parsed.get('artist', '') - from core.metadata_service import get_single_track_import_context + from core.imports.resolution import get_single_track_import_context resolved = get_single_track_import_context( title, From 6ee119ffa9ab86af53b987b3458d31026c5bf1b8 Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sun, 26 Apr 2026 10:31:48 +0300 Subject: [PATCH 16/19] Fix DummyConfigManager position in album completeness job test --- tests/test_album_completeness_job.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_album_completeness_job.py b/tests/test_album_completeness_job.py index 5451839f..05f5736b 100644 --- a/tests/test_album_completeness_job.py +++ b/tests/test_album_completeness_job.py @@ -2,6 +2,14 @@ import sys import types +class _DummyConfigManager: + def get(self, key, default=None): + return default + + def get_active_media_server(self): + return "plex" + + if "spotipy" not in sys.modules: spotipy = types.ModuleType("spotipy") @@ -26,13 +34,6 @@ if "config.settings" not in sys.modules: config_pkg = types.ModuleType("config") settings_mod = types.ModuleType("config.settings") - class _DummyConfigManager: - def get(self, key, default=None): - return default - - def get_active_media_server(self): - return "plex" - settings_mod.config_manager = _DummyConfigManager() config_pkg.settings = settings_mod sys.modules["config"] = config_pkg From 4f236baa6d79a4ecef2ab39ce4879e98a9c2f404 Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sun, 26 Apr 2026 11:26:19 +0300 Subject: [PATCH 17/19] Fix import normalization and task completion locking - Promote legacy _source into source during import normalization. - Keep the normalized import context neutral after stripping aliases. - Avoid re-entering tasks_lock when marking completed download tasks. --- core/imports/context.py | 4 ++++ core/runtime_state.py | 24 +++++++++++++----------- tests/imports/test_import_context.py | 16 ++++++++++++++++ web_server.py | 4 ++-- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/core/imports/context.py b/core/imports/context.py index e24b8663..22b88e54 100644 --- a/core/imports/context.py +++ b/core/imports/context.py @@ -45,16 +45,20 @@ def normalize_import_context(context: Optional[Dict[str, Any]]) -> Dict[str, Any if not isinstance(context, dict): return {} + source = context.get("source") or context.get("_source") or "" artist = _as_dict(context.get("artist") or context.get("spotify_artist")) album = _as_dict(context.get("album") or context.get("spotify_album")) track_info = _as_dict(context.get("track_info")) original_search = _as_dict(context.get("original_search_result")) search_result = _as_dict(context.get("search_result")) + if source: + context["source"] = source context["artist"] = artist context["album"] = album context["track_info"] = track_info context["original_search_result"] = original_search or search_result + context.pop("_source", None) context.pop("spotify_artist", None) context.pop("spotify_album", None) diff --git a/core/runtime_state.py b/core/runtime_state.py index 8e4f4b64..5e70d38f 100644 --- a/core/runtime_state.py +++ b/core/runtime_state.py @@ -51,15 +51,17 @@ def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True): def mark_task_completed(task_id: str, track_info: Optional[Dict[str, Any]] = None) -> bool: - """Mark a download task as completed in the shared task registry.""" - with tasks_lock: - task = download_tasks.get(task_id) - if not task: - return False + """Mark a download task as completed. - task["status"] = "completed" - task["stream_processed"] = True - task["status_change_time"] = time.time() - if track_info is not None: - task["track_info"] = track_info - return True + Callers must already hold `tasks_lock`. + """ + task = download_tasks.get(task_id) + if not task: + return False + + task["status"] = "completed" + task["stream_processed"] = True + task["status_change_time"] = time.time() + if track_info is not None: + task["track_info"] = track_info + return True diff --git a/tests/imports/test_import_context.py b/tests/imports/test_import_context.py index 7e2f242a..ed7ddc1a 100644 --- a/tests/imports/test_import_context.py +++ b/tests/imports/test_import_context.py @@ -63,6 +63,22 @@ def test_normalize_import_context_promotes_neutral_fields_without_legacy_aliases assert get_import_has_full_metadata(normalized) is False +def test_normalize_import_context_promotes_legacy_source_alias(): + context = { + "_source": "spotify", + "artist": {"name": "Artist One", "id": "artist-1"}, + "album": {"name": "Album One", "id": "album-1"}, + "track_info": {"name": "Song One", "id": "track-1"}, + "original_search_result": {"title": "Song One"}, + } + + normalized = normalize_import_context(context) + + assert normalized["source"] == "spotify" + assert "_source" not in normalized + assert get_import_source(normalized) == "spotify" + + def test_neutral_import_context_helpers_work_without_legacy_aliases(): context = { "source": "deezer", diff --git a/web_server.py b/web_server.py index b548b08f..5d76cb76 100644 --- a/web_server.py +++ b/web_server.py @@ -141,7 +141,7 @@ from core.runtime_state import ( download_tasks, matched_context_lock, matched_downloads_context, - mark_task_completed as _core_mark_task_completed, + mark_task_completed, processed_download_ids, set_activity_toast_emitter, tasks_lock, @@ -2483,7 +2483,7 @@ def _mark_task_completed(task_id, track_info=None): Assumes task_id exists in download_tasks (should be called within tasks_lock). """ global session_completed_downloads - _core_mark_task_completed(task_id, track_info) + mark_task_completed(task_id, track_info) # Increment session counter (matches dashboard.py behavior) with session_stats_lock: From 9315e74beafda2f863023db04eefb0cc4aa58fa8 Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Sun, 26 Apr 2026 12:13:46 +0300 Subject: [PATCH 18/19] Broaden import and metadata test coverage - Cover search_result fallback normalization and ambiguous album detection. - Add staging metadata, multi-disc path, and MusicBrainz enrichment cases. - Move the single-track context test next to the imports code it exercises. --- core/imports/context.py | 11 +- tests/imports/test_import_context.py | 93 ++++++++++++ tests/imports/test_import_file_ops.py | 34 +++++ tests/imports/test_import_paths.py | 69 +++++++++ ...import_resolution_single_track_context.py} | 23 +++ tests/metadata/test_metadata_enrichment.py | 142 ++++++++++++++++++ 6 files changed, 367 insertions(+), 5 deletions(-) rename tests/{metadata/test_metadata_service_single_import_context.py => imports/test_import_resolution_single_track_context.py} (88%) diff --git a/core/imports/context.py b/core/imports/context.py index 22b88e54..f0987f8a 100644 --- a/core/imports/context.py +++ b/core/imports/context.py @@ -51,13 +51,14 @@ def normalize_import_context(context: Optional[Dict[str, Any]]) -> Dict[str, Any track_info = _as_dict(context.get("track_info")) original_search = _as_dict(context.get("original_search_result")) search_result = _as_dict(context.get("search_result")) + normalized_search = original_search or search_result if source: context["source"] = source context["artist"] = artist context["album"] = album context["track_info"] = track_info - context["original_search_result"] = original_search or search_result + context["original_search_result"] = normalized_search context.pop("_source", None) context.pop("spotify_artist", None) context.pop("spotify_album", None) @@ -67,11 +68,11 @@ def normalize_import_context(context: Optional[Dict[str, Any]]) -> Dict[str, Any ("clean_album", "spotify_clean_album"), ("clean_artist", "spotify_clean_artist"), ): - if clean_key not in original_search or original_search.get(clean_key) in (None, ""): - legacy_value = original_search.get(legacy_key) + if clean_key not in normalized_search or normalized_search.get(clean_key) in (None, ""): + legacy_value = normalized_search.get(legacy_key) if legacy_value not in (None, ""): - original_search[clean_key] = legacy_value - original_search.pop(legacy_key, None) + normalized_search[clean_key] = legacy_value + normalized_search.pop(legacy_key, None) has_clean = bool(context.get("has_clean_metadata", context.get("has_clean_spotify_data", False))) has_full = bool(context.get("has_full_metadata", context.get("has_full_spotify_metadata", False))) diff --git a/tests/imports/test_import_context.py b/tests/imports/test_import_context.py index ed7ddc1a..65ab1e12 100644 --- a/tests/imports/test_import_context.py +++ b/tests/imports/test_import_context.py @@ -2,6 +2,7 @@ import pytest from core.imports.context import ( build_import_album_info, + detect_album_info_web, get_import_clean_album, get_import_clean_artist, get_import_clean_title, @@ -79,6 +80,33 @@ def test_normalize_import_context_promotes_legacy_source_alias(): assert get_import_source(normalized) == "spotify" +def test_normalize_import_context_promotes_search_result_when_original_search_missing(): + context = { + "source": "spotify", + "track_info": {"name": "Song One", "id": "track-1"}, + "search_result": { + "title": "Song One", + "album": "Album One", + "artist": "Artist One", + "spotify_clean_title": "Song One", + "spotify_clean_album": "Album One", + "spotify_clean_artist": "Artist One", + }, + } + + normalized = normalize_import_context(context) + + assert normalized["original_search_result"]["clean_title"] == "Song One" + assert normalized["original_search_result"]["clean_album"] == "Album One" + assert normalized["original_search_result"]["clean_artist"] == "Artist One" + assert "spotify_clean_title" not in normalized["original_search_result"] + assert "spotify_clean_album" not in normalized["original_search_result"] + assert "spotify_clean_artist" not in normalized["original_search_result"] + assert get_import_clean_title(normalized) == "Song One" + assert get_import_clean_album(normalized) == "Album One" + assert get_import_clean_artist(normalized) == "Artist One" + + def test_neutral_import_context_helpers_work_without_legacy_aliases(): context = { "source": "deezer", @@ -216,3 +244,68 @@ def test_build_import_album_info_uses_normalized_album_context(): assert album_info["clean_track_name"] == "Song One" assert album_info["album_image_url"] == "https://img.example/album.jpg" assert album_info["source"] == "deezer" + + +@pytest.mark.parametrize( + "album_name,track_name,artist_name", + [ + ("Song One", "Song One", "Artist One"), + ("Artist One", "Different Song", "Artist One"), + ], +) +def test_detect_album_info_web_returns_none_for_ambiguous_album_names(album_name, track_name, artist_name): + context = normalize_import_context( + { + "source": "deezer", + "artist": {"name": artist_name}, + "album": {"name": album_name, "total_tracks": 12, "album_type": "album"}, + "track_info": {"name": track_name, "track_number": 4, "disc_number": 1}, + "original_search_result": { + "title": track_name, + "clean_title": track_name, + "clean_album": album_name, + "clean_artist": artist_name, + }, + } + ) + + assert detect_album_info_web(context) is None + + +def test_detect_album_info_web_forces_album_when_track_and_artist_differ(): + context = normalize_import_context( + { + "source": "deezer", + "artist": {"name": "Artist One"}, + "album": { + "name": "Album One", + "image_url": "https://img.example/album.jpg", + "release_date": "2024-05-01", + "total_tracks": 1, + "album_type": "album", + }, + "track_info": { + "name": "Song One", + "track_number": 4, + "disc_number": 2, + "duration_ms": 240000, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "title": "Song One", + "album": "Album One", + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + }, + } + ) + + album_info = detect_album_info_web(context) + + assert album_info is not None + assert album_info["is_album"] is True + assert album_info["confidence"] == 0.5 + assert album_info["album_name"] == "Album One" + assert album_info["track_number"] == 4 + assert album_info["disc_number"] == 2 diff --git a/tests/imports/test_import_file_ops.py b/tests/imports/test_import_file_ops.py index a07c5c4a..16e68ad7 100644 --- a/tests/imports/test_import_file_ops.py +++ b/tests/imports/test_import_file_ops.py @@ -90,3 +90,37 @@ def test_read_staging_file_metadata_falls_back_to_filename_track_number(monkeypa assert metadata["title"] == "07 - Song Two" assert metadata["track_number"] == 7 assert metadata["disc_number"] == 1 + + +def test_read_staging_file_metadata_uses_filename_fallbacks_when_tags_are_invalid(monkeypatch, tmp_path): + file_path = tmp_path / "02 - Song Three.flac" + file_path.write_text("fake") + + class DummyTags: + def __init__(self): + self.values = { + "title": [""], + "artist": "Artist One", + "albumartist": "", + "album": ["Album One"], + "tracknumber": ["not-a-number"], + "discnumber": ["bad/disc"], + } + + def get(self, key, default=None): + return self.values.get(key, default) + + fake_mutagen = types.ModuleType("mutagen") + fake_mutagen.File = lambda path, easy=True: DummyTags() + monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen) + + metadata = read_staging_file_metadata(str(file_path), file_path.name) + + assert metadata == { + "title": "02 - Song Three", + "artist": "Artist One", + "albumartist": "Artist One", + "album": "Album One", + "track_number": 2, + "disc_number": 1, + } diff --git a/tests/imports/test_import_paths.py b/tests/imports/test_import_paths.py index 656279f9..1caaf2eb 100644 --- a/tests/imports/test_import_paths.py +++ b/tests/imports/test_import_paths.py @@ -159,3 +159,72 @@ def test_build_final_path_for_track_uses_template_and_disc_folder(monkeypatch, t tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1" / "04 - Song One.flac" ) assert (tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1").is_dir() + + +def test_build_final_path_for_track_uses_track_disc_number_without_provider_lookup(monkeypatch, tmp_path): + config = _Config( + { + "soulseek.transfer_path": str(tmp_path / "Transfer"), + "file_organization.enabled": True, + "file_organization.templates": { + "album_path": "$albumartist/$albumartist - $album/$track - $title", + "single_path": "$artist/$artist - $title", + }, + "file_organization.collab_artist_mode": "first", + "file_organization.disc_label": "Disc", + } + ) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + + calls = [] + monkeypatch.setattr( + import_paths, + "_get_album_tracks_for_source", + lambda source, album_id: calls.append((source, album_id)) or None, + ) + + context = { + "artist": {"name": "Artist One"}, + "album": { + "name": "Album One", + "id": "album-1", + "release_date": "2026-01-01", + "total_tracks": 12, + "album_type": "album", + "artists": [{"name": "Artist One"}], + }, + "track_info": { + "name": "Song Two", + "id": "track-2", + "track_number": 4, + "disc_number": 2, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "title": "Song Two", + "clean_title": "Song Two", + "clean_album": "Album One", + "clean_artist": "Artist One", + "artists": [{"name": "Artist One"}], + }, + "source": "deezer", + "is_album_download": False, + } + + final_path, created = import_paths.build_final_path_for_track( + context, + {"name": "Artist One"}, + { + "is_album": True, + "album_name": "Album One", + "track_number": 4, + "disc_number": 2, + }, + ".flac", + ) + + assert created is True + assert calls == [] + assert final_path == str( + tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 2" / "04 - Song Two.flac" + ) diff --git a/tests/metadata/test_metadata_service_single_import_context.py b/tests/imports/test_import_resolution_single_track_context.py similarity index 88% rename from tests/metadata/test_metadata_service_single_import_context.py rename to tests/imports/test_import_resolution_single_track_context.py index 0d09d668..90df6a29 100644 --- a/tests/metadata/test_metadata_service_single_import_context.py +++ b/tests/imports/test_import_resolution_single_track_context.py @@ -192,3 +192,26 @@ def test_get_single_track_import_context_uses_explicit_override_source(monkeypat ] assert deezer_client.calls == [] assert spotify_client.calls == [] + + +def test_get_single_track_import_context_returns_fallback_payload_when_no_source_matches(monkeypatch): + deezer_client = FakeClient() + spotify_client = FakeClient() + itunes_client = FakeClient() + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": itunes_client}.get(source), + ) + + result = resolution.get_single_track_import_context("Missing Song", "Missing Artist") + + assert result["success"] is False + assert result["source"] is None + assert result["context"]["artist"]["name"] == "Missing Artist" + assert result["context"]["track_info"]["name"] == "Missing Song" + assert result["context"]["original_search_result"]["clean_title"] == "Missing Song" + assert result["context"]["original_search_result"]["clean_artist"] == "Missing Artist" diff --git a/tests/metadata/test_metadata_enrichment.py b/tests/metadata/test_metadata_enrichment.py index 24c5a302..b3f87f7d 100644 --- a/tests/metadata/test_metadata_enrichment.py +++ b/tests/metadata/test_metadata_enrichment.py @@ -1,4 +1,5 @@ import types +import sqlite3 from core.metadata import enrichment as me from core.metadata import artwork as ma @@ -73,6 +74,16 @@ class _FakeResponse: return False +class _FileDB: + def __init__(self, db_path): + self.db_path = db_path + + def _get_connection(self): + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def _fake_symbols(audio): def _tag_factory(kind): return lambda **kwargs: _FakeTag(kind, **kwargs) @@ -208,6 +219,137 @@ def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatc assert "ITUNES_ALBUM_ID" in legacy_descs +def test_embed_source_ids_skips_disabled_source_specific_tags(monkeypatch): + audio = _FakeAudio() + symbols = _fake_symbols(audio) + + monkeypatch.setattr( + ms, + "get_config_manager", + lambda: _Config({"deezer.embed_tags": False}), + ) + monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "get_database", lambda: None) + + metadata = { + "source": "deezer", + "source_track_id": "dz-track", + "source_artist_id": "dz-artist", + "source_album_id": "dz-album", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + } + + me.embed_source_ids(audio, metadata, context={"track_info": {}, "original_search_result": {}}) + + assert audio.tags.added == [] + + +def test_embed_source_ids_writes_musicbrainz_release_year_and_updates_album_year(tmp_path, monkeypatch): + db_path = tmp_path / "music.db" + conn = sqlite3.connect(db_path) + conn.execute("CREATE TABLE artists (id TEXT PRIMARY KEY, name TEXT)") + conn.execute("CREATE TABLE albums (id TEXT PRIMARY KEY, artist_id TEXT, title TEXT, year INTEGER)") + conn.execute("INSERT INTO artists (id, name) VALUES (?, ?)", ("artist-1", "Artist One")) + conn.execute( + "INSERT INTO albums (id, artist_id, title, year) VALUES (?, ?, ?, ?)", + ("album-1", "artist-1", "Album One", None), + ) + conn.commit() + conn.close() + + class _FakeMBClient: + def get_recording(self, mbid, includes=None): + return { + "isrcs": ["ISRC-123"], + "genres": [{"name": "Post Rock", "count": 10}], + } + + def get_release(self, mbid, includes=None): + return { + "release-group": { + "id": "rg-1", + "primary-type": "album", + "first-release-date": "2021-09-17", + }, + "artist-credit": [{"artist": {"id": "artist-mb-1"}}], + "status": "Official", + "country": "US", + "barcode": "1234567890", + "media": [{"format": "CD", "tracks": [{"position": 1, "id": "reltrack-1", "recording": {"id": "rec-1"}}]}], + "label-info": [{"catalog-number": "CAT-1"}], + "text-representation": {"script": "Latn"}, + "asin": "ASIN1", + } + + class _FakeMBService: + def __init__(self): + self.mb_client = _FakeMBClient() + + def match_recording(self, title, artist): + return {"mbid": "rec-mbid"} + + def match_artist(self, artist): + return {"mbid": "artist-mbid"} + + def match_release(self, album, artist): + return {"mbid": "release-mbid"} + + audio = _FakeAudio() + symbols = _fake_symbols(audio) + runtime = types.SimpleNamespace(mb_worker=types.SimpleNamespace(mb_service=_FakeMBService())) + + monkeypatch.setattr( + ms, + "get_config_manager", + lambda: _Config( + { + "metadata_enhancement.enabled": True, + "metadata_enhancement.embed_album_art": False, + "metadata_enhancement.tags.write_multi_artist": False, + "musicbrainz.embed_tags": True, + } + ), + ) + monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols) + monkeypatch.setattr(ms, "get_database", lambda: _FileDB(str(db_path))) + + metadata = { + "source": "musicbrainz", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + "track_number": 1, + "total_tracks": 12, + "disc_number": 1, + } + + me.embed_source_ids(audio, metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime) + + assert metadata["musicbrainz_release_id"] == "release-mbid" + assert metadata["date"] == "2021" + assert any(frame.kind == "TDRC" for frame in audio.tags.added) + assert any(frame.kind == "TSRC" for frame in audio.tags.added) + + check = sqlite3.connect(db_path) + check.row_factory = sqlite3.Row + row = check.execute( + """ + SELECT albums.year + FROM albums + JOIN artists ON artists.id = albums.artist_id + WHERE albums.title = ? AND artists.name = ? + """, + ("Album One", "Artist One"), + ).fetchone() + check.close() + + assert row["year"] == 2021 + + def test_enhance_file_metadata_forwards_runtime_to_source_embedding(monkeypatch): audio = _FakeAudio() symbols = _fake_symbols(audio) From 02305096a3ab00bf8b0dad537c5ee2893aad41cd Mon Sep 17 00:00:00 2001 From: Antti Kettunen <antti-kettunen@hotmail.com> Date: Mon, 27 Apr 2026 20:25:01 +0300 Subject: [PATCH 19/19] Tighten metadata and import safety - Normalize album import track display handling so queue labels and match rows stay consistent - Bound MusicBrainz caches and avoid caching transient lookup failures - Stop swallowing programmer errors in source enrichment helpers - Restore import config test seams without reintroducing lazy imports - Guard task completion calls and fix the Windows path test expectation - Keep file lock tracking from growing without bound --- core/imports/file_ops.py | 9 +- core/imports/guards.py | 9 +- core/imports/side_effects.py | 8 +- core/metadata/common.py | 7 +- core/metadata/source.py | 561 +++++++++++---------- core/runtime_state.py | 13 + tests/imports/test_import_paths.py | 4 +- tests/metadata/test_metadata_enrichment.py | 107 ++++ tests/test_runtime_state.py | 32 ++ webui/static/stats-automations.js | 25 +- 10 files changed, 477 insertions(+), 298 deletions(-) create mode 100644 tests/test_runtime_state.py diff --git a/core/imports/file_ops.py b/core/imports/file_ops.py index 73185aac..66b26244 100644 --- a/core/imports/file_ops.py +++ b/core/imports/file_ops.py @@ -9,14 +9,11 @@ import subprocess import time from pathlib import Path +from config.settings import config_manager + logger = logging.getLogger("imports.file_ops") -def _get_config_manager(): - from config.settings import config_manager - return config_manager - - def safe_move_file(src, dst): """Move a file safely across filesystems.""" src = Path(src) @@ -176,7 +173,6 @@ def downsample_hires_flac(final_path, context): """Downsample a hi-res FLAC to 16-bit/44.1kHz if enabled.""" from mutagen.flac import FLAC - config_manager = _get_config_manager() if not config_manager.get("lossy_copy.downsample_hires", False): return None @@ -287,7 +283,6 @@ def create_lossy_copy(final_path): """Convert a FLAC file to a lossy copy using the configured codec.""" from mutagen.flac import FLAC - config_manager = _get_config_manager() if not config_manager.get("lossy_copy.enabled", False): return None diff --git a/core/imports/guards.py b/core/imports/guards.py index 4d97c9fb..a95fbd6d 100644 --- a/core/imports/guards.py +++ b/core/imports/guards.py @@ -8,6 +8,7 @@ from datetime import datetime from pathlib import Path from typing import Any, Dict, Optional +from config.settings import config_manager from core.imports.context import ( get_import_clean_artist, get_import_clean_title, @@ -25,15 +26,12 @@ logger = get_logger("imports.guards") def _get_config_manager(): - from config.settings import config_manager - return config_manager def move_to_quarantine(file_path: str, context: dict, reason: str, automation_engine=None) -> str: """Move a file to the quarantine folder and write a metadata sidecar.""" - config_manager = _get_config_manager() - download_dir = config_manager.get("soulseek.download_path", "./downloads") + download_dir = _get_config_manager().get("soulseek.download_path", "./downloads") quarantine_dir = Path(download_dir) / "ss_quarantine" quarantine_dir.mkdir(parents=True, exist_ok=True) @@ -95,7 +93,6 @@ def check_flac_bit_depth(file_path: str, context: dict) -> Optional[str]: if not context.get("_audio_quality", "").startswith("FLAC"): return None - config_manager = _get_config_manager() quality_profile = MusicDatabase().get_quality_profile() flac_config = quality_profile.get("qualities", {}).get("flac", {}) flac_pref = flac_config.get("bit_depth", "any") @@ -107,7 +104,7 @@ def check_flac_bit_depth(file_path: str, context: dict) -> Optional[str]: return None flac_fallback = flac_config.get("bit_depth_fallback", True) - downsample_enabled = config_manager.get("lossy_copy.downsample_hires", False) + downsample_enabled = _get_config_manager().get("lossy_copy.downsample_hires", False) track_info = context.get("track_info", {}) track_name = track_info.get("name", os.path.basename(file_path)) diff --git a/core/imports/side_effects.py b/core/imports/side_effects.py index 9c7c6636..b5936a8a 100644 --- a/core/imports/side_effects.py +++ b/core/imports/side_effects.py @@ -7,6 +7,7 @@ import json import os from typing import Any, Dict, List, Optional +from config.settings import config_manager from core.imports.context import ( extract_artist_name, get_import_clean_album, @@ -31,8 +32,6 @@ logger = get_logger("imports.side_effects") def _get_config_manager(): - from config.settings import config_manager - return config_manager @@ -222,8 +221,7 @@ def record_download_provenance(context: Dict[str, Any]) -> None: def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any]) -> None: """Write imported media to the SoulSync library tables when the active server is SoulSync.""" try: - config_manager = _get_config_manager() - if config_manager.get_active_media_server() != "soulsync": + if _get_config_manager().get_active_media_server() != "soulsync": return context = normalize_import_context(context) @@ -288,7 +286,7 @@ def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[ if genres: from core.genre_filter import filter_genres as _filter_genres - genres = _filter_genres(genres, config_manager) + genres = _filter_genres(genres, _get_config_manager()) genres_json = json.dumps(genres) if genres else "" bitrate = 0 diff --git a/core/metadata/common.py b/core/metadata/common.py index 62408352..9220cd75 100644 --- a/core/metadata/common.py +++ b/core/metadata/common.py @@ -4,8 +4,9 @@ from __future__ import annotations import os import threading +import weakref from types import SimpleNamespace -from typing import Any, Dict +from typing import Any from utils.logging_config import get_logger as _create_logger @@ -26,7 +27,7 @@ __all__ = [ "wipe_source_tags", ] -_FILE_LOCKS: Dict[str, threading.Lock] = {} +_FILE_LOCKS: "weakref.WeakValueDictionary[str, threading.Lock]" = weakref.WeakValueDictionary() _FILE_LOCKS_LOCK = threading.Lock() @@ -117,6 +118,8 @@ def get_mutagen_symbols(): def get_file_lock(file_path: str) -> threading.Lock: + # Keep a per-path lock while it is actively referenced, but let it + # fall out of the cache once nobody is using it anymore. with _FILE_LOCKS_LOCK: lock = _FILE_LOCKS.get(file_path) if lock is None: diff --git a/core/metadata/source.py b/core/metadata/source.py index 5bd790ba..31450f77 100644 --- a/core/metadata/source.py +++ b/core/metadata/source.py @@ -3,10 +3,14 @@ from __future__ import annotations import re +import socket import threading import time +from collections import OrderedDict from typing import Any, Dict +import requests + from core.imports.context import ( extract_artist_name, get_import_clean_artist, @@ -38,12 +42,17 @@ __all__ = [ "mb_release_detail_cache_lock", ] -mb_release_cache: Dict[tuple, str] = {} +_MB_RELEASE_CACHE_MAX_ENTRIES = 4096 +_MB_RELEASE_DETAIL_CACHE_MAX_ENTRIES = 4096 + +mb_release_cache: "OrderedDict[tuple, str]" = OrderedDict() mb_release_cache_lock = threading.RLock() -mb_release_detail_cache: Dict[str, Dict[str, Any]] = {} +mb_release_detail_cache: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() mb_release_detail_cache_lock = threading.RLock() logger = _create_logger("metadata.source") +_SOURCE_NETWORK_EXCEPTIONS = (requests.RequestException, socket.timeout, TimeoutError) + _EDITION_PAREN_RE = re.compile( r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' r'limited|bonus|platinum|gold|super\s*deluxe|standard)' @@ -64,6 +73,29 @@ def normalize_album_cache_key(album_name: str) -> str: return result.lower().strip() +def _bounded_cache_get(cache, key): + value = cache.get(key) + if value is not None and hasattr(cache, "move_to_end"): + cache.move_to_end(key) + return value + + +def _bounded_cache_set(cache, key, value, max_entries: int) -> None: + cache[key] = value + if hasattr(cache, "move_to_end"): + cache.move_to_end(key) + while len(cache) > max_entries: + cache.popitem(last=False) + + +def _call_source_lookup(label: str, func, *args, **kwargs): + try: + return func(*args, **kwargs) + except _SOURCE_NETWORK_EXCEPTIONS as exc: + logger.warning("%s lookup failed (network): %s", label, exc) + return None + + SOURCE_TAG_CONFIG = { "SPOTIFY_TRACK_ID": "spotify.tags.track_id", "SPOTIFY_ARTIST_ID": "spotify.tags.artist_id", @@ -194,119 +226,124 @@ def _process_musicbrainz_source(pp: dict, metadata: dict, cfg, runtime, track_ti if not mb_service: return - try: - result = mb_service.match_recording(track_title, artist_name) - if result and result.get("mbid"): - pp["recording_mbid"] = result["mbid"] - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"] - details = mb_service.mb_client.get_recording(pp["recording_mbid"], includes=["isrcs", "genres"]) - if details: - isrcs = details.get("isrcs", []) - if isrcs: - pp["isrc"] = isrcs[0] - pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)] + result = _call_source_lookup("MusicBrainz recording", mb_service.match_recording, track_title, artist_name) + if result and result.get("mbid"): + pp["recording_mbid"] = result["mbid"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"] + details = _call_source_lookup( + "MusicBrainz recording details", + mb_service.mb_client.get_recording, + pp["recording_mbid"], + includes=["isrcs", "genres"], + ) + if details: + isrcs = details.get("isrcs", []) + if isrcs: + pp["isrc"] = isrcs[0] + pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)] - track_artist_name = metadata.get("artist", "") or artist_name - if ", " in track_artist_name: - track_artist_name = track_artist_name.split(", ")[0] - artist_result = mb_service.match_artist(track_artist_name) - if artist_result and artist_result.get("mbid"): - pp["artist_mbid"] = artist_result["mbid"] - pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"] - - album_name_for_mb = metadata.get("album", "") - if album_name_for_mb: - artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() - rc_key_norm = (normalize_album_cache_key(album_name_for_mb), artist_key) - rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) - with mb_release_cache_lock: - cached = mb_release_cache.get(rc_key_norm) - if cached is None: - cached = mb_release_cache.get(rc_key_exact) - if cached is not None: - pp["release_mbid"] = cached - else: - try: - rc_result = mb_service.match_release(album_name_for_mb, artist_name) - pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" - except Exception: - pp["release_mbid"] = "" - mb_release_cache[rc_key_norm] = pp["release_mbid"] - mb_release_cache[rc_key_exact] = pp["release_mbid"] - if pp["release_mbid"]: - pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] + track_artist_name = metadata.get("artist", "") or artist_name + if ", " in track_artist_name: + track_artist_name = track_artist_name.split(", ")[0] + artist_result = _call_source_lookup("MusicBrainz artist", mb_service.match_artist, track_artist_name) + if artist_result and artist_result.get("mbid"): + pp["artist_mbid"] = artist_result["mbid"] + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"] + album_name_for_mb = metadata.get("album", "") + if album_name_for_mb: + artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() + rc_key_norm = (normalize_album_cache_key(album_name_for_mb), artist_key) + rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) + release_mbid = None + with mb_release_cache_lock: + cached = _bounded_cache_get(mb_release_cache, rc_key_norm) + if cached is None: + cached = _bounded_cache_get(mb_release_cache, rc_key_exact) + if cached: + release_mbid = cached + else: + rc_result = _call_source_lookup("MusicBrainz release", mb_service.match_release, album_name_for_mb, artist_name) + if rc_result and rc_result.get("mbid"): + release_mbid = rc_result["mbid"] + if release_mbid: + _bounded_cache_set(mb_release_cache, rc_key_norm, release_mbid, _MB_RELEASE_CACHE_MAX_ENTRIES) + _bounded_cache_set(mb_release_cache, rc_key_exact, release_mbid, _MB_RELEASE_CACHE_MAX_ENTRIES) + pp["release_mbid"] = release_mbid or "" if pp["release_mbid"]: + pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] + + if pp["release_mbid"]: + with mb_release_detail_cache_lock: + release_detail = _bounded_cache_get(mb_release_detail_cache, pp["release_mbid"]) + if release_detail is None: + release_detail = _call_source_lookup( + "MusicBrainz release details", + mb_service.mb_client.get_release, + pp["release_mbid"], + includes=["release-groups", "labels", "media", "artist-credits", "recordings"], + ) or {} with mb_release_detail_cache_lock: - release_detail = mb_release_detail_cache.get(pp["release_mbid"]) - if release_detail is None: - release_detail = mb_service.mb_client.get_release( - pp["release_mbid"], - includes=["release-groups", "labels", "media", "artist-credits", "recordings"], - ) or {} - with mb_release_detail_cache_lock: - mb_release_detail_cache[pp["release_mbid"]] = release_detail - if release_detail: - rg = release_detail.get("release-group", {}) - if rg.get("id"): - pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"] - ac = release_detail.get("artist-credit", []) - if ac and isinstance(ac[0], dict): - aa = ac[0].get("artist", {}) - if aa.get("id"): - pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"] - if rg.get("primary-type"): - pp["id_tags"]["RELEASETYPE"] = rg["primary-type"] - if rg.get("first-release-date"): - pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"] - if not pp["release_year"] and len(rg["first-release-date"]) >= 4: - year = rg["first-release-date"][:4] - if year.isdigit(): - pp["release_year"] = year - if release_detail.get("status"): - pp["id_tags"]["RELEASESTATUS"] = release_detail["status"] - if release_detail.get("country"): - pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"] - if release_detail.get("barcode"): - pp["id_tags"]["BARCODE"] = release_detail["barcode"] - media_list = release_detail.get("media", []) - if media_list: - fmt = media_list[0].get("format", "") - if fmt: - pp["id_tags"]["MEDIA"] = fmt - pp["id_tags"]["TOTALDISCS"] = str(len(media_list)) - label_info = release_detail.get("label-info", []) - if label_info and isinstance(label_info[0], dict): - cat = label_info[0].get("catalog-number", "") - if cat: - pp["id_tags"]["CATALOGNUMBER"] = cat - text_rep = release_detail.get("text-representation", {}) - if isinstance(text_rep, dict) and text_rep.get("script"): - pp["id_tags"]["SCRIPT"] = text_rep["script"] - if release_detail.get("asin"): - pp["id_tags"]["ASIN"] = release_detail["asin"] - track_num = metadata.get("track_number") - disc_num = metadata.get("disc_number") or 1 - if track_num and media_list: - try: - track_num_int = int(track_num) - disc_num_int = int(disc_num) - for medium in media_list: - if medium.get("position", 1) == disc_num_int: - for mtrack in (medium.get("tracks") or medium.get("track-list", [])): - if mtrack.get("position") == track_num_int: - if mtrack.get("id"): - pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"] - release_recording = mtrack.get("recording", {}) - if release_recording.get("id"): - pp["recording_mbid"] = release_recording["id"] - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"] - break - break - except (ValueError, TypeError): - pass - except Exception as exc: - logger.error("MusicBrainz lookup failed (non-fatal): %s", exc) + _bounded_cache_set(mb_release_detail_cache, pp["release_mbid"], release_detail, _MB_RELEASE_DETAIL_CACHE_MAX_ENTRIES) + if release_detail: + rg = release_detail.get("release-group", {}) + if rg.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"] + ac = release_detail.get("artist-credit", []) + if ac and isinstance(ac[0], dict): + aa = ac[0].get("artist", {}) + if aa.get("id"): + pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"] + if rg.get("primary-type"): + pp["id_tags"]["RELEASETYPE"] = rg["primary-type"] + if rg.get("first-release-date"): + pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"] + if not pp["release_year"] and len(rg["first-release-date"]) >= 4: + year = rg["first-release-date"][:4] + if year.isdigit(): + pp["release_year"] = year + if release_detail.get("status"): + pp["id_tags"]["RELEASESTATUS"] = release_detail["status"] + if release_detail.get("country"): + pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"] + if release_detail.get("barcode"): + pp["id_tags"]["BARCODE"] = release_detail["barcode"] + media_list = release_detail.get("media", []) + if media_list: + fmt = media_list[0].get("format", "") + if fmt: + pp["id_tags"]["MEDIA"] = fmt + pp["id_tags"]["TOTALDISCS"] = str(len(media_list)) + label_info = release_detail.get("label-info", []) + if label_info and isinstance(label_info[0], dict): + cat = label_info[0].get("catalog-number", "") + if cat: + pp["id_tags"]["CATALOGNUMBER"] = cat + text_rep = release_detail.get("text-representation", {}) + if isinstance(text_rep, dict) and text_rep.get("script"): + pp["id_tags"]["SCRIPT"] = text_rep["script"] + if release_detail.get("asin"): + pp["id_tags"]["ASIN"] = release_detail["asin"] + track_num = metadata.get("track_number") + disc_num = metadata.get("disc_number") or 1 + if track_num and media_list: + try: + track_num_int = int(track_num) + disc_num_int = int(disc_num) + for medium in media_list: + if medium.get("position", 1) == disc_num_int: + for mtrack in (medium.get("tracks") or medium.get("track-list", [])): + if mtrack.get("position") == track_num_int: + if mtrack.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"] + release_recording = mtrack.get("recording", {}) + if release_recording.get("id"): + pp["recording_mbid"] = release_recording["id"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"] + break + break + except (ValueError, TypeError): + pass def _process_deezer_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: @@ -315,33 +352,30 @@ def _process_deezer_source(pp: dict, metadata: dict, cfg, runtime, track_title: if not track_title or not artist_name: return - try: - deezer_worker = getattr(runtime, "deezer_worker", None) - dz_client = deezer_worker.client if deezer_worker else None - if not dz_client: - return - dz_result = dz_client.search_track(artist_name, track_title) - if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name): - dz_track_id = dz_result["id"] - pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id) - dz_artist_id = dz_result.get("artist", {}).get("id") - if dz_artist_id: - pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id) - dz_details = dz_client.get_track_details(dz_track_id) - if dz_details: - bpm_val = dz_details.get("bpm") - if bpm_val and bpm_val > 0: - pp["deezer_bpm"] = bpm_val - dz_isrc = dz_details.get("isrc") - if dz_isrc: - pp["deezer_isrc"] = dz_isrc - if not pp["release_year"]: - dz_album = dz_result.get("album", {}) - dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or "" - if len(dz_release) >= 4 and dz_release[:4].isdigit(): - pp["release_year"] = dz_release[:4] - except Exception as exc: - logger.error("Deezer lookup failed (non-fatal): %s", exc) + deezer_worker = getattr(runtime, "deezer_worker", None) + dz_client = deezer_worker.client if deezer_worker else None + if not dz_client: + return + dz_result = _call_source_lookup("Deezer track", dz_client.search_track, artist_name, track_title) + if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name): + dz_track_id = dz_result["id"] + pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id) + dz_artist_id = dz_result.get("artist", {}).get("id") + if dz_artist_id: + pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id) + dz_details = _call_source_lookup("Deezer track details", dz_client.get_track_details, dz_track_id) + if dz_details: + bpm_val = dz_details.get("bpm") + if bpm_val and bpm_val > 0: + pp["deezer_bpm"] = bpm_val + dz_isrc = dz_details.get("isrc") + if dz_isrc: + pp["deezer_isrc"] = dz_isrc + if not pp["release_year"]: + dz_album = dz_result.get("album", {}) + dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or "" + if len(dz_release) >= 4 and dz_release[:4].isdigit(): + pp["release_year"] = dz_release[:4] def _process_audiodb_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: @@ -350,29 +384,26 @@ def _process_audiodb_source(pp: dict, metadata: dict, cfg, runtime, track_title: if not track_title or not artist_name: return - try: - audiodb_worker = getattr(runtime, "audiodb_worker", None) - adb_client = audiodb_worker.client if audiodb_worker else None - if not adb_client: - return - adb_result = adb_client.search_track(artist_name, track_title) - if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name): - adb_track_id = adb_result.get("idTrack") - if adb_track_id: - pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id) - adb_mb_track = adb_result.get("strMusicBrainzID") - if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]: - pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track - pp["recording_mbid"] = adb_mb_track - adb_mb_artist = adb_result.get("strMusicBrainzArtistID") - if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]: - pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist - pp["artist_mbid"] = adb_mb_artist - pp["audiodb_mood"] = adb_result.get("strMood") or None - pp["audiodb_style"] = adb_result.get("strStyle") or None - pp["audiodb_genre"] = adb_result.get("strGenre") or None - except Exception as exc: - logger.error("AudioDB lookup failed (non-fatal): %s", exc) + audiodb_worker = getattr(runtime, "audiodb_worker", None) + adb_client = audiodb_worker.client if audiodb_worker else None + if not adb_client: + return + adb_result = _call_source_lookup("AudioDB track", adb_client.search_track, artist_name, track_title) + if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name): + adb_track_id = adb_result.get("idTrack") + if adb_track_id: + pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id) + adb_mb_track = adb_result.get("strMusicBrainzID") + if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track + pp["recording_mbid"] = adb_mb_track + adb_mb_artist = adb_result.get("strMusicBrainzArtistID") + if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist + pp["artist_mbid"] = adb_mb_artist + pp["audiodb_mood"] = adb_result.get("strMood") or None + pp["audiodb_style"] = adb_result.get("strStyle") or None + pp["audiodb_genre"] = adb_result.get("strGenre") or None def _process_tidal_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: @@ -381,35 +412,32 @@ def _process_tidal_source(pp: dict, metadata: dict, cfg, runtime, track_title: s if not track_title or not artist_name: return - try: - tidal_client = getattr(runtime, "tidal_client", None) - if not (tidal_client and tidal_client.is_authenticated()): - return - td_result = tidal_client.search_track(artist_name, track_title) - if td_result and _names_match(td_result.get("title", ""), track_title): - td_track_id = td_result.get("id") - if td_track_id: - pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id) - td_artist = td_result.get("artist", {}) - if isinstance(td_artist, dict) and td_artist.get("id"): - pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"]) - if td_track_id: - td_details = tidal_client.get_track(str(td_track_id)) - if td_details: - pp["tidal_isrc"] = td_details.get("isrc") - td_copyright = td_details.get("copyright") - if isinstance(td_copyright, dict): - td_copyright = td_copyright.get("text", td_copyright.get("name", "")) - pp["tidal_copyright"] = td_copyright or None - if not pp["release_year"]: - td_album = td_result.get("album", {}) - td_release = "" - if isinstance(td_album, dict): - td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "") - if len(td_release) >= 4 and td_release[:4].isdigit(): - pp["release_year"] = td_release[:4] - except Exception as exc: - logger.error("Tidal lookup failed (non-fatal): %s", exc) + tidal_client = getattr(runtime, "tidal_client", None) + if not (tidal_client and tidal_client.is_authenticated()): + return + td_result = _call_source_lookup("Tidal track", tidal_client.search_track, artist_name, track_title) + if td_result and _names_match(td_result.get("title", ""), track_title): + td_track_id = td_result.get("id") + if td_track_id: + pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id) + td_artist = td_result.get("artist", {}) + if isinstance(td_artist, dict) and td_artist.get("id"): + pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"]) + if td_track_id: + td_details = _call_source_lookup("Tidal track details", tidal_client.get_track, str(td_track_id)) + if td_details: + pp["tidal_isrc"] = td_details.get("isrc") + td_copyright = td_details.get("copyright") + if isinstance(td_copyright, dict): + td_copyright = td_copyright.get("text", td_copyright.get("name", "")) + pp["tidal_copyright"] = td_copyright or None + if not pp["release_year"]: + td_album = td_result.get("album", {}) + td_release = "" + if isinstance(td_album, dict): + td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "") + if len(td_release) >= 4 and td_release[:4].isdigit(): + pp["release_year"] = td_release[:4] def _process_qobuz_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: @@ -418,50 +446,47 @@ def _process_qobuz_source(pp: dict, metadata: dict, cfg, runtime, track_title: s if not track_title or not artist_name: return - try: - qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None) - qz_client = qobuz_worker.client if qobuz_worker else None - if not (qz_client and qz_client.is_authenticated()): - return - qz_result = qz_client.search_track(artist_name, track_title) - if qz_result: - qz_performer = qz_result.get("performer") or {} - if not isinstance(qz_performer, dict): - qz_performer = {} - qz_artist_name = qz_performer.get("name", "") - if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name): - qz_track_id = qz_result.get("id") - if qz_track_id: - pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id) - if qz_performer.get("id"): - pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"]) - qz_isrc = qz_result.get("isrc") - if isinstance(qz_isrc, dict): - qz_isrc = qz_isrc.get("value", qz_isrc.get("id", "")) - if qz_isrc: - pp["qobuz_isrc"] = qz_isrc - qz_copyright = qz_result.get("copyright") - if isinstance(qz_copyright, dict): - qz_copyright = qz_copyright.get("text", qz_copyright.get("name", "")) - if isinstance(qz_copyright, str): - pp["qobuz_copyright"] = qz_copyright - qz_album = qz_result.get("album", {}) - if isinstance(qz_album, dict): - qz_label_info = qz_album.get("label", {}) - if isinstance(qz_label_info, dict) and qz_label_info.get("name"): - pp["qobuz_label"] = qz_label_info["name"] - if not pp["release_year"]: - qz_release = str(qz_album.get("release_date_original", "") or "") - if not qz_release: - qz_ts = qz_album.get("released_at") - if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: - import datetime as _dt + qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None) + qz_client = qobuz_worker.client if qobuz_worker else None + if not (qz_client and qz_client.is_authenticated()): + return + qz_result = _call_source_lookup("Qobuz track", qz_client.search_track, artist_name, track_title) + if qz_result: + qz_performer = qz_result.get("performer") or {} + if not isinstance(qz_performer, dict): + qz_performer = {} + qz_artist_name = qz_performer.get("name", "") + if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name): + qz_track_id = qz_result.get("id") + if qz_track_id: + pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id) + if qz_performer.get("id"): + pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"]) + qz_isrc = qz_result.get("isrc") + if isinstance(qz_isrc, dict): + qz_isrc = qz_isrc.get("value", qz_isrc.get("id", "")) + if qz_isrc: + pp["qobuz_isrc"] = qz_isrc + qz_copyright = qz_result.get("copyright") + if isinstance(qz_copyright, dict): + qz_copyright = qz_copyright.get("text", qz_copyright.get("name", "")) + if isinstance(qz_copyright, str): + pp["qobuz_copyright"] = qz_copyright + qz_album = qz_result.get("album", {}) + if isinstance(qz_album, dict): + qz_label_info = qz_album.get("label", {}) + if isinstance(qz_label_info, dict) and qz_label_info.get("name"): + pp["qobuz_label"] = qz_label_info["name"] + if not pp["release_year"]: + qz_release = str(qz_album.get("release_date_original", "") or "") + if not qz_release: + qz_ts = qz_album.get("released_at") + if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: + import datetime as _dt - qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) - if len(qz_release) >= 4 and qz_release[:4].isdigit(): - pp["release_year"] = qz_release[:4] - except Exception as exc: - logger.error("Qobuz lookup failed (non-fatal): %s", exc) + qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) + if len(qz_release) >= 4 and qz_release[:4].isdigit(): + pp["release_year"] = qz_release[:4] def _process_lastfm_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: @@ -470,25 +495,22 @@ def _process_lastfm_source(pp: dict, metadata: dict, cfg, runtime, track_title: if not track_title or not artist_name: return - try: - lastfm_worker = getattr(runtime, "lastfm_worker", None) - lf_client = lastfm_worker.client if lastfm_worker else None - if not lf_client: - return - lf_result = lf_client.get_track_info(artist_name, track_title) - if lf_result: - lf_url = lf_result.get("url") - if lf_url: - pp["lastfm_url"] = lf_url - lf_toptags = lf_result.get("toptags", {}) - if isinstance(lf_toptags, dict): - tag_list = lf_toptags.get("tag", []) - if isinstance(tag_list, list): - pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")] - elif isinstance(tag_list, dict) and tag_list.get("name"): - pp["lastfm_tags"] = [tag_list["name"]] - except Exception as exc: - logger.error("Last.fm lookup failed (non-fatal): %s", exc) + lastfm_worker = getattr(runtime, "lastfm_worker", None) + lf_client = lastfm_worker.client if lastfm_worker else None + if not lf_client: + return + lf_result = _call_source_lookup("Last.fm track", lf_client.get_track_info, artist_name, track_title) + if lf_result: + lf_url = lf_result.get("url") + if lf_url: + pp["lastfm_url"] = lf_url + lf_toptags = lf_result.get("toptags", {}) + if isinstance(lf_toptags, dict): + tag_list = lf_toptags.get("tag", []) + if isinstance(tag_list, list): + pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")] + elif isinstance(tag_list, dict) and tag_list.get("name"): + pp["lastfm_tags"] = [tag_list["name"]] def _process_genius_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: @@ -497,26 +519,23 @@ def _process_genius_source(pp: dict, metadata: dict, cfg, runtime, track_title: if not track_title or not artist_name: return - try: - import core.genius_client as _genius_module + import core.genius_client as _genius_module - if time.time() < _genius_module._rate_limit_until: - logger.info("Genius rate-limited, skipping (non-blocking)") - return - genius_worker = getattr(runtime, "genius_worker", None) - g_client = genius_worker.client if genius_worker else None - if not g_client: - return - g_result = g_client.search_song(artist_name, track_title) - if g_result: - g_id = g_result.get("id") - if g_id: - pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id) - g_url = g_result.get("url") - if g_url: - pp["genius_url"] = g_url - except Exception as exc: - logger.error("Genius lookup failed (non-fatal): %s", exc) + if time.time() < _genius_module._rate_limit_until: + logger.info("Genius rate-limited, skipping (non-blocking)") + return + genius_worker = getattr(runtime, "genius_worker", None) + g_client = genius_worker.client if genius_worker else None + if not g_client: + return + g_result = _call_source_lookup("Genius track", g_client.search_song, artist_name, track_title) + if g_result: + g_id = g_result.get("id") + if g_id: + pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id) + g_url = g_result.get("url") + if g_url: + pp["genius_url"] = g_url def _process_source_enrichment(source_name: str, pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None: diff --git a/core/runtime_state.py b/core/runtime_state.py index 5e70d38f..f562ad5f 100644 --- a/core/runtime_state.py +++ b/core/runtime_state.py @@ -4,6 +4,7 @@ from __future__ import annotations import threading import time +from functools import wraps from typing import Any, Dict, Optional matched_context_lock = threading.Lock() @@ -20,6 +21,17 @@ activity_feed_lock = threading.Lock() _activity_toast_emitter = None +def caller_must_hold_tasks_lock(func): + """Best-effort guard for helpers that mutate download_tasks in place.""" + @wraps(func) + def wrapper(*args, **kwargs): + if not tasks_lock.locked(): + raise RuntimeError(f"{func.__name__}() requires tasks_lock to be held by the caller") + return func(*args, **kwargs) + + return wrapper + + def set_activity_toast_emitter(emitter) -> None: """Set the WebSocket-style emitter used by add_activity_item.""" global _activity_toast_emitter @@ -50,6 +62,7 @@ def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True): return activity_item +@caller_must_hold_tasks_lock def mark_task_completed(task_id: str, track_info: Optional[Dict[str, Any]] = None) -> bool: """Mark a download task as completed. diff --git a/tests/imports/test_import_paths.py b/tests/imports/test_import_paths.py index 1caaf2eb..e7194089 100644 --- a/tests/imports/test_import_paths.py +++ b/tests/imports/test_import_paths.py @@ -1,3 +1,5 @@ +import os + import core.imports.album_naming as album_naming import core.imports.paths as import_paths @@ -69,7 +71,7 @@ def test_get_file_path_from_template_raw_handles_quality_and_disc_placeholders(m }, ) - assert folder_path == "Artist One/Album One" + assert folder_path == os.path.join("Artist One", "Album One") assert filename == "3 - Song One [FLAC 16bit]" diff --git a/tests/metadata/test_metadata_enrichment.py b/tests/metadata/test_metadata_enrichment.py index b3f87f7d..2c044edd 100644 --- a/tests/metadata/test_metadata_enrichment.py +++ b/tests/metadata/test_metadata_enrichment.py @@ -1,6 +1,10 @@ +from collections import OrderedDict import types import sqlite3 +import pytest +import requests + from core.metadata import enrichment as me from core.metadata import artwork as ma from core.metadata import source as ms @@ -350,6 +354,109 @@ def test_embed_source_ids_writes_musicbrainz_release_year_and_updates_album_year assert row["year"] == 2021 +def test_musicbrainz_release_lookup_failure_does_not_poison_cache(monkeypatch): + class _FakeMBClient: + def get_release(self, mbid, includes=None): + return {} + + class _FakeMBService: + def __init__(self): + self.release_calls = 0 + self.mb_client = _FakeMBClient() + + def match_recording(self, title, artist): + return None + + def match_artist(self, artist): + return {"mbid": "artist-mbid"} + + def match_release(self, album, artist): + self.release_calls += 1 + if self.release_calls == 1: + raise requests.RequestException("temporary MusicBrainz outage") + return {"mbid": "release-mbid"} + + monkeypatch.setattr(ms, "get_config_manager", lambda: _Config({"musicbrainz.embed_tags": True})) + monkeypatch.setattr(ms, "mb_release_cache", {}) + monkeypatch.setattr(ms, "mb_release_detail_cache", {}) + + service = _FakeMBService() + runtime = types.SimpleNamespace(mb_worker=types.SimpleNamespace(mb_service=service)) + pp = { + "id_tags": {}, + "track_title": "Song One", + "artist_name": "Artist One", + "batch_artist_name": "Artist One", + "metadata": {"album": "Album One"}, + "recording_mbid": None, + "artist_mbid": None, + "release_mbid": "", + "mb_genres": [], + "isrc": None, + "deezer_bpm": None, + "deezer_isrc": None, + "audiodb_mood": None, + "audiodb_style": None, + "audiodb_genre": None, + "tidal_isrc": None, + "tidal_copyright": None, + "qobuz_isrc": None, + "qobuz_copyright": None, + "qobuz_label": None, + "lastfm_tags": [], + "lastfm_url": None, + "genius_url": None, + "release_year": None, + } + metadata = {"album": "Album One", "artist": "Artist One"} + + ms._process_musicbrainz_source(pp, metadata, _Config({"musicbrainz.embed_tags": True}), runtime, "Song One", "Artist One") + assert service.release_calls == 1 + assert ms.mb_release_cache == {} + + poisoned_norm_key = (ms.normalize_album_cache_key("Album One"), "artist one") + poisoned_exact_key = ("album one", "artist one") + ms.mb_release_cache[poisoned_norm_key] = "" + ms.mb_release_cache[poisoned_exact_key] = "" + + ms._process_musicbrainz_source(pp, metadata, _Config({"musicbrainz.embed_tags": True}), runtime, "Song One", "Artist One") + assert service.release_calls == 2 + + +def test_source_processors_do_not_swallow_programmer_errors(): + class _BoomClient: + def search_track(self, artist_name, track_title): + raise ValueError("boom") + + runtime = types.SimpleNamespace(deezer_worker=types.SimpleNamespace(client=_BoomClient())) + pp = { + "id_tags": {}, + "batch_artist_name": "Artist One", + "release_year": None, + } + metadata = {"album": "Album One"} + + with pytest.raises(ValueError): + ms._process_deezer_source(pp, metadata, _Config({"deezer.embed_tags": True}), runtime, "Song One", "Artist One") + + +def test_musicbrainz_caches_evict_oldest_entries(): + release_cache = OrderedDict() + detail_cache = OrderedDict() + + ms._bounded_cache_set(release_cache, ("album-1", "artist"), "release-1", 2) + ms._bounded_cache_set(release_cache, ("album-2", "artist"), "release-2", 2) + assert list(release_cache.keys()) == [("album-1", "artist"), ("album-2", "artist")] + + assert ms._bounded_cache_get(release_cache, ("album-1", "artist")) == "release-1" + ms._bounded_cache_set(release_cache, ("album-3", "artist"), "release-3", 2) + assert list(release_cache.keys()) == [("album-1", "artist"), ("album-3", "artist")] + + ms._bounded_cache_set(detail_cache, "release-1", {"title": "One"}, 1) + ms._bounded_cache_set(detail_cache, "release-2", {"title": "Two"}, 1) + assert list(detail_cache.keys()) == ["release-2"] + + def test_enhance_file_metadata_forwards_runtime_to_source_embedding(monkeypatch): audio = _FakeAudio() symbols = _fake_symbols(audio) diff --git a/tests/test_runtime_state.py b/tests/test_runtime_state.py new file mode 100644 index 00000000..87fb779f --- /dev/null +++ b/tests/test_runtime_state.py @@ -0,0 +1,32 @@ +import pytest + +import core.runtime_state as runtime_state + + +def test_mark_task_completed_requires_tasks_lock(): + original_tasks = dict(runtime_state.download_tasks) + runtime_state.download_tasks.clear() + runtime_state.download_tasks["task-1"] = {"status": "running", "stream_processed": False} + + try: + with pytest.raises(RuntimeError, match="tasks_lock"): + runtime_state.mark_task_completed("task-1") + finally: + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(original_tasks) + + +def test_mark_task_completed_succeeds_when_lock_held(): + original_tasks = dict(runtime_state.download_tasks) + runtime_state.download_tasks.clear() + runtime_state.download_tasks["task-1"] = {"status": "running", "stream_processed": False} + + try: + with runtime_state.tasks_lock: + assert runtime_state.mark_task_completed("task-1", {"name": "Song One"}) is True + assert runtime_state.download_tasks["task-1"]["status"] == "completed" + assert runtime_state.download_tasks["task-1"]["stream_processed"] is True + assert runtime_state.download_tasks["task-1"]["track_info"] == {"name": "Song One"} + finally: + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(original_tasks) diff --git a/webui/static/stats-automations.js b/webui/static/stats-automations.js index fe6d1a98..23b6bd4d 100644 --- a/webui/static/stats-automations.js +++ b/webui/static/stats-automations.js @@ -1214,9 +1214,7 @@ function importPageRenderMatchList() { // Build rows let matchedCount = 0; const rows = data.matches.map((m, idx) => { - const track = m.track || m.spotify_track || {}; - const trackNumber = track.track_number ?? track.trackNumber; - const displayTrackNumber = trackNumber ? trackNumber : (idx + 1); + const trackInfo = _importPageGetTrackDisplayInfo(m, idx); let file = null; let confidence = m.confidence; let isOverride = false; @@ -1256,8 +1254,8 @@ function importPageRenderMatchList() { <div class="import-page-match-row ${file ? 'matched' : ''}" ondragover="importPageHandleDragOver(event)" ondragleave="this.classList.remove('drag-over')" ondrop="importPageHandleDrop(event, ${idx})" onclick="importPageTapAssign(${idx})"> - <span class="import-page-match-num">${displayTrackNumber}</span> - <span class="import-page-match-track">${_esc(track.name || track.title || 'Unknown Track')}</span> + <span class="import-page-match-num">${trackInfo.displayTrackNumber}</span> + <span class="import-page-match-track">${_esc(trackInfo.name)}</span> <span class="import-page-match-file ${file ? 'has-file' : ''}"> ${file ? `<span class="import-page-match-file-name">${_esc(file.filename)}</span> @@ -1307,6 +1305,21 @@ function importPageRenderMatchList() { processBtn.textContent = `Process ${matchedCount} Track${matchedCount !== 1 ? 's' : ''}`; } +function _importPageGetTrackDisplayInfo(item, index) { + const track = item?.track || item?.spotify_track || {}; + const rawTrackNumber = track.track_number ?? track.trackNumber ?? null; + const trackNumber = rawTrackNumber === null || rawTrackNumber === undefined || rawTrackNumber === '' + ? null + : String(rawTrackNumber).split('/')[0].trim(); + + return { + track, + name: track.name || track.title || `Track ${index + 1}`, + trackNumber, + displayTrackNumber: trackNumber || String(index + 1), + }; +} + // --- Album Tab: Drag and Drop --- function importPageStartDrag(event, stagingFileIndex) { @@ -1672,7 +1685,7 @@ function _importQueueAdd(job) { async function _importQueueRunJob(entry, job) { for (let i = 0; i < job.items.length; i++) { const itemName = job.type === 'album' - ? (job.items[i].spotify_track?.name || `Track ${i + 1}`) + ? _importPageGetTrackDisplayInfo(job.items[i], i).name : (job.items[i].title || job.items[i].filename || `File ${i + 1}`); // Update status with current track info