From 0bbf44809f382c70bb99d8545ada5f58ebc3e665 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Fri, 24 Apr 2026 16:22:33 +0300 Subject: [PATCH] Move the import flows and related post-processing pipelines into separate modules - Extract the import pipeline, album import, staging, path, file ops, guards, runtime state, side effects, and metadata enrichment out of . - Canonicalize the refactored import path around and remove legacy , , , and request shapes from the import endpoints. - Make album and track metadata lookups follow the configured provider priority instead of hard-coding Spotify, while still falling back when needed. - Update the import routes and frontend payloads to use the new core helpers. - Add coverage for the extracted helpers and the refactored import flows. PS. apologies to anyone who might check this commit out - the intention was to start small, but things kinda snowballed out of control at some point since the logic just kept going on and on, and everything kinda had to be changed all at once for it all to make any sense --- api/downloads.py | 3 +- api/system.py | 4 +- core/import_album.py | 515 ++ core/import_context.py | 326 ++ core/import_file_ops.py | 475 ++ core/import_filename.py | 73 + core/import_guards.py | 121 + core/import_paths.py | 757 +++ core/import_pipeline.py | 925 +++ core/import_runtime_state.py | 121 + core/import_side_effects.py | 567 ++ core/import_staging.py | 454 ++ core/metadata_enrichment.py | 1486 +++++ core/metadata_service.py | 390 +- core/soulseek_client.py | 68 +- core/watchlist_scanner.py | 3 - tests/test_downloads_pagination.py | 20 +- tests/test_import_album.py | 186 + tests/test_import_context.py | 202 + tests/test_import_file_ops.py | 92 + tests/test_import_filename.py | 33 + tests/test_import_guards.py | 47 + tests/test_import_paths.py | 161 + tests/test_import_pipeline.py | 128 + tests/test_import_side_effects.py | 144 + tests/test_import_staging.py | 265 + tests/test_metadata_enrichment.py | 333 ++ ..._metadata_service_single_import_context.py | 193 + tests/test_watchlist_scanner_scan.py | 6 +- web_server.py | 5053 +---------------- 30 files changed, 8238 insertions(+), 4913 deletions(-) create mode 100644 core/import_album.py create mode 100644 core/import_context.py create mode 100644 core/import_file_ops.py create mode 100644 core/import_filename.py create mode 100644 core/import_guards.py create mode 100644 core/import_paths.py create mode 100644 core/import_pipeline.py create mode 100644 core/import_runtime_state.py create mode 100644 core/import_side_effects.py create mode 100644 core/import_staging.py create mode 100644 core/metadata_enrichment.py create mode 100644 tests/test_import_album.py create mode 100644 tests/test_import_context.py create mode 100644 tests/test_import_file_ops.py create mode 100644 tests/test_import_filename.py create mode 100644 tests/test_import_guards.py create mode 100644 tests/test_import_paths.py create mode 100644 tests/test_import_pipeline.py create mode 100644 tests/test_import_side_effects.py create mode 100644 tests/test_import_staging.py create mode 100644 tests/test_metadata_enrichment.py create mode 100644 tests/test_metadata_service_single_import_context.py diff --git a/api/downloads.py b/api/downloads.py index 448b80cd..fd36e91e 100644 --- a/api/downloads.py +++ b/api/downloads.py @@ -5,6 +5,7 @@ Download management endpoints — list, cancel active downloads. from flask import request, current_app from .auth import require_api_key from .helpers import api_success, api_error +from core.import_runtime_state import download_tasks, tasks_lock def _serialize_download(task_id, task): @@ -53,8 +54,6 @@ def register_routes(bp): descending so newest/in-flight tasks appear first. """ try: - from web_server import download_tasks, tasks_lock - # Parse pagination params try: limit = int(request.args.get("limit", 100)) diff --git a/api/system.py b/api/system.py index 4a0ecb03..b6a72141 100644 --- a/api/system.py +++ b/api/system.py @@ -55,7 +55,7 @@ def register_routes(bp): def system_activity(): """Recent activity feed.""" try: - from web_server import activity_feed + from core.import_runtime_state import activity_feed items = list(activity_feed) if activity_feed else [] return api_success({"activities": items}) except Exception as e: @@ -74,7 +74,7 @@ def register_routes(bp): # Active download count download_count = 0 try: - from web_server import download_tasks, tasks_lock + from core.import_runtime_state import download_tasks, tasks_lock with tasks_lock: download_count = sum( 1 for t in download_tasks.values() diff --git a/core/import_album.py b/core/import_album.py new file mode 100644 index 00000000..953e091e --- /dev/null +++ b/core/import_album.py @@ -0,0 +1,515 @@ +"""Album import helpers for staging matching and post-processing context.""" + +from __future__ import annotations + +import os +from typing import Any, Dict, Iterable, List, Optional, Set + +from core.import_context import normalize_import_context +from core.import_file_ops import read_staging_file_metadata +from core.import_staging import AUDIO_EXTENSIONS, get_staging_path +from utils.logging_config import get_logger + + +logger = get_logger("import_album") + +def get_client_for_source(source: str): + from core.metadata_service import get_client_for_source as _get_client_for_source + + return _get_client_for_source(source) + + +def get_artist_album_tracks( + album_id: str, + artist_name: str = "", + album_name: str = "", + source: Optional[str] = None, +): + from core.metadata_service import get_artist_album_tracks as _get_artist_album_tracks + + return _get_artist_album_tracks( + album_id, + artist_name=artist_name, + album_name=album_name, + source_override=source, + ) + + +try: + from core.matching_engine import MusicMatchingEngine + _MATCHING_ENGINE_IMPORT_ERROR = None +except Exception as exc: # pragma: no cover - only hits in stripped-down environments + MusicMatchingEngine = None # type: ignore[assignment] + _MATCHING_ENGINE_IMPORT_ERROR = exc + + +_MATCHING_ENGINE = None + + +def _get_matching_engine() -> Any: + global _MATCHING_ENGINE + if _MATCHING_ENGINE is None: + if MusicMatchingEngine is None: + raise RuntimeError("Music matching engine is unavailable") from _MATCHING_ENGINE_IMPORT_ERROR + _MATCHING_ENGINE = MusicMatchingEngine() + return _MATCHING_ENGINE + + +def _normalize_artist_entries(artists: Any) -> List[Dict[str, Any]]: + if not artists: + return [] + + if isinstance(artists, (str, bytes)): + artists = [artists] + elif isinstance(artists, dict): + artists = [artists] + else: + try: + artists = list(artists) + except TypeError: + artists = [artists] + + normalized: List[Dict[str, Any]] = [] + for artist in artists: + if isinstance(artist, dict): + entry: Dict[str, Any] = {} + name = artist.get("name") or artist.get("artist_name") or artist.get("title") or "" + artist_id = artist.get("id") or artist.get("artist_id") or "" + if name: + entry["name"] = str(name) + if artist_id: + entry["id"] = str(artist_id) + genres = artist.get("genres") + if genres is not None: + entry["genres"] = genres + if entry: + normalized.append(entry) + continue + + name = str(artist).strip() + if name: + normalized.append({"name": name}) + + return normalized + + +def _normalize_album_source(album: Dict[str, Any], source: str = "") -> str: + album_source = source or album.get("source") or "" + return str(album_source).strip().lower() + + +def _strip_legacy_source_fields(payload: Any) -> Any: + if not isinstance(payload, dict): + return payload + + cleaned = dict(payload) + cleaned.pop("_source", None) + cleaned.pop("provider", None) + return cleaned + + +def _extract_track_artist_name(track: Dict[str, Any]) -> str: + artists = track.get("artists") or [] + if isinstance(artists, (str, bytes)): + artists = [artists] + elif isinstance(artists, dict): + artists = [artists] + else: + try: + artists = list(artists) + except TypeError: + artists = [artists] + + if not artists: + return "" + + first = artists[0] + if isinstance(first, dict): + return str(first.get("name") or first.get("artist_name") or first.get("title") or "").strip() + return str(first or "").strip() + + +def _coerce_track_int(value: Any, default: int = 1) -> int: + if value in (None, ""): + return default + try: + return int(str(value).split("/")[0].strip() or default) + except (TypeError, ValueError): + return default + + +def _collect_staging_files(file_paths: Optional[Iterable[str]] = None) -> List[Dict[str, Any]]: + staging_path = get_staging_path() + file_filter: Optional[Set[str]] = set(file_paths) if file_paths else None + staging_files: List[Dict[str, Any]] = [] + + if not os.path.isdir(staging_path): + return staging_files + + for root, _dirs, filenames in os.walk(staging_path): + for filename in filenames: + ext = os.path.splitext(filename)[1].lower() + if ext not in AUDIO_EXTENSIONS: + continue + + full_path = os.path.join(root, filename) + if file_filter is not None and full_path not in file_filter: + continue + + meta = read_staging_file_metadata(full_path, filename) + staging_files.append( + { + "filename": filename, + "full_path": full_path, + "title": meta.get("title", ""), + "artist": meta.get("albumartist") or meta.get("artist") or "", + "album": meta.get("album", ""), + "albumartist": meta.get("albumartist") or meta.get("artist") or "", + "track_number": meta.get("track_number", 1), + "disc_number": meta.get("disc_number", 1), + } + ) + + return staging_files + + +def _normalize_match_track(track: Dict[str, Any], source: str, album: Dict[str, Any]) -> Dict[str, Any]: + track_album = track.get("album") if isinstance(track.get("album"), dict) else album + if isinstance(track_album, dict): + track_album = _strip_legacy_source_fields(track_album) + track_source = _normalize_album_source(track, source) + track_artists = _normalize_artist_entries(track.get("artists") or []) + + if not track_artists and album.get("artists"): + track_artists = _normalize_artist_entries(album.get("artists")) + + return { + "id": track.get("id", ""), + "name": track.get("name", "Unknown Track"), + "track_number": _coerce_track_int(track.get("track_number", 1), default=1), + "disc_number": _coerce_track_int(track.get("disc_number", 1), default=1), + "duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0), + "artists": track_artists, + "uri": track.get("uri", ""), + "album": track_album, + "source": track_source, + } + + +def _score_album_track_match(track: Dict[str, Any], staging_file: Dict[str, Any], album_name: str) -> float: + engine = _get_matching_engine() + + track_name = track.get("name", "") + staging_title = staging_file.get("title", "") + score = 0.0 + + title_sim = engine.similarity_score( + engine.normalize_string(track_name), + engine.normalize_string(staging_title or ""), + ) + score += title_sim * 0.45 + + track_artist_name = _extract_track_artist_name(track) + staging_artist = staging_file.get("artist") or "" + if track_artist_name and staging_artist: + artist_sim = engine.similarity_score( + engine.normalize_string(track_artist_name), + engine.normalize_string(staging_artist), + ) + score += artist_sim * 0.15 + else: + score += 0.075 + + track_number = _coerce_track_int(track.get("track_number", 1), default=1) + staging_track_number = _coerce_track_int(staging_file.get("track_number", 1), default=1) + if staging_track_number and track_number: + if staging_track_number == track_number: + score += 0.30 + elif abs(staging_track_number - track_number) <= 1: + score += 0.12 + + staging_album = staging_file.get("album") or "" + if staging_album and album_name: + album_sim = engine.similarity_score( + engine.normalize_string(staging_album), + engine.normalize_string(album_name), + ) + score += album_sim * 0.10 + + return score + + +def _fetch_artist_data_for_source(client: Any, artist_id: str, source: str) -> Any: + if source == "spotify": + try: + return client.get_artist(artist_id, allow_fallback=False) + except TypeError: + return client.get_artist(artist_id) + return client.get_artist(artist_id) + + +def resolve_album_artist_context(album: Dict[str, Any], source: str = "") -> Dict[str, Any]: + """Build a neutral artist context for album import processing.""" + album = dict(album or {}) + source = _normalize_album_source(album, source) + + artists = _normalize_artist_entries(album.get("artists") or []) + if not artists: + artist_name = album.get("artist") or album.get("artist_name") or "" + artist_id = album.get("artist_id") or "" + if artist_name or artist_id: + artist_entry: Dict[str, Any] = {} + if artist_name: + artist_entry["name"] = str(artist_name) + if artist_id: + artist_entry["id"] = str(artist_id) + artists = [artist_entry] + + primary_artist = artists[0] if artists else {} + artist_name = str( + primary_artist.get("name") + or album.get("artist") + or album.get("artist_name") + or "Unknown Artist" + ).strip() + artist_id = str(primary_artist.get("id") or album.get("artist_id") or "").strip() + + genres: List[Any] = [] + if artist_id and source: + client = get_client_for_source(source) + if client and hasattr(client, "get_artist"): + try: + artist_data = _fetch_artist_data_for_source(client, artist_id, source) + raw_genres = artist_data.get("genres") if isinstance(artist_data, dict) else getattr(artist_data, "genres", []) + if isinstance(raw_genres, str): + genres = [raw_genres] + elif raw_genres: + try: + genres = list(raw_genres) + except TypeError: + genres = [raw_genres] + except Exception as exc: + logger.debug("Could not resolve artist genres for %s on %s: %s", artist_id, source, exc) + + return { + "id": artist_id, + "name": artist_name, + "genres": genres, + "source": source, + } + + +def build_album_import_context( + album: Dict[str, Any], + track: Dict[str, Any], + *, + artist_context: Optional[Dict[str, Any]] = None, + total_discs: int = 1, + source: str = "", +) -> Dict[str, Any]: + """Build a neutral post-processing context for one album track.""" + album = dict(album or {}) + track = dict(track or {}) + source = _normalize_album_source(album, source) + + album_artists = _normalize_artist_entries(album.get("artists") or []) + if not album_artists and artist_context: + album_artists = _normalize_artist_entries([artist_context]) + + if artist_context: + artist_ctx = dict(artist_context) + else: + artist_ctx = resolve_album_artist_context(album, source) + + artist_ctx = _strip_legacy_source_fields(artist_ctx) + artist_ctx.setdefault("genres", []) + artist_ctx.setdefault("source", source) + artist_ctx["genres"] = artist_ctx.get("genres") or [] + + track_artists = _normalize_artist_entries(track.get("artists") or []) + if not track_artists: + track_artists = album_artists or [artist_ctx] + + track_album_value = track.get("album") + if isinstance(track_album_value, dict): + track_album_name = ( + track_album_value.get("name") + or track_album_value.get("title") + or album.get("name") + or album.get("album_name") + or "" + ) + track_album_id = str(track_album_value.get("id") or track_album_value.get("album_id") or "").strip() + track_album_type = track_album_value.get("album_type") or album.get("album_type") or "album" + track_album_release = track_album_value.get("release_date") or album.get("release_date") or "" + track_album_image = track_album_value.get("image_url") or album.get("image_url") or "" + else: + track_album_name = str(track_album_value or album.get("name") or album.get("album_name") or "").strip() + track_album_id = str(album.get("id") or album.get("album_id") or "").strip() + track_album_type = album.get("album_type") or "album" + track_album_release = album.get("release_date") or "" + track_album_image = album.get("image_url") or "" + + album_name = str(album.get("name") or album.get("album_name") or track_album_name or "Unknown Album").strip() + artist_name = str( + artist_ctx.get("name") + or album.get("artist") + or album.get("artist_name") + or "Unknown Artist" + ).strip() + + track_number = _coerce_track_int(track.get("track_number", 1), default=1) + disc_number = _coerce_track_int(track.get("disc_number", 1), default=1) + + normalized_track = { + "id": str(track.get("id") or track.get("track_id") or "").strip(), + "name": str(track.get("name") or "Unknown Track").strip(), + "track_number": track_number, + "disc_number": disc_number, + "duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0), + "artists": track_artists, + "uri": str(track.get("uri") or "").strip(), + "album": track_album_name, + "album_id": track_album_id, + "album_type": track_album_type, + "release_date": track_album_release, + "source": source, + } + + normalized_album = { + "id": str(album.get("id") or album.get("album_id") or track_album_id or "").strip(), + "name": album_name, + "artist": artist_name, + "artist_name": artist_name, + "artist_id": str(artist_ctx.get("id") or album.get("artist_id") or "").strip(), + "artists": album_artists, + "release_date": str(album.get("release_date") or track_album_release or "").strip(), + "total_tracks": int(album.get("total_tracks") or track.get("total_tracks") or 0) or 1, + "total_discs": int(total_discs or 1) if str(total_discs or 1).isdigit() else total_discs or 1, + "album_type": str(album.get("album_type") or track_album_type or "album").strip() or "album", + "image_url": str(album.get("image_url") or track_album_image or "").strip(), + "images": album.get("images") or ([] if not track_album_image else [{"url": track_album_image}]), + "source": source, + } + + original_search = { + "title": normalized_track["name"], + "artist": artist_name, + "album": album_name, + "track_number": track_number, + "disc_number": disc_number, + "clean_title": normalized_track["name"], + "clean_album": album_name, + "clean_artist": artist_name, + "artists": track_artists, + "duration_ms": normalized_track["duration_ms"], + "id": normalized_track["id"], + "source": source, + } + + context = { + "artist": artist_ctx, + "album": normalized_album, + "track_info": normalized_track, + "original_search_result": original_search, + "is_album_download": True, + "has_clean_metadata": bool(normalized_track["id"]), + "has_full_metadata": bool(normalized_track["id"]), + "source": source, + } + + normalized_context = normalize_import_context(context) + normalized_context["artist"] = _strip_legacy_source_fields(normalized_context.get("artist")) + normalized_context["album"] = _strip_legacy_source_fields(normalized_context.get("album")) + normalized_context["track_info"] = _strip_legacy_source_fields(normalized_context.get("track_info")) + normalized_context["original_search_result"] = _strip_legacy_source_fields(normalized_context.get("original_search_result")) + return normalized_context + + +def build_album_import_match_payload( + album_id: str, + *, + album_name: str = "", + album_artist: str = "", + file_paths: Optional[Iterable[str]] = None, + source: Optional[str] = None, +) -> Dict[str, Any]: + """Build the album import match payload using provider-priority metadata lookup.""" + album_response = get_artist_album_tracks( + album_id, + artist_name=album_artist, + album_name=album_name, + source=source, + ) + + album = _strip_legacy_source_fields(dict(album_response.get("album") or {})) + source = _normalize_album_source(album, album_response.get("source") or source or "") + tracks = list(album_response.get("tracks") or []) + if not album_response.get("success") or not tracks: + return { + "success": False, + "error": album_response.get("error", "Album not found"), + "status_code": album_response.get("status_code", 404), + "album": { + "id": album_id, + "name": album_name or album_id, + "artist": album_artist or "Unknown Artist", + "artist_name": album_artist or "Unknown Artist", + "artist_id": "", + "artists": [], + "release_date": "", + "total_tracks": 0, + "total_discs": 1, + "album_type": "album", + "image_url": "", + "images": [], + "source": source, + }, + "matches": [], + "unmatched_files": [], + "source": source, + "source_priority": album_response.get("source_priority", []), + "resolved_album_id": album_response.get("resolved_album_id") or album_id, + } + + staging_files = _collect_staging_files(file_paths) + album_name_for_match = album.get("name") or album_name or "" + matches: List[Dict[str, Any]] = [] + used_files: Set[int] = set() + + for track in tracks: + normalized_track = _normalize_match_track(track, source, album) + best_match = None + best_score = 0.0 + + for index, staging_file in enumerate(staging_files): + if index in used_files: + continue + + score = _score_album_track_match(normalized_track, staging_file, album_name_for_match) + if score > best_score and score >= 0.4: + best_score = score + best_match = index + + matches.append( + { + "track": normalized_track, + "staging_file": staging_files[best_match] if best_match is not None else None, + "confidence": round(best_score, 2) if best_match is not None else 0, + } + ) + + if best_match is not None: + used_files.add(best_match) + + unmatched_files = [sf for index, sf in enumerate(staging_files) if index not in used_files] + + return { + "success": True, + "album": album, + "matches": matches, + "unmatched_files": unmatched_files, + "source": source, + "source_priority": album_response.get("source_priority", []), + "resolved_album_id": album_response.get("resolved_album_id") or album_id, + } diff --git a/core/import_context.py b/core/import_context.py new file mode 100644 index 00000000..cf9504ff --- /dev/null +++ b/core/import_context.py @@ -0,0 +1,326 @@ +"""Helpers for normalizing and reading import contexts. + +These functions keep the single-import pipeline source-agnostic while still +accepting legacy `spotify_*` payloads from older callers. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + + +def _as_dict(value: Any) -> Dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _first_value(mapping: Dict[str, Any], *keys: str, default: Any = "") -> Any: + for key in keys: + if key in mapping: + value = mapping.get(key) + if value not in (None, ""): + return value + return default + + +def _first_id_value(*values: Any) -> str: + for value in values: + if value in (None, ""): + continue + text = str(value).strip() + if text: + return text + return "" + + +def extract_artist_name(artist: Any) -> str: + if isinstance(artist, dict): + return str(artist.get("name", "") or "") + if hasattr(artist, "name"): + return str(getattr(artist, "name") or "") + return str(artist) if artist else "" + + +def normalize_import_context(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """Normalize an import context to neutral fields in place and drop legacy aliases.""" + if not isinstance(context, dict): + return {} + + artist = _as_dict(context.get("artist") or context.get("spotify_artist")) + album = _as_dict(context.get("album") or context.get("spotify_album")) + track_info = _as_dict(context.get("track_info")) + original_search = _as_dict(context.get("original_search_result")) + + context["artist"] = artist + context["album"] = album + context["track_info"] = track_info + context["original_search_result"] = original_search + context.pop("spotify_artist", None) + context.pop("spotify_album", None) + + for clean_key, legacy_key in ( + ("clean_title", "spotify_clean_title"), + ("clean_album", "spotify_clean_album"), + ("clean_artist", "spotify_clean_artist"), + ): + if clean_key not in original_search or original_search.get(clean_key) in (None, ""): + legacy_value = original_search.get(legacy_key) + if legacy_value not in (None, ""): + original_search[clean_key] = legacy_value + original_search.pop(legacy_key, None) + + has_clean = bool(context.get("has_clean_metadata", context.get("has_clean_spotify_data", False))) + has_full = bool(context.get("has_full_metadata", context.get("has_full_spotify_metadata", False))) + context["has_clean_metadata"] = has_clean + context["has_full_metadata"] = has_full + context.pop("has_clean_spotify_data", None) + context.pop("has_full_spotify_metadata", None) + + return context + + +def get_import_context_artist(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("artist") or context.get("spotify_artist")) + + +def get_import_context_album(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("album") or context.get("spotify_album")) + + +def get_import_track_info(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("track_info")) + + +def get_import_original_search(context: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not isinstance(context, dict): + return {} + return _as_dict(context.get("original_search_result")) + + +def get_import_source(context: Optional[Dict[str, Any]]) -> str: + if not isinstance(context, dict): + return "" + + source = context.get("source") + if source: + return str(source) + + track_info = get_import_track_info(context) + source = _first_value(track_info, "source", default="") + if source: + return str(source) + + original_search = get_import_original_search(context) + source = _first_value(original_search, "source", default="") + if source: + return str(source) + + album = get_import_context_album(context) + source = _first_value(album, "source", default="") + if source: + return str(source) + + artist = get_import_context_artist(context) + source = _first_value(artist, "source", default="") + return str(source) if source else "" + + +def get_import_clean_title( + context: Optional[Dict[str, Any]], + album_info: Optional[Dict[str, Any]] = None, + default: str = "Unknown Track", +) -> str: + original_search = get_import_original_search(context) + title = _first_value( + original_search, + "clean_title", + "title", + default="", + ) + if not title and album_info: + title = _first_value(album_info, "clean_track_name", "track_name", default="") + if not title: + track_info = get_import_track_info(context) + title = _first_value(track_info, "name", "title", default="") + return str(title or default) + + +def get_import_clean_album( + context: Optional[Dict[str, Any]], + album_info: Optional[Dict[str, Any]] = None, + default: str = "Unknown Album", +) -> str: + original_search = get_import_original_search(context) + album = _first_value( + original_search, + "clean_album", + "album", + default="", + ) + if not album and album_info: + album = _first_value(album_info, "album_name", "clean_album_name", default="") + if not album: + album_ctx = get_import_context_album(context) + album = _first_value(album_ctx, "name", default="") + return str(album or default) + + +def get_import_clean_artist(context: Optional[Dict[str, Any]], default: str = "Unknown Artist") -> str: + original_search = get_import_original_search(context) + artist = _first_value( + original_search, + "clean_artist", + "artist", + default="", + ) + if not artist: + artist_ctx = get_import_context_artist(context) + artist = _first_value(artist_ctx, "name", default="") + return str(artist or default) + + +def get_import_has_clean_metadata(context: Optional[Dict[str, Any]]) -> bool: + if not isinstance(context, dict): + return False + return bool(context.get("has_clean_metadata", False)) + + +def get_import_has_full_metadata(context: Optional[Dict[str, Any]]) -> bool: + if not isinstance(context, dict): + return False + return bool(context.get("has_full_metadata", False)) + + +def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]: + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + artist = get_import_context_artist(context) + album = get_import_context_album(context) + + return { + "track_id": _first_id_value( + _first_value(track_info, "id", "track_id", "trackId", "source_track_id", default=""), + _first_value(track_info, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""), + _first_value(original_search, "id", "track_id", "source_track_id", default=""), + _first_value(original_search, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""), + ), + "artist_id": _first_id_value( + _first_value(artist, "id", "artist_id", "source_artist_id", default=""), + _first_value(artist, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""), + _first_value(original_search, "artist_id", "source_artist_id", default=""), + _first_value(original_search, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""), + ), + "album_id": _first_id_value( + _first_value(album, "id", "album_id", "collectionId", "source_album_id", default=""), + _first_value(album, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), + _first_value(original_search, "album_id", "source_album_id", default=""), + _first_value(original_search, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), + _first_value(track_info, "album_id", "source_album_id", default=""), + _first_value(track_info, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""), + ), + } + + +def get_source_tag_names(source: str) -> Dict[str, Optional[str]]: + source_name = (source or "").strip().lower() + if source_name == "spotify": + return {"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"} + if source_name == "itunes": + return {"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"} + if source_name == "deezer": + return {"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None} + if source_name == "hydrabase": + return {"track": None, "artist": None, "album": None} + if source_name == "discogs": + return {"track": None, "artist": None, "album": None} + return {"track": None, "artist": None, "album": None} + + +def get_library_source_id_columns(source: str) -> Dict[str, Optional[str]]: + source_name = (source or "").strip().lower() + if source_name == "spotify": + return {"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"} + if source_name == "itunes": + return {"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"} + if source_name == "deezer": + return {"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"} + if source_name == "hydrabase": + return {"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"} + if source_name == "discogs": + return {"artist": "discogs_id", "album": "discogs_id", "track": None} + return {} + + +def build_import_album_info( + context: Optional[Dict[str, Any]], + *, + album_info: Optional[Dict[str, Any]] = None, + force_album: bool = False, +) -> Dict[str, Any]: + """Build the album-info payload used by post-processing.""" + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + artist_ctx = get_import_context_artist(context) + + track_number = ( + (album_info or {}).get("track_number") + or track_info.get("track_number") + or original_search.get("track_number") + or 1 + ) + disc_number = ( + (album_info or {}).get("disc_number") + or track_info.get("disc_number") + or original_search.get("disc_number") + or 1 + ) + + clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track")) + album_name = get_import_clean_album(context, album_info=album_info, default=original_search.get("album", "Unknown Album")) + album_image_url = ( + (album_info or {}).get("album_image_url") + or album_ctx.get("image_url") + or "" + ) + total_tracks = ( + album_ctx.get("total_tracks") + or track_info.get("total_tracks") + or (album_info or {}).get("total_tracks") + or 0 + ) + album_type = (album_ctx.get("album_type") or track_info.get("album_type") or "album") + source = get_import_source(context) + + artist_name = artist_ctx.get("name") or original_search.get("artist") or get_import_clean_artist(context) + normalized_album = str(album_name or "").strip().lower() + normalized_title = str(clean_track_name or "").strip().lower() + normalized_artist = str(artist_name or "").strip().lower() + is_album = bool( + force_album + or ( + normalized_album + and total_tracks + and int(total_tracks) > 1 + and normalized_album != normalized_title + and normalized_album != normalized_artist + ) + ) + + return { + "is_album": is_album, + "album_name": album_name, + "track_number": int(track_number) if str(track_number).isdigit() else track_number, + "disc_number": int(disc_number) if str(disc_number).isdigit() else disc_number, + "clean_track_name": clean_track_name, + "album_image_url": album_image_url, + "confidence": (album_info or {}).get("confidence", 1.0 if is_album or force_album else 0.0), + "source": source, + "album_type": album_type, + "total_tracks": int(total_tracks) if str(total_tracks).isdigit() else total_tracks, + } diff --git a/core/import_file_ops.py b/core/import_file_ops.py new file mode 100644 index 00000000..261687a8 --- /dev/null +++ b/core/import_file_ops.py @@ -0,0 +1,475 @@ +"""Shared file and path helpers for import processing.""" + +from __future__ import annotations + +import logging +import os +import re +import shutil +import subprocess +import time +from pathlib import Path +from typing import Any, Dict, Optional + +logger = logging.getLogger("import_file_ops") + + +def _get_config_manager(): + from config.settings import config_manager + return config_manager + + +def safe_move_file(src, dst): + """Move a file safely across filesystems.""" + src = Path(src) + dst = Path(dst) + + dst.parent.mkdir(parents=True, exist_ok=True) + + if not src.exists(): + if dst.exists(): + logger.info(f"Source gone but destination exists, file already transferred: {dst.name}") + return + raise FileNotFoundError(f"Source file not found and destination does not exist: {src}") + + if dst.exists(): + for _attempt in range(3): + try: + dst.unlink() + break + except PermissionError: + if _attempt < 2: + time.sleep(1) + else: + logger.warning(f"Could not remove locked destination after 3 attempts: {dst.name}") + except Exception: + break + + try: + shutil.move(str(src), str(dst)) + return + except FileNotFoundError: + if dst.exists(): + logger.info(f"Source moved by another thread, destination exists: {dst.name}") + return + raise + except (OSError, PermissionError) as e: + error_msg = str(e).lower() + + if dst.exists() and dst.stat().st_size > 0: + logger.warning(f"Move raised {type(e).__name__} but destination exists, treating as success: {e}") + try: + src.unlink() + except Exception: + logger.info(f"Could not delete source file (may be owned by another process): {src}") + return + + if "cross-device" in error_msg or "operation not permitted" in error_msg or "permission denied" in error_msg: + logger.warning(f"Cross-device move detected, using fallback copy method: {e}") + try: + with open(src, "rb") as f_src: + with open(dst, "wb") as f_dst: + shutil.copyfileobj(f_src, f_dst) + f_dst.flush() + os.fsync(f_dst.fileno()) + + try: + src.unlink() + except PermissionError: + logger.info(f"Could not delete source file (may be owned by another process): {src}") + logger.info(f"Successfully moved file using fallback method: {src} -> {dst}") + return + except Exception as fallback_error: + logger.error(f"Fallback copy also failed: {fallback_error}") + raise + raise + + +def extract_track_number_from_filename(filename: str, title: str = None) -> int: + """Extract track number from a filename. Returns 1 if not found.""" + basename = os.path.splitext(os.path.basename(filename))[0].strip() + + match = re.match(r"^\d[\-\.](\d{1,2})\s*[\-\.]\s*", basename) + if match: + num = int(match.group(1)) + if 1 <= num <= 99: + return num + + match = re.match(r"^\(?(\d{1,3})\)?\s*[\-\.)\]]\s*", basename) + if match: + num = int(match.group(1)) + if 1 <= num <= 999: + return num + + return 1 + + +def _coerce_tag_number(value: Any, default: int = 1) -> int: + if value in (None, ""): + return default + + if isinstance(value, (list, tuple)): + value = value[0] if value else None + + if value in (None, ""): + return default + + text = str(value).strip() + if not text: + return default + + match = re.match(r"^(\d+)", text) + if match: + try: + return int(match.group(1)) + except ValueError: + return default + + try: + return int(text) + except (TypeError, ValueError): + return default + + +def read_staging_file_metadata(file_path: str, filename: Optional[str] = None) -> Dict[str, Any]: + """Read common audio tag metadata from a staging file.""" + try: + from mutagen import File as MutagenFile + + tags = MutagenFile(file_path, easy=True) + except Exception: + tags = None + + filename = filename or os.path.basename(file_path) + stem = os.path.splitext(os.path.basename(filename))[0] + + def _first_tag(*keys: str) -> str: + if not tags: + return "" + for key in keys: + try: + value = tags.get(key) # type: ignore[attr-defined] + except Exception: + value = None + if value: + if isinstance(value, (list, tuple)): + value = value[0] if value else "" + text = str(value).strip() + if text: + return text + return "" + + title = _first_tag("title") + artist = _first_tag("artist") + albumartist = _first_tag("albumartist") + album = _first_tag("album") + + if not title: + title = stem + if not albumartist: + albumartist = artist + + track_number = _coerce_tag_number(_first_tag("tracknumber", "track_number"), default=0) + if not track_number: + track_number = extract_track_number_from_filename(filename or file_path) + + disc_number = _coerce_tag_number(_first_tag("discnumber", "disc_number"), default=1) + + return { + "title": title, + "artist": artist, + "albumartist": albumartist, + "album": album, + "track_number": track_number, + "disc_number": disc_number, + } + + +def cleanup_empty_directories(download_path, moved_file_path): + """Remove empty directories after a move, ignoring hidden files.""" + try: + current_dir = os.path.dirname(moved_file_path) + while current_dir != download_path and current_dir.startswith(download_path): + is_empty = not any(not f.startswith(".") for f in os.listdir(current_dir)) + if is_empty: + logger.warning(f"Removing empty directory: {current_dir}") + os.rmdir(current_dir) + current_dir = os.path.dirname(current_dir) + else: + break + except Exception as e: + logger.error(f"An error occurred during directory cleanup: {e}") + + +def get_audio_quality_string(file_path): + """Return a compact audio quality string for the given file.""" + try: + ext = os.path.splitext(file_path)[1].lower() + + if ext == ".flac": + from mutagen.flac import FLAC + audio = FLAC(file_path) + return f"FLAC {audio.info.bits_per_sample}bit" + + if ext == ".mp3": + from mutagen.mp3 import MP3, BitrateMode + + audio = MP3(file_path) + bitrate_kbps = audio.info.bitrate // 1000 + if audio.info.bitrate_mode == BitrateMode.VBR: + return "MP3-VBR" + return f"MP3-{bitrate_kbps}" + + if ext in (".m4a", ".aac", ".mp4"): + from mutagen.mp4 import MP4 + audio = MP4(file_path) + return f"M4A-{audio.info.bitrate // 1000}" + + if ext == ".ogg": + from mutagen.oggvorbis import OggVorbis + audio = OggVorbis(file_path) + return f"OGG-{audio.info.bitrate // 1000}" + + if ext == ".opus": + from mutagen.oggopus import OggOpus + + audio = OggOpus(file_path) + return f"OPUS-{audio.info.bitrate // 1000}" + + return "" + except Exception as e: + logger.debug(f"Could not determine audio quality for {file_path}: {e}") + return "" + + +def get_quality_tier_from_extension(file_path): + """Classify a file extension into a quality tier.""" + if not file_path: + return ("unknown", 999) + + ext = os.path.splitext(file_path)[1].lower() + quality_tiers = { + "lossless": { + "extensions": [".flac", ".ape", ".wav", ".alac", ".dsf", ".dff", ".aiff", ".aif"], + "tier": 1, + }, + "high_lossy": { + "extensions": [".opus", ".ogg"], + "tier": 2, + }, + "standard_lossy": { + "extensions": [".m4a", ".aac"], + "tier": 3, + }, + "low_lossy": { + "extensions": [".mp3", ".wma"], + "tier": 4, + }, + } + + for tier_name, tier_data in quality_tiers.items(): + if ext in tier_data["extensions"]: + return (tier_name, tier_data["tier"]) + + return ("unknown", 999) + + +def downsample_hires_flac(final_path, context): + """Downsample a hi-res FLAC to 16-bit/44.1kHz if enabled.""" + from mutagen.flac import FLAC + + config_manager = _get_config_manager() + if not config_manager.get("lossy_copy.downsample_hires", False): + return None + + if os.path.splitext(final_path)[1].lower() != ".flac": + return None + + try: + audio = FLAC(final_path) + original_bits = audio.info.bits_per_sample + original_rate = audio.info.sample_rate + except Exception as e: + logger.error(f"[Downsample] Could not read FLAC info: {e}") + return None + + if original_bits <= 16 and original_rate <= 44100: + return None + + logger.info(f"[Downsample] Converting {original_bits}-bit/{original_rate}Hz -> 16-bit/44100Hz: {os.path.basename(final_path)}") + + ffmpeg_bin = shutil.which("ffmpeg") + if not ffmpeg_bin: + local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg") + if os.path.isfile(local): + ffmpeg_bin = local + else: + logger.warning("[Downsample] ffmpeg not found - skipping hi-res conversion") + return None + + temp_path = final_path + ".tmp.flac" + try: + result = subprocess.run( + [ + ffmpeg_bin, "-i", final_path, + "-sample_fmt", "s16", + "-ar", "44100", + "-map_metadata", "0", + "-compression_level", "8", + "-y", temp_path, + ], + capture_output=True, + text=True, + timeout=300, + ) + + if result.returncode != 0: + logger.error(f"[Downsample] ffmpeg failed: {result.stderr[:200]}") + if os.path.exists(temp_path): + os.remove(temp_path) + return None + + if not os.path.isfile(temp_path) or os.path.getsize(temp_path) == 0: + logger.warning("[Downsample] Output file missing or empty") + if os.path.exists(temp_path): + os.remove(temp_path) + return None + + verify_audio = FLAC(temp_path) + if verify_audio.info.bits_per_sample != 16: + logger.info(f"[Downsample] Output not 16-bit ({verify_audio.info.bits_per_sample}-bit), aborting") + os.remove(temp_path) + return None + + os.replace(temp_path, final_path) + logger.info(f"[Downsample] Converted to 16-bit/44.1kHz: {os.path.basename(final_path)}") + + new_quality = "FLAC 16bit" + try: + updated_audio = FLAC(final_path) + updated_audio["QUALITY"] = new_quality + updated_audio.save() + except Exception as tag_err: + logger.error(f"[Downsample] Could not update QUALITY tag: {tag_err}") + + old_quality = context.get("_audio_quality", "") + context["_audio_quality"] = new_quality + + if old_quality and old_quality != new_quality and old_quality in os.path.basename(final_path): + new_basename = os.path.basename(final_path).replace(old_quality, new_quality) + new_path = os.path.join(os.path.dirname(final_path), new_basename) + try: + os.rename(final_path, new_path) + logger.info(f"[Downsample] Renamed: {os.path.basename(final_path)} -> {new_basename}") + for lyrics_ext in (".lrc", ".txt"): + old_lyrics = os.path.splitext(final_path)[0] + lyrics_ext + if os.path.isfile(old_lyrics): + new_lyrics = os.path.splitext(new_path)[0] + lyrics_ext + os.rename(old_lyrics, new_lyrics) + return new_path + except Exception as rename_err: + logger.error(f"[Downsample] Could not rename file: {rename_err}") + + return final_path + except subprocess.TimeoutExpired: + logger.info(f"[Downsample] Conversion timed out for: {os.path.basename(final_path)}") + if os.path.exists(temp_path): + os.remove(temp_path) + except Exception as e: + logger.error(f"[Downsample] Conversion error: {e}") + if os.path.exists(temp_path): + try: + os.remove(temp_path) + except Exception: + pass + return None + + +def create_lossy_copy(final_path): + """Convert a FLAC file to a lossy copy using the configured codec.""" + from mutagen.flac import FLAC + + config_manager = _get_config_manager() + if not config_manager.get("lossy_copy.enabled", False): + return None + + if os.path.splitext(final_path)[1].lower() != ".flac": + return None + + codec = config_manager.get("lossy_copy.codec", "mp3").lower() + bitrate = config_manager.get("lossy_copy.bitrate", "320") + + if codec == "opus" and int(bitrate) > 256: + bitrate = "256" + + codec_map = { + "mp3": ("libmp3lame", ".mp3", f"MP3-{bitrate}", ["-vn", "-id3v2_version", "3"]), + "opus": ("libopus", ".opus", f"OPUS-{bitrate}", ["-vn", "-map", "0:a", "-vbr", "on"]), + "aac": ("aac", ".m4a", f"AAC-{bitrate}", ["-vn", "-movflags", "+faststart"]), + } + + if codec not in codec_map: + logger.info(f"[Lossy Copy] Unknown codec '{codec}' - skipping conversion") + return None + + ffmpeg_codec, out_ext, quality_label, extra_args = codec_map[codec] + out_path = os.path.splitext(final_path)[0] + out_ext + + original_quality = get_audio_quality_string(final_path) + if original_quality: + out_basename = os.path.basename(out_path) + if original_quality in out_basename: + out_basename = out_basename.replace(original_quality, quality_label) + out_path = os.path.join(os.path.dirname(out_path), out_basename) + + ffmpeg_bin = shutil.which("ffmpeg") + if not ffmpeg_bin: + local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg") + if os.path.isfile(local): + ffmpeg_bin = local + else: + logger.warning(f"[Lossy Copy] ffmpeg not found - skipping {codec.upper()} conversion") + return None + + try: + logger.info(f"[Lossy Copy] Converting to {quality_label}: {os.path.basename(final_path)}") + cmd = [ + ffmpeg_bin, "-i", final_path, + "-codec:a", ffmpeg_codec, + "-b:a", f"{bitrate}k", + "-map_metadata", "0", + ] + extra_args + ["-y", out_path] + + result = subprocess.run(cmd, capture_output=True, text=True, timeout=120) + + if result.returncode == 0: + logger.info(f"[Lossy Copy] Created {quality_label} copy: {os.path.basename(out_path)}") + try: + from mutagen import File as MutagenFile + audio = MutagenFile(out_path) + if audio is not None: + if codec == "mp3": + from mutagen.id3 import TXXX + audio.tags.add(TXXX(encoding=3, desc="QUALITY", text=[quality_label])) + elif codec == "opus": + audio["QUALITY"] = [quality_label] + elif codec == "aac": + from mutagen.mp4 import MP4FreeForm + audio["----:com.apple.iTunes:QUALITY"] = [MP4FreeForm(quality_label.encode("utf-8"))] + audio.save() + except Exception as tag_err: + logger.error(f"[Lossy Copy] Could not update QUALITY tag: {tag_err}") + return out_path + + logger.error(f"[Lossy Copy] ffmpeg failed: {result.stderr[:200]}") + if os.path.exists(out_path): + try: + os.remove(out_path) + except Exception: + pass + return None + except subprocess.TimeoutExpired: + logger.warning(f"[Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}") + except Exception as e: + logger.error(f"[Lossy Copy] Conversion error: {e}") + return None diff --git a/core/import_filename.py b/core/import_filename.py new file mode 100644 index 00000000..9298750d --- /dev/null +++ b/core/import_filename.py @@ -0,0 +1,73 @@ +"""Filename parsing helpers used by import flows.""" + +from __future__ import annotations + +import os +import re +from typing import Any, Dict + + +_TRACK_PATTERNS = ( + r"^(\d+)\s*[-\.]\s*(.+?)\s*[-–]\s*(.+)$", + r"^(.+?)\s*[-–]\s*(.+)$", + r"^(\d+)\s*[-\.]\s*(.+)$", +) + + +def parse_filename_metadata(filename: str) -> Dict[str, Any]: + """Extract artist/title/album hints from a loose filename.""" + raw_path = str(filename or "") + normalized_path = raw_path.replace("\\", "/") + base_name = os.path.splitext(os.path.basename(normalized_path))[0] + + result: Dict[str, Any] = { + "artist": "", + "title": "", + "album": "", + "track_number": None, + } + + if not base_name: + return result + + for pattern in _TRACK_PATTERNS: + match = re.match(pattern, base_name) + if not match: + continue + + groups = match.groups() + if len(groups) == 3: + try: + result["track_number"] = int(groups[0]) + result["artist"] = result["artist"] or groups[1].strip() + result["title"] = result["title"] or groups[2].strip() + except ValueError: + result["artist"] = result["artist"] or groups[0].strip() + result["title"] = result["title"] or f"{groups[1]} - {groups[2]}".strip() + elif len(groups) == 2: + if groups[0].isdigit(): + try: + result["track_number"] = int(groups[0]) + result["title"] = result["title"] or groups[1].strip() + except ValueError: + pass + else: + result["artist"] = result["artist"] or groups[0].strip() + result["title"] = result["title"] or groups[1].strip() + break + + if not result["title"]: + result["title"] = base_name + + if not result["album"] and "/" in normalized_path: + path_parts = normalized_path.split("/") + for part in reversed(path_parts[:-1]): + if not part or part.startswith("@"): + continue + + cleaned = re.sub(r"^\d+\s*[-\.]\s*", "", part).strip() + if len(cleaned) > 3: + result["album"] = cleaned + break + + return result diff --git a/core/import_guards.py b/core/import_guards.py new file mode 100644 index 00000000..de2a3185 --- /dev/null +++ b/core/import_guards.py @@ -0,0 +1,121 @@ +"""Import post-processing guards and quarantine helpers.""" + +from __future__ import annotations + +import json +import os +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Optional + +from core.import_context import ( + get_import_clean_artist, + get_import_clean_title, + get_import_context_artist, + get_import_original_search, + get_import_track_info, + normalize_import_context, +) +from core.import_file_ops import safe_move_file +from database.music_database import MusicDatabase +from utils.logging_config import get_logger + + +logger = get_logger("import_guards") + + +def _get_config_manager(): + from config.settings import config_manager + + return config_manager + + +def move_to_quarantine(file_path: str, context: dict, reason: str, automation_engine=None) -> str: + """Move a file to the quarantine folder and write a metadata sidecar.""" + config_manager = _get_config_manager() + download_dir = config_manager.get("soulseek.download_path", "./downloads") + quarantine_dir = Path(download_dir) / "ss_quarantine" + quarantine_dir.mkdir(parents=True, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + original_name = Path(file_path).stem + file_ext = Path(file_path).suffix + + quarantine_filename = f"{timestamp}_{original_name}{file_ext}.quarantined" + quarantine_path = quarantine_dir / quarantine_filename + + safe_move_file(file_path, str(quarantine_path)) + + metadata_path = quarantine_dir / f"{timestamp}_{original_name}.json" + context = normalize_import_context(context) + original_search = get_import_original_search(context) + artist_context = get_import_context_artist(context) + + metadata = { + "original_filename": Path(file_path).name, + "quarantine_reason": reason, + "timestamp": datetime.now().isoformat(), + "expected_track": get_import_clean_title(context, default=original_search.get("title", "Unknown")), + "expected_artist": get_import_clean_artist(context, default=(artist_context.get("name", "") if isinstance(artist_context, dict) else "Unknown")), + "context_key": context.get("context_key", "unknown"), + } + + try: + with open(metadata_path, "w", encoding="utf-8") as f: + json.dump(metadata, f, indent=2, ensure_ascii=False) + except Exception as exc: + logger.warning("Failed to write quarantine metadata: %s", exc) + + logger.warning("File quarantined: %s - Reason: %s", quarantine_path, reason) + + if automation_engine: + try: + ti = context.get("track_info", {}) + artists = ti.get("artists", []) + artist_name = "" + if artists: + first = artists[0] + artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first) + automation_engine.emit( + "download_quarantined", + { + "artist": artist_name, + "title": ti.get("name", ""), + "reason": reason or "Unknown", + }, + ) + except Exception: + pass + + return str(quarantine_path) + + +def check_flac_bit_depth(file_path: str, context: dict) -> Optional[str]: + """Return a rejection message if a FLAC file violates the configured bit depth.""" + if not context.get("_audio_quality", "").startswith("FLAC"): + return None + + config_manager = _get_config_manager() + quality_profile = MusicDatabase().get_quality_profile() + flac_config = quality_profile.get("qualities", {}).get("flac", {}) + flac_pref = flac_config.get("bit_depth", "any") + if flac_pref == "any": + return None + + actual_bits = context["_audio_quality"].replace("FLAC ", "").replace("bit", "") + if actual_bits == flac_pref: + return None + + flac_fallback = flac_config.get("bit_depth_fallback", True) + downsample_enabled = config_manager.get("lossy_copy.downsample_hires", False) + track_info = context.get("track_info", {}) + track_name = track_info.get("name", os.path.basename(file_path)) + + if flac_fallback or downsample_enabled: + if downsample_enabled: + logger.info("[FLAC Downsample] Accepted %s-bit FLAC (will be downsampled to %s-bit): %s", actual_bits, flac_pref, track_name) + else: + logger.warning("[FLAC Fallback] Accepted %s-bit FLAC (preferred %s-bit): %s", actual_bits, flac_pref, track_name) + return None + + return f"FLAC bit depth mismatch: file is {actual_bits}-bit, preference is {flac_pref}-bit" diff --git a/core/import_paths.py b/core/import_paths.py new file mode 100644 index 00000000..7848425c --- /dev/null +++ b/core/import_paths.py @@ -0,0 +1,757 @@ +"""Shared path and naming helpers for import processing.""" + +from __future__ import annotations + +import json +import logging +import os +import re +import threading +from pathlib import Path +from typing import Any + +from core.import_context import ( + get_import_clean_title, + get_import_context_album, + get_import_original_search, + get_import_source, + get_import_track_info, + normalize_import_context, +) + +logger = logging.getLogger("import_paths") + +_album_cache_lock = threading.Lock() +_album_editions: dict[str, str] = {} +_album_name_cache: dict[str, str] = {} + + +def _get_config_manager(): + try: + from config.settings import config_manager + return config_manager + except Exception: + class _FallbackConfig: + @staticmethod + def get(key, default=None): + return default + + return _FallbackConfig() + + +def _get_itunes_client(): + try: + from core.metadata_service import get_itunes_client + return get_itunes_client() + except Exception: + return None + + +def _get_album_tracks_for_source(source: str, album_id: str): + try: + from core.metadata_service import get_album_tracks_for_source + return get_album_tracks_for_source(source, album_id) + except Exception: + return None + + +def _extract_artist_name(artist_context: Any) -> str: + if not artist_context: + return "" + if isinstance(artist_context, dict): + return str(artist_context.get("name", "") or "").strip() + return str(artist_context).strip() + + +def docker_resolve_path(path_str: str) -> str: + """Resolve Docker-hosted Windows paths into container paths.""" + if os.path.exists("/.dockerenv") and len(path_str) >= 3 and path_str[1] == ":" and path_str[0].isalpha(): + drive_letter = path_str[0].lower() + rest_of_path = path_str[2:].replace("\\", "/") + return f"/host/mnt/{drive_letter}{rest_of_path}" + return path_str + + +def build_simple_download_destination(context, file_path: str): + """Build the destination path for a simple download into Transfer.""" + context = normalize_import_context(context) + search_result = context.get("search_result", {}) or {} + if not isinstance(search_result, dict): + search_result = {} + + transfer_dir = Path(docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer"))) + album_name = None + original_filename = search_result.get("filename", "") + if "/" in original_filename or "\\" in original_filename: + path_parts = original_filename.replace("\\", "/").split("/") + if len(path_parts) >= 2: + album_name = path_parts[-2] + if not album_name: + album_value = search_result.get("album") + if isinstance(album_value, dict): + album_name = album_value.get("name", "") + else: + album_name = album_value + + filename = Path(file_path).name + if album_name and str(album_name).lower() not in {"unknown", "unknown album", ""}: + album_name = sanitize_filename(str(album_name)) + destination_dir = transfer_dir / album_name + else: + album_name = "" + destination_dir = transfer_dir + + destination_dir.mkdir(parents=True, exist_ok=True) + return destination_dir / filename, album_name, filename + + +def sanitize_filename(filename: str) -> str: + """Sanitize filename for file system compatibility.""" + sanitized = re.sub(r'[<>:"/\\|?*]', "_", filename) + sanitized = re.sub(r"\s+", " ", sanitized).strip() + sanitized = sanitized.rstrip(". ") or "_" + if re.match(r"^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)", sanitized, re.IGNORECASE): + sanitized = "_" + sanitized + return sanitized[:200] + + +def sanitize_context_values(context: dict) -> dict: + """Sanitize all string values in a template context for path safety.""" + sanitized = {} + for key, value in context.items(): + if isinstance(value, str) and value: + sanitized[key] = sanitize_filename(value) + else: + sanitized[key] = value + return sanitized + + +def clean_track_title(track_title: str, artist_name: str) -> str: + """Clean up track title by removing artist prefix and other noise.""" + original = (track_title or "").strip() + cleaned = original + cleaned = re.sub(r"^\d{1,2}[\.\s\-]+", "", cleaned) + artist_pattern = re.escape(artist_name or "") + r"\s*-\s*" + cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"^[A-Za-z0-9\.]+\s*-\s*\d{1,2}\s*-\s*", "", cleaned) + quality_patterns = [ + r"\s*[\[\(][0-9]+\s*kbps[\]\)]\s*", + r"\s*[\[\(]flac[\]\)]\s*", + r"\s*[\[\(]mp3[\]\)]\s*", + ] + for pattern in quality_patterns: + cleaned = re.sub(pattern, "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"^[-\s\.]+", "", cleaned) + cleaned = re.sub(r"[-\s\.]+$", "", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned).strip() + return cleaned if cleaned else original + + +def get_base_album_name(album_name: str) -> str: + """Extract the base album name without edition indicators.""" + base_name = album_name or "" + base_name = re.sub( + r"\s*[\[\(][^)\]]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^)\]]*[\]\)]\s*$", + "", + base_name, + flags=re.IGNORECASE, + ) + base_name = re.sub(r"\s*[\[\(][^)\]]*\bedition\b[^)\]]*[\]\)]\s*$", "", base_name, flags=re.IGNORECASE) + base_name = re.sub( + r"\s+(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", + "", + base_name, + flags=re.IGNORECASE, + ) + return base_name.strip() + + +def detect_deluxe_edition(album_name: str) -> bool: + """Detect if an album name indicates a deluxe/special edition.""" + if not album_name: + return False + + album_lower = album_name.lower() + deluxe_indicators = [ + "deluxe", + "deluxe edition", + "special edition", + "expanded edition", + "extended edition", + "bonus", + "remastered", + "anniversary", + "collectors edition", + "limited edition", + "silver edition", + "gold edition", + "platinum edition", + ] + for indicator in deluxe_indicators: + if indicator in album_lower: + logger.info("Detected deluxe edition: %r contains %r", album_name, indicator) + return True + return False + + +def normalize_base_album_name(base_album: str, artist_name: str) -> str: + """Normalize the base album name to handle case variations and known corrections.""" + normalized_lower = (base_album or "").lower().strip() + known_corrections = { + # Add specific album name corrections here as needed. + } + + for variant, correction in known_corrections.items(): + if normalized_lower == variant.lower(): + logger.info("Album correction applied: %r -> %r", base_album, correction) + return correction + + normalized = base_album or "" + normalized = re.sub(r"\s*&\s*", " & ", normalized) + normalized = re.sub(r"\s+", " ", normalized) + normalized = normalized.strip() + logger.info("Album variant normalization: %r -> %r", base_album, normalized) + return normalized + + +def clean_album_title(album_title: str, artist_name: str) -> str: + """Clean up album title by removing common prefixes, suffixes, and artist redundancy.""" + original = (album_title or "").strip() + cleaned = original + logger.info("Album Title Cleaning: %r (artist: %r)", original, artist_name) + + cleaned = re.sub(r"^Album\s*-\s*", "", cleaned, flags=re.IGNORECASE) + artist_pattern = re.escape(artist_name or "") + r"\s*-\s*" + cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*[\[\(]\d{4}[\]\)]\s*", " ", cleaned) + + quality_patterns = [ + r"\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*", + r"\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*", + r"\s*[\[\(].*?flac.*?[\]\)]\s*", + r"\s*[\[\(].*?mp3.*?[\]\)]\s*", + r"\s*[\[\(].*?itunes.*?[\]\)]\s*", + r"\s*[\[\(].*?web.*?[\]\)]\s*", + r"\s*[\[\(].*?cd.*?[\]\)]\s*", + ] + for pattern in quality_patterns: + cleaned = re.sub(pattern, " ", cleaned, flags=re.IGNORECASE) + + cleaned = re.sub(r"\s*[\[\(][^\]\)]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*[\[\(][^\]\)]*\bedition\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"\s*(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", "", cleaned, flags=re.IGNORECASE) + cleaned = re.sub(r"^[-\s\.]+", "", cleaned) + cleaned = re.sub(r"[-\s\.]+$", "", cleaned) + cleaned = re.sub(r"\s+", " ", cleaned).strip() + return cleaned if cleaned else original + + +def resolve_album_group(artist_context: dict, album_info: dict, original_album: str = None) -> str: + """Smart album grouping: upgrade to deluxe if any track is deluxe.""" + try: + with _album_cache_lock: + artist_name = _extract_artist_name(artist_context) + detected_album = (album_info or {}).get("album_name", "") + + if detected_album: + base_album = get_base_album_name(detected_album) + elif original_album: + cleaned_original = clean_album_title(original_album, artist_name) + base_album = get_base_album_name(cleaned_original) + else: + base_album = get_base_album_name(detected_album) + + base_album = normalize_base_album_name(base_album, artist_name) + album_key = f"{artist_name}::{base_album}" + is_deluxe_track = False + if detected_album: + is_deluxe_track = detect_deluxe_edition(detected_album) + elif original_album: + is_deluxe_track = detect_deluxe_edition(original_album) + + if album_key in _album_name_cache: + cached_name = _album_name_cache[album_key] + current_edition = _album_editions.get(album_key, "standard") + if is_deluxe_track and current_edition == "standard": + final_album_name = f"{base_album} (Deluxe Edition)" + _album_editions[album_key] = "deluxe" + _album_name_cache[album_key] = final_album_name + logger.info("Album cache upgrade: %r -> %r", album_key, final_album_name) + return final_album_name + logger.info("Using cached album name for %r: %r", album_key, cached_name) + return cached_name + + logger.info("Album grouping - Key: %r, Detected: %r", album_key, detected_album) + + current_edition = _album_editions.get(album_key, "standard") + if is_deluxe_track and current_edition == "standard": + logger.info("UPGRADE: Album %r upgraded from standard to deluxe!", base_album) + _album_editions[album_key] = "deluxe" + current_edition = "deluxe" + + if current_edition == "deluxe": + final_album_name = f"{base_album} (Deluxe Edition)" + else: + final_album_name = base_album + + _album_name_cache[album_key] = final_album_name + + logger.info("Album resolution: %r -> %r (edition: %s)", detected_album, final_album_name, current_edition) + return final_album_name + except Exception as e: + logger.error("Error resolving album group: %s", e) + album_name = (album_info or {}).get("album_name", "Unknown Album") + return album_name + + +def get_album_type_display(raw_type, track_count) -> str: + """Return the display form of an album's type for the $albumtype template variable.""" + raw = (raw_type or "").strip().lower() + try: + tc = int(track_count or 0) + except (TypeError, ValueError): + tc = 0 + + if raw in ("compilation", "compile"): + return "Compilation" + if raw == "album": + return "Album" + if raw in ("single", "ep"): + if tc <= 3: + return "Single" + if tc <= 6: + return "EP" + return "Album" + + if tc <= 0: + return "Album" + if tc <= 3: + return "Single" + if tc <= 6: + return "EP" + return "Album" + + +def _replace_template_variables(template: str, context: dict) -> str: + clean_context = sanitize_context_values(context) + result = template + + album_artist_value = clean_context.get("albumartist", clean_context.get("artist", "Unknown Artist")) + collab_mode = _get_config_manager().get("file_organization.collab_artist_mode", "first") + if collab_mode == "first" and album_artist_value: + artists_list = context.get("_artists_list") + if artists_list and len(artists_list) > 1: + first = artists_list[0] + album_artist_value = first.get("name", first) if isinstance(first, dict) else str(first) + elif artists_list and len(artists_list) == 1: + itunes_artist_id = context.get("_itunes_artist_id") + if itunes_artist_id and ("," in album_artist_value or " & " in album_artist_value): + try: + resolved_client = _get_itunes_client() + if resolved_client and hasattr(resolved_client, "resolve_primary_artist"): + resolved = resolved_client.resolve_primary_artist(itunes_artist_id) + if resolved and resolved != album_artist_value: + album_artist_value = resolved + except Exception: + pass + + bracket_map = { + "albumartist": album_artist_value, + "albumtype": clean_context.get("albumtype", "Album"), + "playlist": clean_context.get("playlist_name", ""), + "artistletter": (clean_context.get("artist", "U") or "U")[0].upper(), + "artist": clean_context.get("artist", "Unknown Artist"), + "album": clean_context.get("album", "Unknown Album"), + "title": clean_context.get("title", "Unknown Track"), + "track": f"{_coerce_int(clean_context.get('track_number', 1), 1):02d}", + "disc": str(_coerce_int(clean_context.get("disc_number", 1), 1)), + "discnum": str(_coerce_int(clean_context.get("disc_number", 1), 1)), + "year": str(clean_context.get("year", "")), + "quality": clean_context.get("quality", ""), + } + for var_name, val in bracket_map.items(): + result = result.replace("${" + var_name + "}", val) + + result = result.replace("$albumartist", album_artist_value) + result = result.replace("$albumtype", clean_context.get("albumtype", "Album")) + result = result.replace("$playlist", clean_context.get("playlist_name", "")) + result = result.replace("$artistletter", (clean_context.get("artist", "U") or "U")[0].upper()) + result = result.replace("$artist", clean_context.get("artist", "Unknown Artist")) + result = result.replace("$album", clean_context.get("album", "Unknown Album")) + result = result.replace("$title", clean_context.get("title", "Unknown Track")) + result = result.replace("$track", f"{clean_context.get('track_number', 1):02d}") + result = result.replace("$year", str(clean_context.get("year", ""))) + + result = re.sub(r"\s+", " ", result) + result = re.sub(r"\s*-\s*-\s*", " - ", result) + result = result.strip() + return result + + +def apply_path_template(template: str, context: dict) -> str: + """Apply a template to build a path string.""" + return _replace_template_variables(template, context) + + +def get_file_path_from_template_raw(template: str, context: dict) -> tuple[str, str]: + """Build file path using a user-provided template string directly.""" + full_path = apply_path_template(template, context) + + quality_value = context.get("quality", "") + disc_number = _coerce_int(context.get("disc_number", 1), 1) + disc_value = f"{disc_number:02d}" + disc_value_raw = str(disc_number) + + path_parts = full_path.split("/") + if len(path_parts) > 1: + folder_parts = path_parts[:-1] + filename_base = path_parts[-1] + + cleaned_folders = [] + for part in folder_parts: + part = part.replace("$quality", "") + part = part.replace("$discnum", "") + part = part.replace("$disc", "") + part = re.sub(r"\s*\[\s*\]", "", part) + part = re.sub(r"\s*\(\s*\)", "", part) + part = re.sub(r"\s*\{\s*\}", "", part) + part = re.sub(r"\s*-\s*$", "", part) + part = re.sub(r"^\s*-\s*", "", part) + part = re.sub(r"\s+", " ", part).strip() + if part: + cleaned_folders.append(part) + + filename_base = filename_base.replace("$quality", quality_value) + filename_base = filename_base.replace("$discnum", disc_value_raw) + filename_base = filename_base.replace("$disc", disc_value) + filename_base = re.sub(r"\s*\[\s*\]", "", filename_base) + filename_base = re.sub(r"\s*\(\s*\)", "", filename_base) + filename_base = re.sub(r"\s*\{\s*\}", "", filename_base) + filename_base = re.sub(r"\s*-\s*$", "", filename_base) + filename_base = re.sub(r"\s+", " ", filename_base).strip() + + sanitized_folders = [sanitize_filename(part) for part in cleaned_folders] + folder_path = os.path.join(*sanitized_folders) if sanitized_folders else "" + return folder_path, sanitize_filename(filename_base) + + full_path = full_path.replace("$quality", quality_value) + full_path = full_path.replace("$discnum", disc_value_raw) + full_path = full_path.replace("$disc", disc_value) + full_path = re.sub(r"\s*\[\s*\]", "", full_path) + full_path = re.sub(r"\s*\(\s*\)", "", full_path) + full_path = re.sub(r"\s*\{\s*\}", "", full_path) + full_path = re.sub(r"\s*-\s*$", "", full_path) + full_path = re.sub(r"\s+", " ", full_path).strip() + return "", sanitize_filename(full_path) + + +def get_file_path_from_template(context: dict, template_type: str = "album_path") -> tuple[str, str]: + """Build complete file path using configured templates.""" + if not _get_config_manager().get("file_organization.enabled", True): + return None, None + + templates = _get_config_manager().get("file_organization.templates", {}) + template = templates.get(template_type) + if not template: + default_templates = { + "album_path": "$albumartist/$albumartist - $album/$track - $title", + "single_path": "$artist/$artist - $title/$title", + "compilation_path": "Compilations/$album/$track - $artist - $title", + "playlist_path": "$playlist/$artist - $title", + } + template = default_templates.get(template_type, "$artist/$album/$track - $title") + + full_path = apply_path_template(template, context) + + path_parts = full_path.split("/") + quality_value = context.get("quality", "") + disc_number = _coerce_int(context.get("disc_number", 1), 1) + disc_value = f"{disc_number:02d}" + disc_value_raw = str(disc_number) + + if len(path_parts) > 1: + folder_parts = path_parts[:-1] + filename_base = path_parts[-1] + + cleaned_folders = [] + for part in folder_parts: + part = part.replace("$quality", "") + part = part.replace("$discnum", "") + part = part.replace("$disc", "") + part = re.sub(r"\s*\[\s*\]", "", part) + part = re.sub(r"\s*\(\s*\)", "", part) + part = re.sub(r"\s*\{\s*\}", "", part) + part = re.sub(r"\s*-\s*$", "", part) + part = re.sub(r"^\s*-\s*", "", part) + part = re.sub(r"\s+", " ", part).strip() + if part: + cleaned_folders.append(part) + + filename_base = filename_base.replace("$quality", quality_value) + filename_base = filename_base.replace("$discnum", disc_value_raw) + filename_base = filename_base.replace("$disc", disc_value) + filename_base = re.sub(r"\s*\[\s*\]", "", filename_base) + filename_base = re.sub(r"\s*\(\s*\)", "", filename_base) + filename_base = re.sub(r"\s*\{\s*\}", "", filename_base) + filename_base = re.sub(r"\s*-\s*$", "", filename_base) + filename_base = re.sub(r"\s+", " ", filename_base).strip() + + sanitized_folders = [sanitize_filename(part) for part in cleaned_folders] + folder_path = os.path.join(*sanitized_folders) if sanitized_folders else "" + filename = sanitize_filename(filename_base) + return folder_path, filename + + full_path = full_path.replace("$quality", quality_value) + full_path = full_path.replace("$discnum", disc_value_raw) + full_path = full_path.replace("$disc", disc_value) + full_path = re.sub(r"\s*\[\s*\]", "", full_path) + full_path = re.sub(r"\s*\(\s*\)", "", full_path) + full_path = re.sub(r"\s*\{\s*\}", "", full_path) + full_path = re.sub(r"\s*-\s*$", "", full_path) + full_path = re.sub(r"\s+", " ", full_path).strip() + return "", sanitize_filename(full_path) + + +def _max_disc_number(album_tracks: Any) -> int: + items = [] + if isinstance(album_tracks, dict): + items = album_tracks.get("items") or album_tracks.get("tracks") or [] + elif isinstance(album_tracks, list): + items = album_tracks + + max_disc = 1 + for track in items: + if not isinstance(track, dict): + continue + try: + disc_number = int(track.get("disc_number", 1) or 1) + except (TypeError, ValueError): + disc_number = 1 + if disc_number > max_disc: + max_disc = disc_number + return max_disc + + +def _coerce_int(value: Any, default: int = 1) -> int: + try: + coerced = int(value) + except (TypeError, ValueError): + return default + return coerced if coerced > 0 else default + + +def build_final_path_for_track(context, artist_context, album_info, file_ext): + """Shared path builder used by both post-processing and verification.""" + transfer_dir = docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer")) + context = normalize_import_context(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + album_context = get_import_context_album(context) + source = get_import_source(context) + playlist_folder_mode = track_info.get("_playlist_folder_mode", False) + artist_name = _extract_artist_name(artist_context) + + source_info = track_info.get("source_info") or {} + if isinstance(source_info, str): + try: + source_info = json.loads(source_info) + except (json.JSONDecodeError, TypeError): + source_info = {} + if source_info.get("enhance") and source_info.get("original_file_path"): + original_path = source_info["original_file_path"] + original_dir = os.path.dirname(original_path) + original_stem = os.path.splitext(os.path.basename(original_path))[0] + final_path = os.path.join(original_dir, original_stem + file_ext) + os.makedirs(original_dir, exist_ok=True) + logger.info("[Enhance] Using original file location: %s", final_path) + return final_path, True + + year = "" + if album_context and album_context.get("release_date"): + release_date = album_context["release_date"] + if release_date and len(release_date) >= 4: + year = release_date[:4] + + raw_album_type = "" + if album_context: + raw_album_type = album_context.get("album_type", "") or "" + total_tracks = (album_context.get("total_tracks", 0) or 0) if album_context else 0 + album_type_display = get_album_type_display(raw_album_type, total_tracks) + + if playlist_folder_mode: + playlist_name = track_info.get("_playlist_name", "Unknown Playlist") + track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track")) + _artists = original_search.get("artists") or track_info.get("artists") or [] + + template_context = { + "artist": artist_name, + "albumartist": artist_name, + "album": track_name, + "title": track_name, + "playlist_name": playlist_name, + "track_number": 1, + "disc_number": 1, + "year": year, + "quality": context.get("_audio_quality", ""), + "albumtype": album_type_display, + "_artists_list": _artists, + "_itunes_artist_id": str(artist_context.get("id", "")) if isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source == "itunes" else None, + } + + folder_path, filename_base = get_file_path_from_template(template_context, "playlist_path") + if folder_path and filename_base: + final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + return final_path, True + + playlist_name_sanitized = sanitize_filename(playlist_name) + playlist_dir = os.path.join(transfer_dir, playlist_name_sanitized) + os.makedirs(playlist_dir, exist_ok=True) + artist_name_sanitized = sanitize_filename(template_context["artist"]) + track_name_sanitized = sanitize_filename(track_name) + new_filename = f"{artist_name_sanitized} - {track_name_sanitized}{file_ext}" + return os.path.join(playlist_dir, new_filename), True + + if album_info and album_info.get("is_album"): + clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track")) + track_number = _coerce_int(album_info.get("track_number", 1), 1) + disc_number = _coerce_int(album_info.get("disc_number", 1), 1) + _artists = original_search.get("artists") or track_info.get("artists") or [] + _album_ctx = album_context + _itunes_aid = None + _is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer") + if _is_itunes and isinstance(artist_context, dict): + _aid = artist_context.get("id", "") + if str(_aid).isdigit(): + _itunes_aid = str(_aid) + if not _itunes_aid and _album_ctx: + _ext = _album_ctx.get("external_urls", {}) + if isinstance(_ext, dict) and _ext.get("itunes_artist_id"): + _itunes_aid = _ext["itunes_artist_id"] + + _artist_name = artist_name + _album_artist_name = _artist_name + _album_artists_for_collab = None + _explicit_artist_ctx = track_info.get("_explicit_artist_context") if isinstance(track_info, dict) else None + if isinstance(_explicit_artist_ctx, dict) and _explicit_artist_ctx.get("name"): + _album_artist_name = _explicit_artist_ctx["name"] + _album_artists_for_collab = [_explicit_artist_ctx] + elif isinstance(_explicit_artist_ctx, str) and _explicit_artist_ctx: + _album_artist_name = _explicit_artist_ctx + _album_artists_for_collab = [{"name": _explicit_artist_ctx}] + else: + _sa_artists = _album_ctx.get("artists", []) if _album_ctx else [] + if _sa_artists: + _first_sa = _sa_artists[0] + if isinstance(_first_sa, dict) and _first_sa.get("name"): + _album_artist_name = _first_sa["name"] + elif isinstance(_first_sa, str) and _first_sa: + _album_artist_name = _first_sa + _album_artists_for_collab = _sa_artists + + template_context = { + "artist": _artist_name, + "albumartist": _album_artist_name, + "album": album_info["album_name"], + "title": clean_track_name, + "track_number": track_number, + "disc_number": disc_number, + "year": year, + "quality": context.get("_audio_quality", ""), + "albumtype": album_type_display, + "_artists_list": _album_artists_for_collab if _album_artists_for_collab else _artists, + "_itunes_artist_id": _itunes_aid, + } + total_discs = _coerce_int(album_context.get("total_discs", 1) if album_context else 1, 1) + + if total_discs <= 1 and album_context and album_context.get("id"): + if disc_number > 1: + total_discs = disc_number + else: + try: + _album_tracks = _get_album_tracks_for_source(source, str(album_context["id"])) + if _album_tracks: + total_discs = _max_disc_number(_album_tracks) + if total_discs > 1: + album_context["total_discs"] = total_discs + logger.info( + "[Multi-Disc] Resolved %s discs for single-track download of %r", + total_discs, + album_context.get("name"), + ) + except Exception as _disc_err: + logger.warning("[Multi-Disc] Could not resolve total_discs: %s", _disc_err) + + album_template = _get_config_manager().get("file_organization.templates.album_path", "") + user_controls_disc = "$disc" in album_template + disc_label = _get_config_manager().get("file_organization.disc_label", "Disc") + + folder_path, filename_base = get_file_path_from_template(template_context, "album_path") + if folder_path and filename_base: + if total_discs > 1 and not user_controls_disc: + disc_folder = f"{disc_label} {disc_number}" + final_path = os.path.join(transfer_dir, folder_path, disc_folder, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path, disc_folder), exist_ok=True) + else: + final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + return final_path, True + + artist_name_sanitized = sanitize_filename(template_context["albumartist"]) + album_name_sanitized = sanitize_filename(album_info["album_name"]) + artist_dir = os.path.join(transfer_dir, artist_name_sanitized) + album_folder_name = f"{artist_name_sanitized} - {album_name_sanitized}" + album_dir = os.path.join(artist_dir, album_folder_name) + if total_discs > 1: + album_dir = os.path.join(album_dir, f"{disc_label} {disc_number}") + os.makedirs(album_dir, exist_ok=True) + final_track_name_sanitized = sanitize_filename(clean_track_name) + new_filename = f"{track_number:02d} - {final_track_name_sanitized}{file_ext}" + return os.path.join(album_dir, new_filename), True + + clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track")) + _artists = original_search.get("artists") or track_info.get("artists") or [] + _album_ctx = album_context + _itunes_aid = None + _is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer") + if _is_itunes and isinstance(artist_context, dict): + _aid = artist_context.get("id", "") + if str(_aid).isdigit(): + _itunes_aid = str(_aid) + if not _itunes_aid and _album_ctx: + _ext = _album_ctx.get("external_urls", {}) + if isinstance(_ext, dict) and _ext.get("itunes_artist_id"): + _itunes_aid = _ext["itunes_artist_id"] + + template_context = { + "artist": artist_name, + "albumartist": artist_name, + "album": album_info.get("album_name", clean_track_name) if album_info else clean_track_name, + "title": clean_track_name, + "track_number": 1, + "disc_number": 1, + "year": year, + "quality": context.get("_audio_quality", ""), + "albumtype": album_type_display, + "_artists_list": _artists, + "_itunes_artist_id": _itunes_aid, + } + + folder_path, filename_base = get_file_path_from_template(template_context, "single_path") + if filename_base: + if folder_path: + final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext) + os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True) + else: + final_path = os.path.join(transfer_dir, filename_base + file_ext) + os.makedirs(transfer_dir, exist_ok=True) + return final_path, True + + artist_name_sanitized = sanitize_filename(template_context["artist"]) + final_track_name_sanitized = sanitize_filename(clean_track_name) + artist_dir = os.path.join(transfer_dir, artist_name_sanitized) + single_folder_name = f"{artist_name_sanitized} - {final_track_name_sanitized}" + single_dir = os.path.join(artist_dir, single_folder_name) + os.makedirs(single_dir, exist_ok=True) + new_filename = f"{final_track_name_sanitized}{file_ext}" + return os.path.join(single_dir, new_filename), True diff --git a/core/import_pipeline.py b/core/import_pipeline.py new file mode 100644 index 00000000..ede3ae18 --- /dev/null +++ b/core/import_pipeline.py @@ -0,0 +1,925 @@ +"""Import/post-processing pipeline for downloads and imported files.""" + +from __future__ import annotations + +import json +import os +import threading +import time + +from config.settings import config_manager +from core.import_file_ops import ( + cleanup_empty_directories, + create_lossy_copy, + downsample_hires_flac, + extract_track_number_from_filename, + get_audio_quality_string, + get_quality_tier_from_extension, + safe_move_file, +) +from core.import_context import ( + build_import_album_info, + extract_artist_name, + get_import_clean_artist, + get_import_clean_title, + get_import_context_artist, + get_import_has_clean_metadata, + get_import_original_search, + get_import_source, + get_import_track_info, + normalize_import_context, +) +from core.import_guards import check_flac_bit_depth, move_to_quarantine +from core.import_side_effects import ( + check_and_remove_from_wishlist, + emit_track_downloaded, + record_download_provenance, + record_library_history_download, + record_retag_download, + record_soulsync_library_entry, +) +from core.import_runtime_state import ( + add_activity_item, + detect_album_info_web, + download_batches, + download_tasks, + matched_context_lock, + matched_downloads_context, + mark_task_completed as _mark_task_completed, + _post_process_locks, + _post_process_locks_lock, + _processed_download_ids, + tasks_lock, +) +from core.metadata_enrichment import ( + download_cover_art, + enhance_file_metadata, + generate_lrc_file, + wipe_source_tags, +) +from core.import_paths import ( + build_final_path_for_track, + build_simple_download_destination, + docker_resolve_path, + resolve_album_group, +) +from database.music_database import get_database +from utils.logging_config import get_logger + + +logger = get_logger("import_pipeline") +pp_logger = get_logger("post_processing") + + +def post_process_matched_download(context_key, context, file_path, runtime): + on_download_completed = getattr(runtime, "on_download_completed", None) + automation_engine = getattr(runtime, "automation_engine", None) + web_scan_manager = getattr(runtime, "web_scan_manager", None) + repair_worker = getattr(runtime, "repair_worker", None) + + def _notify_download_completed(batch_id, task_id, success=True): + if on_download_completed: + on_download_completed(batch_id, task_id, success=success) + + with _post_process_locks_lock: + if context_key not in _post_process_locks: + _post_process_locks[context_key] = threading.Lock() + file_lock = _post_process_locks[context_key] + + file_lock.acquire() + try: + if not os.path.exists(file_path): + existing_final = context.get('_final_processed_path') + if existing_final and os.path.exists(existing_final): + logger.info( + f"[Race Guard] Source gone but destination exists — already processed by another thread: " + f"{os.path.basename(existing_final)}" + ) + return + logger.error( + f"[Race Guard] Source file gone and no known destination — marking as failed: " + f"{os.path.basename(file_path)}" + ) + context['_race_guard_failed'] = True + return + + _basename = os.path.basename(file_path) + _prev_size = -1 + for _stability_check in range(5): + try: + _cur_size = os.path.getsize(file_path) + except OSError: + _cur_size = -1 + if _cur_size == _prev_size and _cur_size > 0: + break + _prev_size = _cur_size + if _stability_check == 0: + logger.info(f"Waiting for file to stabilise: {_basename} ({_cur_size} bytes)") + time.sleep(1.5) + else: + logger.info(f"File may still be writing after stability checks: {_basename} ({_prev_size} bytes)") + + _skip_acoustid = False + try: + from core.acoustid_verification import AcoustIDVerification, VerificationResult + + verifier = AcoustIDVerification() + available, available_reason = verifier.quick_check_available() + if available and not _skip_acoustid: + context = normalize_import_context(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + artist_context = get_import_context_artist(context) + + expected_track = get_import_clean_title(context, default=original_search.get('title', '')) + expected_artist = '' + track_artists = track_info.get('artists', []) + if track_artists: + first = track_artists[0] + if isinstance(first, dict): + expected_artist = first.get('name', '') + elif isinstance(first, str): + expected_artist = first + if not expected_artist: + expected_artist = extract_artist_name(artist_context) or get_import_clean_artist(context, default='') + + if expected_track and expected_artist: + logger.info(f"Running AcoustID verification for: '{expected_track}' by '{expected_artist}'") + verification_result, verification_msg = verifier.verify_audio_file( + file_path, + expected_track, + expected_artist, + context, + ) + logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}") + context['_acoustid_result'] = verification_result.value + + if verification_result == VerificationResult.FAIL: + try: + quarantine_path = move_to_quarantine( + file_path, + context, + verification_msg, + automation_engine, + ) + logger.error(f"File quarantined due to verification failure: {quarantine_path}") + except Exception as quarantine_error: + logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}") + logger.error(f"Quarantine failed, deleting wrong file: {file_path}") + try: + os.remove(file_path) + except Exception as del_error: + logger.error(f"Could not delete wrong file either: {del_error}") + + context['_acoustid_quarantined'] = True + context['_acoustid_failure_msg'] = verification_msg + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"AcoustID verification failed: {verification_msg}" + ) + + if task_id and batch_id: + _notify_download_completed(batch_id, task_id, success=False) + return + else: + logger.warning("AcoustID verification skipped: missing track/artist info") + context['_acoustid_result'] = 'skip' + else: + logger.info(f"ℹ️ AcoustID verification not available: {available_reason}") + context['_acoustid_result'] = 'disabled' + except Exception as verify_error: + logger.error(f"AcoustID verification error (continuing normally): {verify_error}") + context['_acoustid_result'] = 'error' + + search_result = context.get('search_result', {}) or {} + if not isinstance(search_result, dict): + search_result = {} + is_simple_download = search_result.get('is_simple_download', False) + if is_simple_download: + logger.info(f"Processing simple download (no metadata enhancement): {file_path}") + + destination, album_name, filename = build_simple_download_destination(context, file_path) + if album_name: + logger.info(f"Moving to album folder: {album_name}") + else: + logger.info("Moving to Transfer root (single track)") + + safe_move_file(file_path, destination) + logger.info(f"Moved simple download to: {destination}") + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + if web_scan_manager: + threading.Thread( + target=lambda: web_scan_manager.request_scan("Simple download completed"), + daemon=True, + ).start() + + activity_target = f"{album_name}/{filename}" if album_name else filename + add_activity_item("", "Download Complete", activity_target, "Now") + logger.info(f"Simple download post-processing complete: {activity_target}") + context['_simple_download_completed'] = True + context['_final_path'] = str(destination) + emit_track_downloaded(context, automation_engine) + record_library_history_download(context) + record_download_provenance(context) + return + + logger.info(f"Starting robust post-processing for: {context_key}") + + context = normalize_import_context(context) + artist_context = get_import_context_artist(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + has_clean_metadata = get_import_has_clean_metadata(context) + + if not artist_context: + logger.error("Post-processing failed: Missing artist context.") + return + + _junk_artist_names = {'', 'unknown', 'unknown artist', 'various artists', 'none', 'null'} + _artist_name = (artist_context.get('name', '') if isinstance(artist_context, dict) else '').strip() + if _artist_name.lower() in _junk_artist_names: + logger.info(f"[Unknown Artist Guard] Artist name is '{_artist_name}' — attempting to resolve") + _resolved = False + track_info_guard = track_info or {} + original_search_guard = original_search or {} + + _ti_artists = track_info_guard.get('artists', []) + if isinstance(_ti_artists, list) and _ti_artists: + _first = _ti_artists[0] + _name = _first.get('name', '') if isinstance(_first, dict) else str(_first) + if _name and _name.strip().lower() not in _junk_artist_names: + artist_context['name'] = _name.strip() + logger.info(f"[Unknown Artist Guard] Resolved from track_info.artists: '{_name}'") + _resolved = True + + if not _resolved: + _os_artist = original_search_guard.get('artist') or original_search_guard.get('artist_name') or '' + if isinstance(_os_artist, str) and _os_artist.strip().lower() not in _junk_artist_names: + artist_context['name'] = _os_artist.strip() + logger.info(f"[Unknown Artist Guard] Resolved from original_search_result: '{_os_artist}'") + _resolved = True + + if not _resolved: + _track_id = track_info_guard.get('id') or track_info_guard.get('track_id') or '' + if _track_id: + try: + from core.metadata_service import get_client_for_source, get_primary_source + + _guard_source = get_import_source(context) or get_primary_source() + _fb_client = get_client_for_source(_guard_source) or get_client_for_source(get_primary_source()) + if hasattr(_fb_client, 'get_track_details'): + _details = _fb_client.get_track_details(str(_track_id)) + if _details and isinstance(_details, dict): + _d_artists = _details.get('artists', []) + if isinstance(_d_artists, list) and _d_artists: + _d_first = _d_artists[0] + _d_name = _d_first.get('name', '') if isinstance(_d_first, dict) else str(_d_first) + if _d_name and _d_name.strip().lower() not in _junk_artist_names: + artist_context['name'] = _d_name.strip() + logger.info(f"[Unknown Artist Guard] Resolved from metadata API: '{_d_name}'") + _resolved = True + except Exception as _guard_err: + logger.error(f"[Unknown Artist Guard] Metadata re-fetch failed: {_guard_err}") + + if not _resolved: + logger.error(f"[Unknown Artist Guard] Could not resolve artist — proceeding with '{_artist_name}'") + + context['artist'] = artist_context + + playlist_folder_mode = track_info.get("_playlist_folder_mode", False) + logger.debug(f"[Debug] Post-processing - track_info type: {type(track_info)}, is None: {track_info is None}, is empty: {not track_info}") + logger.debug(f"[Debug] Post-processing - playlist_folder_mode: {playlist_folder_mode}") + if track_info: + logger.debug(f"[Debug] Post-processing - track_info keys: {list(track_info.keys())}") + + if playlist_folder_mode: + playlist_name = track_info.get("_playlist_name", "Unknown Playlist") + logger.info(f"[Playlist Folder Mode] Organizing in playlist folder: {playlist_name}") + + file_ext = os.path.splitext(file_path)[1] + final_path, _ = build_final_path_for_track(context, artist_context, None, file_ext) + logger.info(f"Playlist mode final path: '{final_path}'") + + if not os.path.exists(file_path): + if os.path.exists(final_path): + logger.info( + f"[Playlist Folder Mode] Source gone but destination exists — already processed by another thread: " + f"{os.path.basename(final_path)}" + ) + context['_final_processed_path'] = final_path + return + pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: Source file not found and destination does not exist: {file_path}") + raise FileNotFoundError(f"Source file not found and destination does not exist: {file_path}") + + context['_audio_quality'] = get_audio_quality_string(file_path) + if context['_audio_quality']: + logger.info(f"Audio quality detected: {context['_audio_quality']}") + + rejection_reason = check_flac_bit_depth(file_path, context) + if rejection_reason: + try: + quarantine_path = move_to_quarantine( + file_path, + context, + rejection_reason, + automation_engine, + ) + logger.info(f"File quarantined due to bit depth filter: {quarantine_path}") + except Exception as quarantine_error: + logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") + try: + os.remove(file_path) + except Exception: + pass + + context['_bitdepth_rejected'] = True + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}" + if task_id and batch_id: + _notify_download_completed(batch_id, task_id, success=False) + return + + try: + logger.warning( + f"[Metadata Input] Playlist mode - artist: '{artist_context.get('name', 'MISSING')}' " + f"(id: {artist_context.get('id', 'MISSING')})" + ) + enhance_file_metadata(file_path, context, artist_context, None) + except Exception as meta_err: + import traceback + pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") + wipe_source_tags(file_path) + + logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") + safe_move_file(file_path, final_path) + context['_final_processed_path'] = final_path + + if config_manager.get('post_processing.replaygain_enabled', False): + try: + from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF + if _rg_ffmpeg_ok(): + lufs, peak_dbfs = _rg_analyze(final_path) + gain_db = _RG_REF - lufs + _rg_write(final_path, gain_db, peak_dbfs) + pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB — {os.path.basename(final_path)}") + except Exception as rg_err: + pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") + + downsampled_path = downsample_hires_flac(final_path, context) + if downsampled_path: + final_path = downsampled_path + context['_final_processed_path'] = final_path + + blasphemy_path = create_lossy_copy(final_path) + if blasphemy_path: + context['_final_processed_path'] = blasphemy_path + + downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + cleanup_empty_directories(downloads_path, file_path) + + logger.info(f"[Playlist Folder Mode] Post-processing complete: {final_path}") + + try: + check_and_remove_from_wishlist(context) + except Exception as wishlist_error: + logger.error(f"[Playlist Folder] Error checking wishlist removal: {wishlist_error}") + + emit_track_downloaded(context, automation_engine) + record_library_history_download(context) + record_download_provenance(context) + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id and batch_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['stream_processed'] = True + download_tasks[task_id]['status'] = 'completed' + logger.info(f"[Playlist Folder Mode] Marked task {task_id} as completed") + _notify_download_completed(batch_id, task_id, success=True) + return + + is_album_download = bool(context.get("is_album_download", False)) + album_info = build_import_album_info(context, force_album=is_album_download) + + if is_album_download: + if has_clean_metadata: + logger.info("Album context with clean metadata found - using normalized album info") + else: + logger.warning("Album context found without clean metadata - using normalized album info") + elif not album_info.get('is_album'): + logger.info("Single track download - attempting album detection") + detected_album_info = detect_album_info_web(context, artist_context) + if detected_album_info: + album_info = detected_album_info + + if album_info and album_info['is_album'] and not is_album_download: + logger.info( + "SMART ALBUM GROUPING for track=%r original_album=%r", + album_info.get('clean_track_name', 'Unknown'), + album_info.get('album_name', 'None'), + ) + original_album = original_search.get("album") if original_search.get("album") else None + consistent_album_name = resolve_album_group(artist_context, album_info, original_album) + album_info['album_name'] = consistent_album_name + logger.info("Album grouping complete: final_album=%r", consistent_album_name) + elif album_info and album_info['is_album'] and is_album_download: + logger.info( + "EXPLICIT ALBUM DOWNLOAD - preserving album name=%r; skipping smart grouping", + album_info.get('album_name', 'None'), + ) + + context['_audio_quality'] = get_audio_quality_string(file_path) + if context['_audio_quality']: + logger.info(f"Audio quality detected: {context['_audio_quality']}") + + rejection_reason = check_flac_bit_depth(file_path, context) + if rejection_reason: + try: + quarantine_path = move_to_quarantine( + file_path, + context, + rejection_reason, + automation_engine, + ) + logger.info(f"File quarantined due to bit depth filter: {quarantine_path}") + except Exception as quarantine_error: + logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") + try: + os.remove(file_path) + except Exception: + pass + + context['_bitdepth_rejected'] = True + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id: + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}" + if task_id and batch_id: + _notify_download_completed(batch_id, task_id, success=False) + return + + file_ext = os.path.splitext(file_path)[1] + clean_track_name = get_import_clean_title( + context, + album_info=album_info, + default=original_search.get('title', 'Unknown Track'), + ) + track_number = album_info.get('track_number', 1) + logger.debug( + "Final track_number processing: source=%s album_info_track_number=%s track_number=%s", + album_info.get('source', 'unknown'), + album_info.get('track_number', 'NOT_FOUND'), + track_number, + ) + if track_number is None: + track_number = extract_track_number_from_filename(file_path) + logger.info( + "Track number was None; extracted from filename=%r -> %s", + os.path.basename(file_path), + track_number, + ) + if not isinstance(track_number, int) or track_number < 1: + logger.error(f"Invalid track number ({track_number}), defaulting to 1") + track_number = 1 + + logger.debug(f"FINAL track_number used for filename: {track_number}") + album_info['track_number'] = track_number + album_info['clean_track_name'] = clean_track_name + logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata") + + final_path, _ = build_final_path_for_track(context, artist_context, album_info, file_ext) + logger.info(f"Resolved path: '{final_path}'") + context['_final_processed_path'] = final_path + + try: + logger.warning(f"[Metadata Input] artist: '{artist_context.get('name', 'MISSING')}' (id: {artist_context.get('id', 'MISSING')})") + if album_info: + logger.warning( + f"[Metadata Input] album: '{album_info.get('album_name', 'MISSING')}', " + f"track#: {album_info.get('track_number', 'MISSING')}, disc#: {album_info.get('disc_number', 'MISSING')}, " + f"source: {album_info.get('source', 'unknown')}" + ) + else: + logger.info("[Metadata Input] album_info: None (single track)") + enhance_file_metadata(file_path, context, artist_context, album_info) + except Exception as meta_err: + import traceback + pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") + wipe_source_tags(file_path) + + _enhance_source_info = context.get('track_info', {}).get('source_info') or {} + if isinstance(_enhance_source_info, str): + try: + _enhance_source_info = json.loads(_enhance_source_info) + except (json.JSONDecodeError, TypeError): + _enhance_source_info = {} + is_enhance_download = _enhance_source_info.get('enhance', False) + + logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") + if os.path.exists(final_path): + if not os.path.exists(file_path): + logger.info(f"[Protection] Destination exists and source already gone - file already transferred: {os.path.basename(final_path)}") + return + try: + from mutagen import File as MutagenFile + existing_file = MutagenFile(final_path) + has_metadata = existing_file is not None and len(existing_file.tags or {}) > 2 + if has_metadata and not is_enhance_download: + _replace_lower = config_manager.get('import.replace_lower_quality', False) + if _replace_lower: + _existing_tier = get_quality_tier_from_extension(final_path) + _incoming_tier = get_quality_tier_from_extension(file_path) + if _incoming_tier[1] < _existing_tier[1]: + logger.info(f"[Quality Replace] Replacing {_existing_tier[0]} with {_incoming_tier[0]}: {os.path.basename(final_path)}") + try: + os.remove(final_path) + except Exception as e: + logger.error(f"[Quality Replace] Could not remove existing file: {e}") + else: + logger.info( + f"[Protection] Existing file is same or better quality ({_existing_tier[0]} vs {_incoming_tier[0]}) - skipping: " + f"{os.path.basename(final_path)}" + ) + try: + os.remove(file_path) + except FileNotFoundError: + pass + except Exception as e: + logger.error(f"[Protection] Error removing redundant file: {e}") + return + else: + logger.info(f"[Protection] Existing file already has metadata enhancement - skipping overwrite: {os.path.basename(final_path)}") + logger.info(f"[Protection] Removing redundant download file: {os.path.basename(file_path)}") + try: + os.remove(file_path) + except FileNotFoundError: + logger.error(f"[Protection] Could not remove redundant file (already gone): {file_path}") + except Exception as e: + logger.error(f"[Protection] Error removing redundant file: {e}") + return + elif is_enhance_download: + logger.info(f"[Enhance] Quality enhance mode — replacing existing file: {os.path.basename(final_path)}") + try: + os.remove(final_path) + except Exception as e: + logger.error(f"[Enhance] Could not remove existing file for replacement: {e}") + else: + logger.info(f"[Protection] Existing file lacks metadata - safe to overwrite: {os.path.basename(final_path)}") + try: + os.remove(final_path) + except FileNotFoundError: + pass + except Exception as check_error: + logger.error(f"[Protection] Error checking existing file metadata, proceeding with overwrite: {check_error}") + try: + if os.path.exists(final_path): + os.remove(final_path) + except Exception as e: + logger.error(f"[Protection] Failed to remove existing file for overwrite: {e}") + + if not os.path.exists(file_path): + if os.path.exists(final_path): + logger.info(f"[Pre-Move] Source already gone and destination exists - another thread completed transfer: {os.path.basename(final_path)}") + download_cover_art(album_info, os.path.dirname(final_path), context) + generate_lrc_file(final_path, context, artist_context, album_info) + return + expected_dir = os.path.dirname(final_path) + expected_stem = os.path.splitext(os.path.basename(final_path))[0] + expected_ext = os.path.splitext(final_path)[1] + found_variant = None + check_exts = {expected_ext} + if expected_ext == '.flac' and config_manager.get('lossy_copy.enabled', False) and config_manager.get('lossy_copy.delete_original', False): + _lossy_ext_map = {'mp3': '.mp3', 'opus': '.opus', 'aac': '.m4a'} + _lossy_codec = config_manager.get('lossy_copy.codec', 'mp3') + check_exts.add(_lossy_ext_map.get(_lossy_codec, '.mp3')) + if os.path.exists(expected_dir): + for f in os.listdir(expected_dir): + f_ext = os.path.splitext(f)[1].lower() + if f_ext in check_exts and os.path.splitext(f)[0].startswith(expected_stem): + found_variant = os.path.join(expected_dir, f) + break + if found_variant: + logger.debug(f"[Pre-Move] Source gone but found variant in destination (stream processor handled it): {os.path.basename(found_variant)}") + context['_final_processed_path'] = found_variant + download_cover_art(album_info, expected_dir, context) + generate_lrc_file(found_variant, context, artist_context, album_info) + return + logger.warning(f"[Pre-Move] Source file gone and no matching file in destination: {os.path.basename(file_path)}") + raise FileNotFoundError(f"Source file vanished before move and destination does not exist: {file_path}") + + safe_move_file(file_path, final_path) + + if is_enhance_download and _enhance_source_info.get('original_file_path'): + original_enhance_path = _enhance_source_info['original_file_path'] + if os.path.normpath(original_enhance_path) != os.path.normpath(final_path) and os.path.exists(original_enhance_path): + try: + os.remove(original_enhance_path) + old_fmt = os.path.splitext(original_enhance_path)[1] + new_fmt = os.path.splitext(final_path)[1] + logger.info(f"[Enhance] Upgraded {old_fmt} → {new_fmt}: {os.path.basename(final_path)}") + except Exception as e: + logger.error(f"[Enhance] Could not remove old-format file: {e}") + elif is_enhance_download: + old_fmt = _enhance_source_info.get('original_format', 'unknown') + new_fmt = os.path.splitext(final_path)[1] + logger.info(f"[Enhance] Replaced in-place ({old_fmt} → {new_fmt}): {os.path.basename(final_path)}") + + download_cover_art(album_info, os.path.dirname(final_path), context) + generate_lrc_file(final_path, context, artist_context, album_info) + + if config_manager.get('post_processing.replaygain_enabled', False): + try: + from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF + if _rg_ffmpeg_ok(): + lufs, peak_dbfs = _rg_analyze(final_path) + gain_db = _RG_REF - lufs + _rg_write(final_path, gain_db, peak_dbfs) + pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB, peak {peak_dbfs:.2f} dBFS — {os.path.basename(final_path)}") + except Exception as rg_err: + pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") + + downsampled_path = downsample_hires_flac(final_path, context) + if downsampled_path: + final_path = downsampled_path + context['_final_processed_path'] = final_path + + blasphemy_path = create_lossy_copy(final_path) + if blasphemy_path: + context['_final_processed_path'] = blasphemy_path + + downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) + cleanup_empty_directories(downloads_path, file_path) + + logger.info(f"Post-processing complete for: {context.get('_final_processed_path', final_path)}") + + emit_track_downloaded(context, automation_engine) + record_library_history_download(context) + record_download_provenance(context) + record_soulsync_library_entry(context, artist_context, album_info) + + try: + if not playlist_folder_mode: + completed_path = context.get('_final_processed_path', final_path) + record_retag_download(context, artist_context, album_info, completed_path) + except Exception as retag_err: + logger.error(f"[Post-Process] Retag data capture failed (non-fatal): {retag_err}") + + try: + completed_path = context.get('_final_processed_path', final_path) + batch_id_for_repair = context.get('batch_id') + if completed_path and batch_id_for_repair and repair_worker: + album_folder = os.path.dirname(str(completed_path)) + if album_folder: + repair_worker.register_folder(batch_id_for_repair, album_folder) + except Exception as repair_err: + logger.error(f"[Post-Process] Repair folder registration failed: {repair_err}") + + try: + completed_path = context.get('_final_processed_path', final_path) + batch_id_for_consistency = context.get('batch_id') + if completed_path and batch_id_for_consistency and album_info and album_info.get('is_album'): + _file_info = { + 'path': str(completed_path), + 'track_number': album_info.get('track_number', 1), + 'disc_number': album_info.get('disc_number', 1), + 'title': get_import_clean_title( + context, + album_info=album_info, + default=album_info.get('clean_track_name', ''), + ), + } + with tasks_lock: + if batch_id_for_consistency in download_batches: + download_batches[batch_id_for_consistency].setdefault('_consistency_files', []).append(_file_info) + except Exception as cons_err: + logger.error(f"[Post-Process] Album consistency registration failed: {cons_err}") + + try: + check_and_remove_from_wishlist(context) + except Exception as wishlist_error: + logger.error(f"[Post-Process] Error checking wishlist removal: {wishlist_error}") + + task_id = context.get('task_id') + batch_id = context.get('batch_id') + if task_id and batch_id: + logger.info(f"[Post-Process] Calling completion callback for task {task_id} in batch {batch_id}") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['stream_processed'] = True + download_tasks[task_id]['status'] = 'completed' + logger.info(f"[Post-Process] Marked task {task_id} as completed") + _notify_download_completed(batch_id, task_id, success=True) + + except Exception as e: + import traceback + pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: {e}") + pp_logger.info(traceback.format_exc()) + logger.error(f"\nCRITICAL ERROR in post-processing for {context_key}: {e}") + traceback.print_exc() + + source_exists = os.path.exists(file_path) if file_path else False + if source_exists: + if context_key in _processed_download_ids: + _processed_download_ids.remove(context_key) + logger.warning(f"Removed {context_key} from processed set - will retry on next check") + with matched_context_lock: + if context_key not in matched_downloads_context: + matched_downloads_context[context_key] = context + logger.warning(f"Re-added {context_key} to context for retry") + else: + logger.warning(f"Source file gone, not retrying: {context_key}") + finally: + file_lock.release() + with _post_process_locks_lock: + _post_process_locks.pop(context_key, None) + + +def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime): + on_download_completed = getattr(runtime, "on_download_completed", None) + + def _notify_download_completed(batch_id, task_id, success=True): + if on_download_completed: + on_download_completed(batch_id, task_id, success=success) + + logger = pp_logger + try: + original_task_id = context.pop('task_id', None) + original_batch_id = context.pop('batch_id', None) + post_process_matched_download(context_key, context, file_path, runtime) + if original_task_id: + context['task_id'] = original_task_id + if original_batch_id: + context['batch_id'] = original_batch_id + + if context.get('_race_guard_failed'): + logger.info(f"Race guard: source file gone for task {task_id} — marking as failed") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = 'Source file was already processed or removed by another task' + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) + return + + if context.get('_acoustid_quarantined'): + failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed') + logger.info(f"File was quarantined by AcoustID verification (task={task_id}): {failure_msg}") + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {failure_msg}" + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) + return + + if context.get('_simple_download_completed'): + expected_final_path = context.get('_final_path') + if expected_final_path and os.path.exists(expected_final_path): + with tasks_lock: + if task_id in download_tasks: + _mark_task_completed(task_id, context.get('track_info')) + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=True) + return + logger.info( + f"FAILED simple download file not found at: {expected_final_path} " + f"(task={task_id}, context={context_key})" + ) + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f"Downloaded file not found at expected location: {os.path.basename(expected_final_path)}" + ) + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) + return + + expected_final_path = context.get('_final_processed_path') + if not expected_final_path: + logger.info(f"No _final_processed_path in context for task {task_id} — cannot verify, assuming success") + with tasks_lock: + if task_id in download_tasks: + _mark_task_completed(task_id, context.get('track_info')) + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=True) + return + + if os.path.exists(expected_final_path): + redownload_ctx = None + with tasks_lock: + if task_id in download_tasks: + _mark_task_completed(task_id, context.get('track_info')) + download_tasks[task_id]['metadata_enhanced'] = True + redownload_ctx = download_tasks[task_id].get('_redownload_context') + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + if redownload_ctx: + try: + old_path = redownload_ctx.get('old_file_path') + lib_track_id = redownload_ctx.get('library_track_id') + if redownload_ctx.get('delete_old_file') and old_path and os.path.exists(old_path): + if os.path.normpath(old_path) != os.path.normpath(expected_final_path): + os.remove(old_path) + logger.info(f"[Redownload] Deleted old file: {old_path}") + if lib_track_id and expected_final_path: + _rd_db = get_database() + _rd_conn = _rd_db._get_connection() + _rd_cursor = _rd_conn.cursor() + _rd_cursor.execute( + """ + UPDATE tracks SET file_path = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (expected_final_path, lib_track_id), + ) + _rd_conn.commit() + _rd_conn.close() + logger.info(f"[Redownload] Updated DB path for track {lib_track_id}") + except Exception as e: + logger.error(f"[Redownload] Post-processing hook error: {e}") + + _notify_download_completed(batch_id, task_id, success=True) + else: + track_name = get_import_clean_title(context, default=context_key) + logger.info(f"FAILED verification for '{track_name}' (task={task_id})") + logger.info(f" expected_final_path: {expected_final_path}") + logger.info(f" file_path (source): {file_path}, exists={os.path.exists(file_path)}") + logger.info( + f" is_album={context.get('is_album_download', False)}, " + f"has_clean_data={get_import_has_clean_metadata(context)}" + ) + expected_dir = os.path.dirname(expected_final_path) + if os.path.exists(expected_dir): + dir_contents = os.listdir(expected_dir) + logger.info(f" directory contains {len(dir_contents)} files: {dir_contents[:20]}") + else: + logger.info(f" directory does not exist: {expected_dir}") + + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = ( + f'File verification failed: expected file at {os.path.basename(expected_final_path)} but it was not found after processing' + ) + + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + + _notify_download_completed(batch_id, task_id, success=False) + except Exception as e: + import traceback + logger.info(f"EXCEPTION in post-processing for '{context_key}' (task={task_id}): {e}") + logger.info(traceback.format_exc()) + with tasks_lock: + if task_id in download_tasks: + download_tasks[task_id]['status'] = 'failed' + download_tasks[task_id]['error_message'] = f"Post-processing verification failed: {str(e)}" + with matched_context_lock: + if context_key in matched_downloads_context: + del matched_downloads_context[context_key] + _notify_download_completed(batch_id, task_id, success=False) diff --git a/core/import_runtime_state.py b/core/import_runtime_state.py new file mode 100644 index 00000000..9abf2baa --- /dev/null +++ b/core/import_runtime_state.py @@ -0,0 +1,121 @@ +"""Shared runtime state and tiny helpers for import/post-processing code.""" + +from __future__ import annotations + +import threading +import time +from typing import Any, Dict, Optional + +from core.import_context import ( + build_import_album_info, + extract_artist_name, + get_import_clean_artist, + get_import_context_album, + get_import_original_search, + get_import_track_info, + normalize_import_context, +) + +matched_context_lock = threading.Lock() +matched_downloads_context: Dict[str, Dict[str, Any]] = {} +tasks_lock = threading.Lock() +download_tasks: Dict[str, Dict[str, Any]] = {} +download_batches: Dict[str, Dict[str, Any]] = {} +_processed_download_ids = set() +_post_process_locks: Dict[str, threading.Lock] = {} +_post_process_locks_lock = threading.Lock() + +activity_feed = [] +activity_feed_lock = threading.Lock() +_activity_toast_emitter = None + + +def set_activity_toast_emitter(emitter) -> None: + """Set the WebSocket-style emitter used by add_activity_item.""" + global _activity_toast_emitter + _activity_toast_emitter = emitter + + +def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True): + """Append an activity item and emit a toast if an emitter is configured.""" + activity_item = { + "icon": icon, + "title": title, + "subtitle": subtitle, + "time": time_ago, + "timestamp": time.time(), + "show_toast": show_toast, + } + with activity_feed_lock: + activity_feed.append(activity_item) + if len(activity_feed) > 20: + activity_feed.pop(0) + + if show_toast and _activity_toast_emitter is not None: + try: + _activity_toast_emitter("dashboard:toast", activity_item) + except Exception: + pass + + return activity_item + + +def mark_task_completed(task_id: str, track_info: Optional[Dict[str, Any]] = None) -> bool: + """Mark a download task as completed in the shared task registry.""" + with tasks_lock: + task = download_tasks.get(task_id) + if not task: + return False + + task["status"] = "completed" + task["stream_processed"] = True + task["status_change_time"] = time.time() + if track_info is not None: + task["track_info"] = track_info + return True + + +def detect_album_info_web(context, artist_context=None): + """Best-effort album detection for single-track downloads.""" + context = normalize_import_context(context) + if artist_context is None: + artist_context = context.get("artist") or {} + + album_info = build_import_album_info(context) + if album_info.get("is_album"): + return album_info + + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + + album_name = ( + album_ctx.get("name") + or track_info.get("album") + or original_search.get("album") + or "" + ) + track_name = ( + track_info.get("name") + or original_search.get("title") + or "" + ) + artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="") + + if album_name and track_name and album_name.strip().lower() not in { + track_name.strip().lower(), + artist_name.strip().lower(), + }: + return build_import_album_info( + context, + album_info={ + "album_name": album_name, + "track_number": track_info.get("track_number", 1), + "disc_number": track_info.get("disc_number", 1), + "album_image_url": album_ctx.get("image_url", ""), + "confidence": 0.5, + }, + force_album=True, + ) + + return None diff --git a/core/import_side_effects.py b/core/import_side_effects.py new file mode 100644 index 00000000..6c456e3d --- /dev/null +++ b/core/import_side_effects.py @@ -0,0 +1,567 @@ +"""Import post-processing side effects that do not need web runtime state.""" + +from __future__ import annotations + +import hashlib +import json +import os +from typing import Any, Dict, List, Optional + +from core.import_context import ( + extract_artist_name, + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_context_album, + get_import_context_artist, + get_import_original_search, + get_import_source, + get_import_source_ids, + get_import_track_info, + normalize_import_context, + get_library_source_id_columns, +) +from core.wishlist_service import get_wishlist_service +from database.music_database import get_database +from utils.logging_config import get_logger + + +logger = get_logger("import_side_effects") + + +def _get_config_manager(): + from config.settings import config_manager + + return config_manager + + +def _primary_track_artist_name(track_info: Dict[str, Any]) -> str: + artists = (track_info or {}).get("artists", []) + if isinstance(artists, list) and artists: + first = artists[0] + if isinstance(first, dict): + return str(first.get("name", "") or "") + return str(first or "") + if isinstance(artists, str): + return artists + return str((track_info or {}).get("artist", "") or "") + + +def _all_profile_wishlist_tracks(wishlist_service) -> List[Dict[str, Any]]: + database = get_database() + all_profiles = database.get_all_profiles() + wishlist_tracks: List[Dict[str, Any]] = [] + for profile in all_profiles: + wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=profile["id"])) + return wishlist_tracks + + +def _stable_soulsync_id(text: str) -> str: + return str(abs(int(hashlib.md5(text.encode("utf-8", errors="replace")).hexdigest(), 16)) % (10 ** 9)) + + +def emit_track_downloaded(context: Dict[str, Any], automation_engine=None) -> None: + """Emit the track_downloaded automation event.""" + try: + if not automation_engine: + return + + ti = context.get("track_info") or context.get("search_result") or {} + artist_name = "" + artists = ti.get("artists", []) + if artists: + first = artists[0] + artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first) + + automation_engine.emit( + "track_downloaded", + { + "artist": artist_name, + "title": ti.get("name", ti.get("title", "")), + "album": ti.get("album", ""), + "quality": context.get("_audio_quality", "Unknown"), + }, + ) + except Exception: + pass + + +def record_library_history_download(context: Dict[str, Any]) -> None: + """Record a completed download to the library_history table.""" + try: + search_result = context.get("original_search_result") or context.get("search_result") or {} + username = search_result.get("username", context.get("_download_username", "")) + source_map = { + "youtube": "YouTube", + "tidal": "Tidal", + "qobuz": "Qobuz", + "hifi": "HiFi", + "deezer_dl": "Deezer", + "lidarr": "Lidarr", + } + download_source = source_map.get(username, "Soulseek") + + ti = context.get("track_info") or context.get("search_result") or {} + artist_name = _primary_track_artist_name(ti) + if not artist_name: + artist_name = ti.get("artist", "") + + album_raw = ti.get("album", "") + album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "") + title = ti.get("name", ti.get("title", "")) + quality = context.get("_audio_quality", "") + file_path = context.get("_final_processed_path", context.get("_final_path", "")) + + thumb_url = "" + album_context = get_import_context_album(context) + if album_context: + thumb_url = album_context.get("image_url", "") + if not thumb_url: + images = album_context.get("images", []) + if images: + thumb_url = images[0].get("url", "") + if not thumb_url: + album_info = context.get("album_info", {}) + if isinstance(album_info, dict): + thumb_url = album_info.get("album_image_url", "") + + source_filename = search_result.get("filename", "") + source_track_id = search_result.get("track_id", "") or search_result.get("id", "") or ti.get("id", "") + source_track_title = search_result.get("title", "") or search_result.get("name", "") + source_artist = search_result.get("artist", "") + if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr"): + stream_id = source_filename.split("||")[0] + if stream_id and not source_track_id: + source_track_id = stream_id + + acoustid_result = context.get("_acoustid_result", "") + + db = get_database() + db.add_library_history_entry( + event_type="download", + title=title, + artist_name=artist_name, + album_name=album_name, + quality=quality, + file_path=file_path, + thumb_url=thumb_url, + download_source=download_source, + source_track_id=source_track_id, + source_track_title=source_track_title, + source_filename=source_filename, + acoustid_result=acoustid_result, + source_artist=source_artist, + ) + except Exception: + pass + + +def record_download_provenance(context: Dict[str, Any]) -> None: + """Record source provenance for a completed download.""" + try: + search_result = context.get("original_search_result") or context.get("search_result") or {} + username = search_result.get("username", context.get("_download_username", "")) + filename = search_result.get("filename", "") + source_service = { + "youtube": "youtube", + "tidal": "tidal", + "qobuz": "qobuz", + "hifi": "hifi", + "deezer_dl": "deezer", + "lidarr": "lidarr", + }.get(username, "soulseek") + + ti = context.get("track_info") or context.get("search_result") or {} + artist_name = _primary_track_artist_name(ti) + if not artist_name: + artist_name = ti.get("artist", "") + + album_raw = ti.get("album", "") + album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "") + title = ti.get("name", ti.get("title", "")) + + file_path = context.get("_final_processed_path", context.get("_final_path", "")) + quality = context.get("_audio_quality", "") + size = search_result.get("size", 0) + + bit_depth = None + sample_rate = None + bitrate = None + try: + if file_path and os.path.isfile(file_path): + from mutagen import File as MutagenFile + + audio = MutagenFile(file_path) + if audio and audio.info: + sample_rate = getattr(audio.info, "sample_rate", None) + bitrate = getattr(audio.info, "bitrate", None) + bit_depth = getattr(audio.info, "bits_per_sample", None) + except Exception: + pass + + db = get_database() + db.record_track_download( + file_path=file_path, + source_service=source_service, + source_username=username, + source_filename=filename, + source_size=size or 0, + audio_quality=quality, + track_title=title, + track_artist=artist_name, + track_album=album_name, + bit_depth=bit_depth, + sample_rate=sample_rate, + bitrate=bitrate, + ) + except Exception: + pass + + +def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any]) -> None: + """Write imported media to the SoulSync library tables when the active server is SoulSync.""" + try: + config_manager = _get_config_manager() + if config_manager.get_active_media_server() != "soulsync": + return + + context = normalize_import_context(context) + final_path = context.get("_final_processed_path") + if not final_path: + return + + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + source = get_import_source(context) + source_ids = get_import_source_ids(context) + source_columns = get_library_source_id_columns(source) + + artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="") + if not artist_name or artist_name in ("Unknown", "Unknown Artist"): + return + + album_name = "" + if album_info and isinstance(album_info, dict): + album_name = album_info.get("album_name", "") + if not album_name: + album_name = album_ctx.get("name", "") or original_search.get("album", "") + if not album_name: + album_name = track_info.get("name", "Unknown") + + track_name = get_import_clean_title( + context, + album_info=album_info, + default=track_info.get("name", "") or original_search.get("title", ""), + ) + track_number = (track_info.get("track_number") or (album_info.get("track_number") if isinstance(album_info, dict) else None)) or 1 + duration_ms = track_info.get("duration_ms", 0) or 0 + + year = None + release_date = album_ctx.get("release_date", "") + if release_date and len(release_date) >= 4: + try: + year = int(release_date[:4]) + except ValueError: + pass + + image_url = album_ctx.get("image_url", "") + if not image_url: + images = album_ctx.get("images", []) + if images and isinstance(images, list) and len(images) > 0: + img = images[0] + image_url = img.get("url", "") if isinstance(img, dict) else str(img) + + artist_source_id = source_ids.get("artist_id", "") + album_source_id = source_ids.get("album_id", "") + track_source_id = source_ids.get("track_id", "") + for key in ("auto_import", "from_sync_modal", "explicit_artist", "explicit_album", ""): + if artist_source_id == key: + artist_source_id = "" + if album_source_id == key: + album_source_id = "" + if track_source_id == key: + track_source_id = "" + + genres = (artist_context or {}).get("genres", []) if isinstance(artist_context, dict) else [] + if genres: + from core.genre_filter import filter_genres as _filter_genres + + genres = _filter_genres(genres, config_manager) + genres_json = json.dumps(genres) if genres else "" + + bitrate = 0 + try: + from mutagen import File as MutagenFile + + audio = MutagenFile(final_path) + if audio and hasattr(audio, "info") and audio.info and hasattr(audio.info, "bitrate"): + bitrate = int(audio.info.bitrate / 1000) if audio.info.bitrate else 0 + except Exception: + pass + + artist_id = _stable_soulsync_id(artist_name.lower().strip()) + album_id = _stable_soulsync_id(f"{artist_name}::{album_name}".lower().strip()) + track_id = _stable_soulsync_id(final_path) + total_tracks = album_ctx.get("total_tracks", 0) or 0 + + db = get_database() + with db._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute("SELECT id FROM artists WHERE id = ? AND server_source = 'soulsync'", (artist_id,)) + if not cursor.fetchone(): + cursor.execute( + "SELECT id FROM artists WHERE name COLLATE NOCASE = ? AND server_source = 'soulsync' LIMIT 1", + (artist_name,), + ) + existing_by_name = cursor.fetchone() + if existing_by_name: + artist_id = existing_by_name[0] + else: + cursor.execute("SELECT id FROM artists WHERE id = ?", (artist_id,)) + if cursor.fetchone(): + artist_id = _stable_soulsync_id(artist_name.lower().strip() + "::soulsync") + cursor.execute( + """ + INSERT INTO artists (id, name, genres, thumb_url, server_source, created_at, updated_at) + VALUES (?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + (artist_id, artist_name, genres_json, image_url), + ) + artist_source_col = source_columns.get("artist") + if artist_source_col and artist_source_id: + try: + cursor.execute( + f"UPDATE artists SET {artist_source_col} = ? WHERE id = ?", + (artist_source_id, artist_id), + ) + except Exception: + pass + + cursor.execute("SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'", (album_id,)) + if not cursor.fetchone(): + cursor.execute( + "SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? AND server_source = 'soulsync' LIMIT 1", + (album_name, artist_id), + ) + existing_album_by_name = cursor.fetchone() + if existing_album_by_name: + album_id = existing_album_by_name[0] + else: + cursor.execute("SELECT id FROM albums WHERE id = ?", (album_id,)) + if cursor.fetchone(): + album_id = _stable_soulsync_id(f"{artist_name}::{album_name}::soulsync".lower().strip()) + cursor.execute( + """ + INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, track_count, + duration, server_source, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + (album_id, artist_id, album_name, year, image_url, genres_json, total_tracks, duration_ms), + ) + album_source_col = source_columns.get("album") + if album_source_col and album_source_id: + try: + cursor.execute( + f"UPDATE albums SET {album_source_col} = ? WHERE id = ?", + (album_source_id, album_id), + ) + except Exception: + pass + + track_artist = None + track_artists_list = track_info.get("artists", []) or original_search.get("artists", []) + if track_artists_list: + first_track_artist = track_artists_list[0] + if isinstance(first_track_artist, dict): + ta_name = first_track_artist.get("name", "") + else: + ta_name = str(first_track_artist) + if ta_name and ta_name.lower() != artist_name.lower(): + track_artist = ta_name + + cursor.execute("SELECT id FROM tracks WHERE file_path = ?", (final_path,)) + if not cursor.fetchone(): + cursor.execute( + """ + INSERT INTO tracks (id, album_id, artist_id, title, track_number, + duration, file_path, bitrate, track_artist, server_source, + created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + ( + track_id, + album_id, + artist_id, + track_name, + track_number, + duration_ms, + final_path, + bitrate, + track_artist, + ), + ) + track_source_col = source_columns.get("track") + if track_source_col and track_source_id: + try: + cursor.execute( + f"UPDATE tracks SET {track_source_col} = ? WHERE id = ?", + (track_source_id, track_id), + ) + track_album_col = source_columns.get("track_album") + if track_album_col and album_source_id: + cursor.execute( + f"UPDATE tracks SET {track_album_col} = ? WHERE id = ?", + (album_source_id, track_id), + ) + except Exception: + pass + + conn.commit() + logger.info("[SoulSync Library] Added: %s / %s / %s", artist_name, album_name, track_name) + except Exception as exc: + logger.error("[SoulSync Library] Could not record library entry: %s", exc) + + +def record_retag_download(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any], final_path: str) -> None: + """Record a completed download for later re-tagging.""" + try: + db = get_database() + + context = normalize_import_context(context) + artist_context = get_import_context_artist(context) or (artist_context if isinstance(artist_context, dict) else {}) + album_context = get_import_context_album(context) + track_info = get_import_track_info(context) + original_search = get_import_original_search(context) + source = get_import_source(context) + source_ids = get_import_source_ids(context) + + artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="Unknown Artist") + is_album = album_info and album_info.get("is_album", False) + group_type = "album" if is_album else "single" + album_name = album_info.get("album_name", "") if album_info else get_import_clean_album(context, default=original_search.get("album", "Unknown")) + + image_url = album_info.get("album_image_url") if album_info else None + if not image_url: + image_url = album_context.get("image_url", "") + if not image_url and album_context.get("images"): + images = album_context.get("images", []) + if images and isinstance(images[0], dict): + image_url = images[0].get("url", "") + + total_tracks = album_context.get("total_tracks", 1) if album_context else 1 + release_date = album_context.get("release_date", "") if album_context else "" + + spotify_album_id = None + itunes_album_id = None + if source == "spotify": + spotify_album_id = source_ids.get("album_id", "") or None + elif source == "itunes": + itunes_album_id = source_ids.get("album_id", "") or None + + group_id = db.find_retag_group(artist_name, album_name) + if group_id is None: + group_id = db.add_retag_group( + group_type=group_type, + artist_name=artist_name, + album_name=album_name, + image_url=image_url, + spotify_album_id=spotify_album_id, + itunes_album_id=itunes_album_id, + total_tracks=total_tracks, + release_date=release_date, + ) + if group_id is None: + return + + track_number = album_info.get("track_number", 1) if album_info else (track_info.get("track_number", 1) or 1) + disc_number = original_search.get("disc_number") or (album_info.get("disc_number", 1) if album_info else track_info.get("disc_number", 1) or 1) + title = get_import_clean_title( + context, + album_info=album_info, + default=album_info.get("clean_track_name", "Unknown Track") if album_info else "Unknown Track", + ) + file_format = os.path.splitext(str(final_path))[1].lstrip(".").lower() + + spotify_track_id = None + itunes_track_id = None + if source == "spotify": + spotify_track_id = source_ids.get("track_id", "") or None + elif source == "itunes": + itunes_track_id = source_ids.get("track_id", "") or None + + if not db.retag_track_exists(group_id, str(final_path)): + db.add_retag_track( + group_id=group_id, + track_number=track_number, + disc_number=disc_number, + title=title, + file_path=str(final_path), + file_format=file_format, + spotify_track_id=spotify_track_id, + itunes_track_id=itunes_track_id, + ) + logger.info("[Retag] Recorded track for retag: '%s' in '%s'", title, album_name) + + db.trim_retag_groups(100) + except Exception as exc: + logger.error("[Retag] Could not record track for retag: %s", exc) + + +def check_and_remove_from_wishlist(context: Dict[str, Any]) -> None: + """Check whether a successful download should be removed from the wishlist.""" + try: + wishlist_service = get_wishlist_service() + spotify_track_id = None + + track_info = context.get("track_info", {}) + if track_info.get("id"): + spotify_track_id = track_info["id"] + logger.info("[Wishlist] Found Spotify ID from track_info: %s", spotify_track_id) + elif context.get("original_search_result", {}).get("id"): + spotify_track_id = context["original_search_result"]["id"] + logger.info("[Wishlist] Found Spotify ID from original_search_result: %s", spotify_track_id) + elif "wishlist_id" in track_info: + wishlist_id = track_info["wishlist_id"] + logger.info("[Wishlist] Found wishlist_id in context: %s", wishlist_id) + wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service) + for wishlist_track in wishlist_tracks: + if wishlist_track.get("wishlist_id") == wishlist_id: + spotify_track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id") + logger.info("[Wishlist] Found Spotify ID from wishlist entry: %s", spotify_track_id) + break + + if not spotify_track_id: + track_name = track_info.get("name") or context.get("original_search_result", {}).get("title", "") + artist_name = _primary_track_artist_name(track_info) or _primary_track_artist_name(context.get("original_search_result", {})) + + if track_name and artist_name: + logger.warning("[Wishlist] No Spotify ID found, checking for fuzzy match: '%s' by '%s'", track_name, artist_name) + + wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service) + for wishlist_track in wishlist_tracks: + wl_name = wishlist_track.get("name", "").lower() + wl_artists = wishlist_track.get("artists", []) + wl_artist_name = "" + if wl_artists: + if isinstance(wl_artists[0], dict): + wl_artist_name = wl_artists[0].get("name", "").lower() + else: + wl_artist_name = str(wl_artists[0]).lower() + if wl_name == track_name.lower() and wl_artist_name == artist_name.lower(): + spotify_track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id") + logger.info("[Wishlist] Found fuzzy match - Spotify ID: %s", spotify_track_id) + break + + if spotify_track_id: + logger.info("[Wishlist] Attempting to remove track from wishlist: %s", spotify_track_id) + removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) + if removed: + logger.info("[Wishlist] Successfully removed track from wishlist: %s", spotify_track_id) + else: + logger.warning("ℹ️ [Wishlist] Track not found in wishlist or already removed: %s", spotify_track_id) + else: + logger.warning("ℹ️ [Wishlist] No Spotify track ID found for wishlist removal check") + except Exception as exc: + logger.error("[Wishlist] Error in wishlist removal check: %s", exc) diff --git a/core/import_staging.py b/core/import_staging.py new file mode 100644 index 00000000..da93580a --- /dev/null +++ b/core/import_staging.py @@ -0,0 +1,454 @@ +"""Shared staging folder and import suggestion helpers.""" + +from __future__ import annotations + +import os +import threading +from typing import Any, Dict, List, Optional, Tuple + +from core.import_paths import docker_resolve_path +from utils.logging_config import get_logger + +logger = get_logger("import_staging") + +AUDIO_EXTENSIONS = {".mp3", ".flac", ".ogg", ".opus", ".m4a", ".aac", ".wav", ".wma", ".aiff", ".aif", ".ape"} + +_import_suggestions_cache_lock = threading.Lock() +_import_suggestions_cache: Dict[str, Any] = { + "suggestions": [], + "building": False, + "built": False, +} + + +def _get_config_manager(): + try: + from config.settings import config_manager + + return config_manager + except Exception: + class _FallbackConfig: + @staticmethod + def get(key, default=None): + return default + + return _FallbackConfig() + + +def get_staging_path() -> str: + """Resolve the configured staging folder path.""" + raw = _get_config_manager().get("import.staging_path", "./Staging") + return docker_resolve_path(raw) + + +def get_import_suggestions_cache() -> Dict[str, Any]: + """Expose the shared import suggestions cache.""" + return _import_suggestions_cache + + +def get_primary_source() -> str: + from core.metadata_service import get_primary_source as _get_primary_source + + return _get_primary_source() + + +def get_source_priority(preferred_source: str): + from core.metadata_service import get_source_priority as _get_source_priority + + return _get_source_priority(preferred_source) + + +def get_client_for_source(source: str): + from core.metadata_service import get_client_for_source as _get_client_for_source + + return _get_client_for_source(source) + + +def _search_albums_for_source(source: str, client: Any, query: str, limit: int = 5): + from core.metadata_service import _search_albums_for_source as _metadata_search_albums_for_source + + return _metadata_search_albums_for_source(source, client, query, limit=limit) + + +def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 5): + from core.metadata_service import _search_tracks_for_source as _metadata_search_tracks_for_source + + return _metadata_search_tracks_for_source(source, client, query, limit=limit) + + +def _extract_value(value: Any, *names: str, default: Any = None) -> Any: + if value is None: + return default + + if isinstance(value, (str, bytes)): + return default + + for name in names: + if isinstance(value, dict): + if name in value and value[name] is not None: + return value[name] + else: + candidate = getattr(value, name, None) + if candidate is not None: + return candidate + + return default + + +def _extract_artist_names(artists: Any) -> List[str]: + if not artists: + return [] + + if isinstance(artists, (str, bytes)): + artist = str(artists).strip() + return [artist] if artist else [] + + try: + items = list(artists) + except TypeError: + items = [artists] + + names: List[str] = [] + for artist in items: + if isinstance(artist, dict): + name = str(_extract_value(artist, "name", "artist_name", "title", default="") or "").strip() + else: + candidate = getattr(artist, "name", None) + if candidate is None: + candidate = artist + name = str(candidate or "").strip() + if name: + names.append(name) + + return names + + +def _normalize_album_result(album: Any, source: str) -> Dict[str, Any]: + album_id = str(_extract_value(album, "id", "album_id", "release_id", default="") or "").strip() + album_name = str(_extract_value(album, "name", "title", default="") or "").strip() + artists = _extract_artist_names(_extract_value(album, "artists", default=[])) + artist_name = ", ".join(artists) if artists else str( + _extract_value(album, "artist_name", "artist", default="Unknown Artist") or "Unknown Artist" + ).strip() + release_date = str(_extract_value(album, "release_date", "releaseDate", default="") or "").strip() + album_type = str(_extract_value(album, "album_type", "type", default="album") or "album").strip() or "album" + + total_tracks = _extract_value(album, "total_tracks", "track_count", default=0) + if isinstance(total_tracks, (list, tuple, set)): + total_tracks = len(total_tracks) + try: + total_tracks = int(total_tracks or 0) + except (TypeError, ValueError): + total_tracks = 0 + + image_url = _extract_value(album, "image_url", "thumb_url", "cover_image", "cover_url", default="") + if not image_url: + images = _extract_value(album, "images", default=[]) or [] + if isinstance(images, dict): + images = [images] + elif isinstance(images, (str, bytes)): + images = [images] + try: + images = list(images) + except TypeError: + images = [images] + if images: + first_image = images[0] + if isinstance(first_image, (str, bytes)): + image_url = str(first_image).strip() + else: + image_url = _extract_value(first_image, "url", "image_url", "src", default="") + + return { + "id": album_id or album_name or "unknown-album", + "name": album_name or album_id or "Unknown Album", + "artist": artist_name or "Unknown Artist", + "release_date": release_date, + "total_tracks": total_tracks, + "image_url": str(image_url or ""), + "album_type": album_type, + "source": source, + } + + +def _album_fingerprint(album: Dict[str, Any]) -> Tuple[str, str, str, str]: + return ( + str(album.get("name", "") or "").strip().casefold(), + str(album.get("artist", "") or "").strip().casefold(), + str(album.get("release_date", "") or "").strip()[:10].casefold(), + str(album.get("album_type", "") or "").strip().casefold(), + ) + + +def _normalize_track_result(track: Any, source: str) -> Dict[str, Any]: + track_id = str(_extract_value(track, "id", "track_id", "trackId", default="") or "").strip() + track_name = str(_extract_value(track, "name", "title", "track_name", default="") or "").strip() + artists = _extract_artist_names(_extract_value(track, "artists", default=[])) + artist_name = ", ".join(artists) if artists else str( + _extract_value(track, "artist", "artist_name", default="Unknown Artist") or "Unknown Artist" + ).strip() + + album_value = _extract_value(track, "album", default=None) + album_name = "" + album_id = str(_extract_value(track, "album_id", "collectionId", "albumId", default="") or "").strip() + if isinstance(album_value, dict): + album_name = str(_extract_value(album_value, "name", "title", default="") or "").strip() + album_id = album_id or str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip() + if not album_name: + album_name = album_id + elif isinstance(album_value, (str, bytes)): + album_name = str(album_value).strip() + elif album_value is not None: + album_name = str(_extract_value(album_value, "name", "title", default=album_value) or "").strip() + if not album_id: + album_id = str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip() + + image_url = _extract_value(track, "image_url", "thumb_url", "cover_image", default="") + if not image_url: + images = _extract_value(track, "images", default=[]) or [] + if isinstance(images, dict): + images = [images] + elif isinstance(images, (str, bytes)): + images = [images] + try: + images = list(images) + except TypeError: + images = [images] + if images: + first_image = images[0] + if isinstance(first_image, (str, bytes)): + image_url = str(first_image).strip() + else: + image_url = _extract_value(first_image, "url", "image_url", "src", default="") + if not image_url and album_value is not None: + album_images = _extract_value(album_value, "images", default=[]) or [] + if isinstance(album_images, dict): + album_images = [album_images] + elif isinstance(album_images, (str, bytes)): + album_images = [album_images] + try: + album_images = list(album_images) + except TypeError: + album_images = [album_images] + if album_images: + first_album_image = album_images[0] + if isinstance(first_album_image, (str, bytes)): + image_url = str(first_album_image).strip() + else: + image_url = _extract_value(first_album_image, "url", "image_url", "src", default="") + + duration_ms = _extract_value(track, "duration_ms", "duration", "trackTimeMillis", default=0) + try: + duration_ms = int(duration_ms or 0) + except (TypeError, ValueError): + duration_ms = 0 + + track_number = _extract_value(track, "track_number", "trackNumber", default=1) + try: + track_number = int(track_number or 1) + except (TypeError, ValueError): + track_number = 1 + + return { + "id": track_id or track_name or "unknown-track", + "name": track_name or track_id or "Unknown Track", + "artist": artist_name or "Unknown Artist", + "album": album_name or "", + "album_id": album_id or "", + "duration_ms": duration_ms, + "image_url": str(image_url or ""), + "track_number": track_number, + "source": source, + } + + +def _read_staging_audio_tags(file_path: str) -> Tuple[Optional[str], Optional[str]]: + try: + from mutagen import File as MutagenFile + + tags = MutagenFile(file_path, easy=True) + if not tags: + return None, None + + album = (tags.get("album") or [None])[0] + artist = (tags.get("artist") or (tags.get("albumartist") or [None]))[0] + album_text = str(album).strip() if album else "" + artist_text = str(artist).strip() if artist else "" + return (album_text or None, artist_text or None) + except Exception: + return None, None + + +def _collect_import_suggestion_queries(staging_path: str) -> List[str]: + tag_albums: Dict[Tuple[str, str], int] = {} + folder_hints: Dict[str, int] = {} + + for root, _dirs, filenames in os.walk(staging_path): + audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS] + if not audio_files: + continue + + rel_dir = os.path.relpath(root, staging_path) + if rel_dir != ".": + top_folder = rel_dir.split(os.sep)[0] + folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files) + + for fname in audio_files: + full_path = os.path.join(root, fname) + album, artist = _read_staging_audio_tags(full_path) + if album: + key = (album.strip(), (artist or "").strip()) + tag_albums[key] = tag_albums.get(key, 0) + 1 + + queries: List[str] = [] + seen_lower = set() + + for (album, artist), _count in sorted(tag_albums.items(), key=lambda item: -item[1]): + q = f"{album} {artist}".strip() if artist else album + if q and q.lower() not in seen_lower: + seen_lower.add(q.lower()) + queries.append(q) + + for folder, _count in sorted(folder_hints.items(), key=lambda item: -item[1]): + q = folder.replace("_", " ") + if q and q.lower() not in seen_lower: + seen_lower.add(q.lower()) + queries.append(q) + + return queries[:5] + + +def search_import_albums(query: str, limit: int = 12) -> List[Dict[str, Any]]: + """Search albums using the configured metadata provider first.""" + query = (query or "").strip() + if not query: + return [] + + results: List[Dict[str, Any]] = [] + seen = set() + source_chain = get_source_priority(get_primary_source()) + + for source in source_chain: + client = get_client_for_source(source) + if not client: + continue + + source_results = _search_albums_for_source(source, client, query, limit=limit) + if not source_results: + continue + + added_for_source = False + for album in source_results: + suggestion = _normalize_album_result(album, source) + fingerprint = _album_fingerprint(suggestion) + if fingerprint in seen: + continue + seen.add(fingerprint) + results.append(suggestion) + added_for_source = True + if len(results) >= limit: + return results[:limit] + + if added_for_source: + break + + return results[:limit] + + +def search_import_tracks(query: str, limit: int = 30) -> List[Dict[str, Any]]: + """Search tracks using the configured metadata provider priority order.""" + query = (query or "").strip() + if not query: + return [] + + results: List[Dict[str, Any]] = [] + source_chain = get_source_priority(get_primary_source()) + + for source in source_chain: + client = get_client_for_source(source) + if not client: + continue + + source_results = _search_tracks_for_source(source, client, query, limit=limit) + if not source_results: + continue + + for track in source_results: + results.append(_normalize_track_result(track, source)) + if len(results) >= limit: + return results[:limit] + break + + return results[:limit] + + +def _build_import_suggestions_background(): + cache = _import_suggestions_cache + + with _import_suggestions_cache_lock: + if cache["building"]: + return + cache["building"] = True + + try: + staging_path = get_staging_path() + if not os.path.isdir(staging_path): + with _import_suggestions_cache_lock: + cache["suggestions"] = [] + cache["built"] = True + return + + queries = _collect_import_suggestion_queries(staging_path) + if not queries: + with _import_suggestions_cache_lock: + cache["suggestions"] = [] + cache["built"] = True + return + + suggestions: List[Dict[str, Any]] = [] + seen = set() + for query in queries: + try: + albums = search_import_albums(query, limit=2) + for album in albums: + fingerprint = _album_fingerprint(album) + if fingerprint in seen: + continue + seen.add(fingerprint) + suggestions.append(album) + except Exception as exc: + logger.warning("Import suggestion search failed for %r: %s", query, exc) + + with _import_suggestions_cache_lock: + cache["suggestions"] = suggestions[:8] + cache["built"] = True + + logger.info( + "Import suggestions cache built: %s suggestions from %s hints", + len(cache["suggestions"]), + len(queries), + ) + except Exception as exc: + logger.error("Error building import suggestions cache: %s", exc) + with _import_suggestions_cache_lock: + cache["suggestions"] = [] + cache["built"] = True + finally: + with _import_suggestions_cache_lock: + cache["building"] = False + + +def start_import_suggestions_cache(): + """Start building the import suggestions cache in a background thread.""" + threading.Thread( + target=_build_import_suggestions_background, + daemon=True, + name="import-suggestions-cache", + ).start() + + +def refresh_import_suggestions_cache(): + """Invalidate and rebuild the suggestions cache.""" + with _import_suggestions_cache_lock: + _import_suggestions_cache["built"] = False + start_import_suggestions_cache() diff --git a/core/metadata_enrichment.py b/core/metadata_enrichment.py new file mode 100644 index 00000000..293af162 --- /dev/null +++ b/core/metadata_enrichment.py @@ -0,0 +1,1486 @@ +"""Source-aware metadata enrichment helpers for imported audio files.""" + +from __future__ import annotations + +import json +import os +import re +import threading +import time +import urllib.request +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Dict, Optional + +from core.import_context import ( + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_context_album, + get_import_context_artist, + get_import_original_search, + get_import_source, + get_import_source_ids, + get_import_track_info, + get_source_tag_names, + normalize_import_context, +) +from config.settings import config_manager +from core.metadata_service import get_itunes_client +from database.music_database import get_database +from utils.logging_config import get_logger + + +logger = get_logger("metadata_enrichment") + +_FILE_LOCKS: Dict[str, threading.Lock] = {} +_FILE_LOCKS_LOCK = threading.Lock() + +_MB_RELEASE_CACHE: Dict[tuple, str] = {} +_MB_RELEASE_CACHE_LOCK = threading.RLock() +_MB_RELEASE_DETAIL_CACHE: Dict[str, Dict[str, Any]] = {} +_MB_RELEASE_DETAIL_CACHE_LOCK = threading.RLock() + +_EDITION_PAREN_RE = re.compile( + r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' + r'limited|bonus|platinum|gold|super\s*deluxe|standard)' + r'(?:\s+(?:edition|version))?[^)\]]*[\)\]]', + re.IGNORECASE, +) +_EDITION_BARE_RE = re.compile( + r'\s+(?:-\s+)?(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' + r'limited|bonus|platinum|gold|super\s*deluxe|standard)' + r'(?:\s+(?:edition|version))?\s*$', + re.IGNORECASE, +) + + +class _NullConfigManager: + def get(self, _key: str, default: Any = None) -> Any: + return default + + +def _get_logger(runtime=None): + return logger + + +def _get_config_manager(runtime=None): + return config_manager + + +def _get_database(runtime=None): + try: + return get_database() + except Exception: + return None + + +def _get_itunes_client(runtime=None): + try: + return get_itunes_client() + except Exception: + worker = getattr(runtime, "itunes_enrichment_worker", None) + if worker and getattr(worker, "client", None): + return worker.client + return getattr(runtime, "itunes_client", None) + + +def _extract_artist_name(artist: Any) -> str: + if isinstance(artist, dict): + return str(artist.get("name", "") or "") + if hasattr(artist, "name"): + return str(getattr(artist, "name") or "") + return str(artist) if artist else "" + + +def _get_mutagen_symbols(runtime=None): + """Lazy mutagen import so tests can monkeypatch this without the package installed.""" + try: + from mutagen import File as MutagenFile + from mutagen.apev2 import APEv2, APENoHeaderError + from mutagen.flac import FLAC, Picture + from mutagen.id3 import ( + APIC, + ID3, + TBPM, + TCOP, + TDOR, + TDRC, + TCON, + TIT2, + TALB, + TPE1, + TPE2, + TPOS, + TPUB, + TRCK, + TSRC, + TXXX, + UFID, + TMED, + ) + from mutagen.mp4 import MP4, MP4Cover, MP4FreeForm + from mutagen.oggvorbis import OggVorbis + try: + from mutagen.oggopus import OggOpus + except Exception: + OggOpus = None + except Exception as exc: + _get_logger(runtime).debug("Mutagen unavailable for metadata enrichment: %s", exc) + return None + + return SimpleNamespace( + File=MutagenFile, + APEv2=APEv2, + APENoHeaderError=APENoHeaderError, + FLAC=FLAC, + Picture=Picture, + ID3=ID3, + APIC=APIC, + TBPM=TBPM, + TCOP=TCOP, + TDOR=TDOR, + TDRC=TDRC, + TCON=TCON, + TIT2=TIT2, + TALB=TALB, + TPE1=TPE1, + TPE2=TPE2, + TPOS=TPOS, + TPUB=TPUB, + TRCK=TRCK, + TSRC=TSRC, + TXXX=TXXX, + UFID=UFID, + TMED=TMED, + MP4=MP4, + MP4Cover=MP4Cover, + MP4FreeForm=MP4FreeForm, + OggVorbis=OggVorbis, + OggOpus=OggOpus, + ) + + +def _get_file_lock(file_path: str) -> threading.Lock: + with _FILE_LOCKS_LOCK: + lock = _FILE_LOCKS.get(file_path) + if lock is None: + lock = threading.Lock() + _FILE_LOCKS[file_path] = lock + return lock + + +def _is_ogg_opus(audio_file: Any) -> bool: + return type(audio_file).__name__ == "OggOpus" + + +def _is_vorbis_like(audio_file: Any, symbols: Any) -> bool: + vorbis_classes = tuple( + cls for cls in ( + getattr(symbols, "FLAC", None), + getattr(symbols, "OggVorbis", None), + ) if cls is not None + ) + return bool(vorbis_classes) and isinstance(audio_file, vorbis_classes) or _is_ogg_opus(audio_file) + + +def _save_audio_file(audio_file: Any, symbols: Any) -> None: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.save(v1=0, v2_version=4) + elif isinstance(audio_file, symbols.FLAC): + audio_file.save(deleteid3=True) + else: + audio_file.save() + + +def _strip_all_non_audio_tags(file_path: str, runtime=None) -> dict: + summary = {"apev2_stripped": False, "apev2_tag_count": 0} + if os.path.splitext(file_path)[1].lower() != ".mp3": + return summary + + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return summary + + try: + apev2_tags = symbols.APEv2(file_path) + tag_count = len(apev2_tags) + tag_keys = list(apev2_tags.keys()) + apev2_tags.delete(file_path) + summary["apev2_stripped"] = True + summary["apev2_tag_count"] = tag_count + _get_logger(runtime).info("Stripped %s APEv2 tags: %s", tag_count, ", ".join(tag_keys[:10])) + except symbols.APENoHeaderError: + pass + except Exception as exc: + _get_logger(runtime).error("Could not strip APEv2 tags (non-fatal): %s", exc) + return summary + + +def _verify_metadata_written(file_path: str, runtime=None) -> bool: + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return False + + try: + check = symbols.File(file_path) + if check is None or check.tags is None: + _get_logger(runtime).info("[VERIFY] Tags are None after save: %s", file_path) + return False + + title_found = False + artist_found = False + if isinstance(check.tags, symbols.ID3): + title_found = bool(check.tags.getall("TIT2")) + artist_found = bool(check.tags.getall("TPE1")) + try: + symbols.APEv2(file_path) + _get_logger(runtime).info("[VERIFY] APEv2 tags still present after processing!") + return False + except symbols.APENoHeaderError: + pass + elif _is_vorbis_like(check, symbols): + title_found = bool(check.get("title")) + artist_found = bool(check.get("artist")) + elif isinstance(check, symbols.MP4): + title_found = bool(check.get("\xa9nam")) + artist_found = bool(check.get("\xa9ART")) + + if not title_found or not artist_found: + _get_logger(runtime).warning("[VERIFY] Missing metadata - title:%s artist:%s", title_found, artist_found) + return False + + _get_logger(runtime).info("[VERIFY] Metadata verified OK") + return True + except Exception as exc: + _get_logger(runtime).error("[VERIFY] Verification error (non-fatal): %s", exc) + return False + + +def _get_image_dimensions(data: bytes): + try: + if data[:8] == b"\x89PNG\r\n\x1a\n": + import struct + + w, h = struct.unpack(">II", data[16:24]) + return w, h + if data[:2] == b"\xff\xd8": + import struct + + i = 2 + while i < len(data) - 9: + if data[i] != 0xFF: + break + marker = data[i + 1] + if marker in (0xC0, 0xC2): + h, w = struct.unpack(">HH", data[i + 5 : i + 9]) + return w, h + length = struct.unpack(">H", data[i + 2 : i + 4])[0] + i += 2 + length + except Exception: + pass + return None, None + + +def extract_source_metadata(context: dict, artist: dict, album_info: dict, runtime=None) -> dict: + if album_info is None: + album_info = {} + + cfg = _get_config_manager(runtime) + context = normalize_import_context(context) + original_search = get_import_original_search(context) + album_ctx = get_import_context_album(context) + track_info = get_import_track_info(context) + source = get_import_source(context) + source_ids = get_import_source_ids(context) + + artist_dict = artist if isinstance(artist, dict) else { + "name": _extract_artist_name(artist), + "id": getattr(artist, "id", ""), + "genres": list(getattr(artist, "genres", []) or []), + } + + metadata: Dict[str, Any] = { + "source": source, + "source_track_id": source_ids["track_id"], + "source_artist_id": source_ids["artist_id"], + "source_album_id": source_ids["album_id"], + } + + metadata["title"] = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "")) + if original_search.get("clean_title"): + _get_logger(runtime).info("Metadata: Using clean title: '%s'", metadata["title"]) + elif album_info.get("clean_track_name"): + _get_logger(runtime).info("Metadata: Using album info clean name: '%s'", metadata["title"]) + else: + _get_logger(runtime).warning("Metadata: Using original title as fallback: '%s'", metadata["title"]) + + artists = original_search.get("artists") + if isinstance(artists, list) and artists: + all_artists = [] + for artist_item in artists: + if isinstance(artist_item, dict) and artist_item.get("name"): + all_artists.append(artist_item["name"]) + elif isinstance(artist_item, str): + all_artists.append(artist_item) + else: + all_artists.append(str(artist_item)) + metadata["artist"] = ", ".join(all_artists) + _get_logger(runtime).info("Metadata: Using all artists: '%s'", metadata["artist"]) + else: + metadata["artist"] = artist_dict.get("name", "") or get_import_clean_artist(context) + _get_logger(runtime).info("Metadata: Using primary artist: '%s'", metadata["artist"]) + + raw_album_artist = artist_dict.get("name", "") or metadata["artist"] + track_info_ctx = track_info or {} + explicit_artist = track_info_ctx.get("_explicit_artist_context") if isinstance(track_info_ctx, dict) else None + album_artists_for_collab = None + + if isinstance(explicit_artist, dict) and explicit_artist.get("name"): + raw_album_artist = explicit_artist["name"] + album_artists_for_collab = [explicit_artist] + elif isinstance(explicit_artist, str) and explicit_artist: + raw_album_artist = explicit_artist + album_artists_for_collab = [{"name": explicit_artist}] + elif album_ctx and isinstance(album_ctx, dict): + album_artists = album_ctx.get("artists", []) + if album_artists: + first_album_artist = album_artists[0] + if isinstance(first_album_artist, dict) and first_album_artist.get("name"): + raw_album_artist = first_album_artist["name"] + elif isinstance(first_album_artist, str) and first_album_artist: + raw_album_artist = first_album_artist + album_artists_for_collab = album_artists + + collab_mode = cfg.get("file_organization.collab_artist_mode", "first") + if collab_mode == "first" and raw_album_artist: + context_artists = album_artists_for_collab or original_search.get("artists") or track_info_ctx.get("artists") or [] + if len(context_artists) > 1: + first = context_artists[0] + raw_album_artist = first.get("name", first) if isinstance(first, dict) else str(first) + elif len(context_artists) == 1 and ("," in raw_album_artist or " & " in raw_album_artist): + artist_id = str(artist_dict.get("id", "")) + if source == "itunes" and artist_id.isdigit(): + try: + itunes_client = _get_itunes_client(runtime) + if itunes_client and hasattr(itunes_client, "resolve_primary_artist"): + resolved = itunes_client.resolve_primary_artist(artist_id) + if resolved and resolved != raw_album_artist: + raw_album_artist = resolved + except Exception: + pass + metadata["album_artist"] = raw_album_artist + + if album_info.get("is_album"): + metadata["album"] = album_info.get("album_name", "Unknown Album") + metadata["track_number"] = album_info.get("track_number", 1) + metadata["total_tracks"] = album_ctx.get("total_tracks", 1) if album_ctx else 1 + _get_logger(runtime).info("[METADATA] Album track - track_number: %s, album: %s", metadata["track_number"], metadata["album"]) + else: + if album_ctx and album_ctx.get("name"): + _get_logger(runtime).info("[SAFEGUARD] Using album context name instead of track title for album metadata") + metadata["album"] = album_ctx["name"] + metadata["track_number"] = album_info.get("track_number", 1) if album_info else 1 + metadata["total_tracks"] = album_ctx.get("total_tracks", 1) + else: + metadata["album"] = metadata["title"] + metadata["track_number"] = 1 + metadata["total_tracks"] = 1 + + disc_num = original_search.get("disc_number") + if disc_num is None and album_info: + disc_num = album_info.get("disc_number") + metadata["disc_number"] = disc_num if disc_num is not None else 1 + + if album_ctx and album_ctx.get("release_date"): + metadata["date"] = album_ctx["release_date"][:4] + + genres = artist_dict.get("genres") or [] + if genres: + from core.genre_filter import filter_genres + + filtered = filter_genres(list(genres[:2]), cfg) + if filtered: + metadata["genre"] = ", ".join(filtered) + + metadata["album_art_url"] = album_info.get("album_image_url") if album_info else None + if not metadata["album_art_url"] and album_ctx: + album_image = album_ctx.get("image_url") + if not album_image and album_ctx.get("images"): + first_image = album_ctx["images"][0] + album_image = first_image.get("url") if isinstance(first_image, dict) else None + metadata["album_art_url"] = album_image + + _get_logger(runtime).info( + "[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | track=%s/%s | disc=%s", + metadata.get("title"), + metadata.get("artist"), + metadata.get("album_artist"), + metadata.get("album"), + metadata.get("track_number"), + metadata.get("total_tracks"), + metadata.get("disc_number"), + ) + + return metadata + + +def embed_album_art_metadata(audio_file, metadata: dict, runtime=None): + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return + + try: + image_data = None + mime_type = None + + release_mbid = metadata.get("musicbrainz_release_id") + if release_mbid and cfg.get("metadata_enhancement.prefer_caa_art", False): + try: + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) + with urllib.request.urlopen(req, timeout=10) as response: + image_data = response.read() + mime_type = response.info().get_content_type() or "image/jpeg" + if not image_data or len(image_data) <= 1000: + image_data = None + except Exception: + image_data = None + + if not image_data: + art_url = metadata.get("album_art_url") + if not art_url: + logger_.warning("No album art URL available for embedding.") + return + with urllib.request.urlopen(art_url, timeout=10) as response: + image_data = response.read() + mime_type = response.info().get_content_type() or "image/jpeg" + + if not image_data: + logger_.error("Failed to download album art data.") + return + + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.APIC(encoding=3, mime=mime_type, type=3, desc="Cover", data=image_data)) + elif isinstance(audio_file, symbols.FLAC): + picture = symbols.Picture() + picture.data = image_data + picture.type = 3 + picture.mime = mime_type + width, height = _get_image_dimensions(image_data) + picture.width = width or 640 + picture.height = height or 640 + picture.depth = 24 + audio_file.add_picture(picture) + elif isinstance(audio_file, symbols.MP4): + fmt = symbols.MP4Cover.FORMAT_JPEG if "jpeg" in mime_type else symbols.MP4Cover.FORMAT_PNG + audio_file["covr"] = [symbols.MP4Cover(image_data, imageformat=fmt)] + + logger_.info("Album art successfully embedded.") + except Exception as exc: + logger_.error("Error embedding album art: %s", exc) + + +def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=None): + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return + + try: + tag_config = { + "SPOTIFY_TRACK_ID": "spotify.tags.track_id", + "SPOTIFY_ARTIST_ID": "spotify.tags.artist_id", + "SPOTIFY_ALBUM_ID": "spotify.tags.album_id", + "ITUNES_TRACK_ID": "itunes.tags.track_id", + "ITUNES_ARTIST_ID": "itunes.tags.artist_id", + "ITUNES_ALBUM_ID": "itunes.tags.album_id", + "MUSICBRAINZ_RECORDING_ID": "musicbrainz.tags.recording_id", + "MUSICBRAINZ_ARTIST_ID": "musicbrainz.tags.artist_id", + "MUSICBRAINZ_RELEASE_ID": "musicbrainz.tags.release_id", + "MUSICBRAINZ_RELEASEGROUPID": "musicbrainz.tags.release_group_id", + "MUSICBRAINZ_ALBUMARTISTID": "musicbrainz.tags.album_artist_id", + "MUSICBRAINZ_RELEASETRACKID": "musicbrainz.tags.release_track_id", + "RELEASETYPE": "musicbrainz.tags.release_type", + "ORIGINALDATE": "musicbrainz.tags.original_date", + "RELEASESTATUS": "musicbrainz.tags.release_status", + "RELEASECOUNTRY": "musicbrainz.tags.release_country", + "BARCODE": "musicbrainz.tags.barcode", + "MEDIA": "musicbrainz.tags.media", + "TOTALDISCS": "musicbrainz.tags.total_discs", + "CATALOGNUMBER": "musicbrainz.tags.catalog_number", + "SCRIPT": "musicbrainz.tags.script", + "ASIN": "musicbrainz.tags.asin", + "DEEZER_TRACK_ID": "deezer.tags.track_id", + "DEEZER_ARTIST_ID": "deezer.tags.artist_id", + "AUDIODB_TRACK_ID": "audiodb.tags.track_id", + "TIDAL_TRACK_ID": "tidal.tags.track_id", + "TIDAL_ARTIST_ID": "tidal.tags.artist_id", + "QOBUZ_TRACK_ID": "qobuz.tags.track_id", + "QOBUZ_ARTIST_ID": "qobuz.tags.artist_id", + "GENIUS_TRACK_ID": "genius.tags.track_id", + } + + def _tag_enabled(path: str) -> bool: + return cfg.get(path, True) is not False + + def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: + if not a or not b: + return False + from difflib import SequenceMatcher + + norm = lambda s: re.sub(r"[^a-z0-9 ]", "", re.sub(r"\(.*?\)", "", s).lower()).strip() + return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold + + context = normalize_import_context(context) + source = (metadata.get("source") or "").strip().lower() + source_ids = {} + if source: + source_tag_names = get_source_tag_names(source) + source_track_id = metadata.get("source_track_id") + source_artist_id = metadata.get("source_artist_id") + source_album_id = metadata.get("source_album_id") + if cfg.get(f"{source}.embed_tags", True) is not False: + if source_tag_names.get("track") and source_track_id: + source_ids[source_tag_names["track"]] = source_track_id + if source_tag_names.get("artist") and source_artist_id: + source_ids[source_tag_names["artist"]] = source_artist_id + if source_tag_names.get("album") and source_album_id: + source_ids[source_tag_names["album"]] = source_album_id + + if not source_ids: + if cfg.get("spotify.embed_tags", True) is not False: + if metadata.get("spotify_track_id"): + source_ids["SPOTIFY_TRACK_ID"] = metadata["spotify_track_id"] + if metadata.get("spotify_artist_id"): + source_ids["SPOTIFY_ARTIST_ID"] = metadata["spotify_artist_id"] + if metadata.get("spotify_album_id"): + source_ids["SPOTIFY_ALBUM_ID"] = metadata["spotify_album_id"] + if cfg.get("itunes.embed_tags", True) is not False: + if metadata.get("itunes_track_id"): + source_ids["ITUNES_TRACK_ID"] = metadata["itunes_track_id"] + if metadata.get("itunes_artist_id"): + source_ids["ITUNES_ARTIST_ID"] = metadata["itunes_artist_id"] + if metadata.get("itunes_album_id"): + source_ids["ITUNES_ALBUM_ID"] = metadata["itunes_album_id"] + + track_title = metadata.get("title", "") + artist_name = metadata.get("album_artist", "") or metadata.get("artist", "") + track_info = get_import_track_info(context) + explicit_artist = (track_info or {}).get("_explicit_artist_context") if isinstance(track_info, dict) else None + batch_artist_name = None + if isinstance(explicit_artist, dict) and explicit_artist.get("name"): + batch_artist_name = explicit_artist["name"] + elif isinstance(explicit_artist, str) and explicit_artist: + batch_artist_name = explicit_artist + + pp = { + "id_tags": source_ids, + "track_title": track_title, + "artist_name": artist_name, + "batch_artist_name": batch_artist_name, + "metadata": metadata, + "recording_mbid": None, + "artist_mbid": None, + "release_mbid": "", + "mb_genres": [], + "isrc": None, + "deezer_bpm": None, + "deezer_isrc": None, + "audiodb_mood": None, + "audiodb_style": None, + "audiodb_genre": None, + "tidal_isrc": None, + "tidal_copyright": None, + "qobuz_isrc": None, + "qobuz_copyright": None, + "qobuz_label": None, + "lastfm_tags": [], + "lastfm_url": None, + "genius_url": None, + "release_year": None, + } + + source_order = cfg.get("metadata_enhancement.post_process_order", None) + if not isinstance(source_order, list) or not source_order: + source_order = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"] + + db = _get_database(runtime) + + for source_name in source_order: + if source_name == "musicbrainz": + if cfg.get("musicbrainz.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + mb_worker = getattr(runtime, "mb_worker", None) + mb_service = mb_worker.mb_service if mb_worker else None + if not mb_service: + continue + try: + result = mb_service.match_recording(track_title, artist_name) + if result and result.get("mbid"): + pp["recording_mbid"] = result["mbid"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"] + details = mb_service.mb_client.get_recording(pp["recording_mbid"], includes=["isrcs", "genres"]) + if details: + isrcs = details.get("isrcs", []) + if isrcs: + pp["isrc"] = isrcs[0] + pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)] + + track_artist_name = metadata.get("artist", "") or artist_name + if ", " in track_artist_name: + track_artist_name = track_artist_name.split(", ")[0] + artist_result = mb_service.match_artist(track_artist_name) + if artist_result and artist_result.get("mbid"): + pp["artist_mbid"] = artist_result["mbid"] + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"] + + album_name_for_mb = metadata.get("album", "") + if album_name_for_mb: + artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip() + rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), artist_key) + rc_key_exact = (album_name_for_mb.lower().strip(), artist_key) + with _MB_RELEASE_CACHE_LOCK: + cached = _MB_RELEASE_CACHE.get(rc_key_norm) + if cached is None: + cached = _MB_RELEASE_CACHE.get(rc_key_exact) + if cached is not None: + pp["release_mbid"] = cached + else: + try: + rc_result = mb_service.match_release(album_name_for_mb, artist_name) + pp["release_mbid"] = rc_result.get("mbid", "") if rc_result else "" + except Exception: + pp["release_mbid"] = "" + _MB_RELEASE_CACHE[rc_key_norm] = pp["release_mbid"] + _MB_RELEASE_CACHE[rc_key_exact] = pp["release_mbid"] + if pp["release_mbid"]: + pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"] + + if pp["release_mbid"]: + with _MB_RELEASE_DETAIL_CACHE_LOCK: + release_detail = _MB_RELEASE_DETAIL_CACHE.get(pp["release_mbid"]) + if release_detail is None: + release_detail = mb_service.mb_client.get_release( + pp["release_mbid"], + includes=["release-groups", "labels", "media", "artist-credits", "recordings"], + ) or {} + with _MB_RELEASE_DETAIL_CACHE_LOCK: + _MB_RELEASE_DETAIL_CACHE[pp["release_mbid"]] = release_detail + if release_detail: + rg = release_detail.get("release-group", {}) + if rg.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"] + ac = release_detail.get("artist-credit", []) + if ac and isinstance(ac[0], dict): + aa = ac[0].get("artist", {}) + if aa.get("id"): + pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"] + if rg.get("primary-type"): + pp["id_tags"]["RELEASETYPE"] = rg["primary-type"] + if rg.get("first-release-date"): + pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"] + if not pp["release_year"] and len(rg["first-release-date"]) >= 4: + year = rg["first-release-date"][:4] + if year.isdigit(): + pp["release_year"] = year + if release_detail.get("status"): + pp["id_tags"]["RELEASESTATUS"] = release_detail["status"] + if release_detail.get("country"): + pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"] + if release_detail.get("barcode"): + pp["id_tags"]["BARCODE"] = release_detail["barcode"] + media_list = release_detail.get("media", []) + if media_list: + fmt = media_list[0].get("format", "") + if fmt: + pp["id_tags"]["MEDIA"] = fmt + pp["id_tags"]["TOTALDISCS"] = str(len(media_list)) + label_info = release_detail.get("label-info", []) + if label_info and isinstance(label_info[0], dict): + cat = label_info[0].get("catalog-number", "") + if cat: + pp["id_tags"]["CATALOGNUMBER"] = cat + text_rep = release_detail.get("text-representation", {}) + if isinstance(text_rep, dict) and text_rep.get("script"): + pp["id_tags"]["SCRIPT"] = text_rep["script"] + if release_detail.get("asin"): + pp["id_tags"]["ASIN"] = release_detail["asin"] + track_num = metadata.get("track_number") + disc_num = metadata.get("disc_number") or 1 + if track_num and media_list: + try: + track_num_int = int(track_num) + disc_num_int = int(disc_num) + for medium in media_list: + if medium.get("position", 1) == disc_num_int: + for mtrack in (medium.get("tracks") or medium.get("track-list", [])): + if mtrack.get("position") == track_num_int: + if mtrack.get("id"): + pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"] + release_recording = mtrack.get("recording", {}) + if release_recording.get("id"): + pp["recording_mbid"] = release_recording["id"] + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"] + break + break + except (ValueError, TypeError): + pass + except Exception as exc: + logger_.error("MusicBrainz lookup failed (non-fatal): %s", exc) + continue + + if source_name == "deezer": + if cfg.get("deezer.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + deezer_worker = getattr(runtime, "deezer_worker", None) + dz_client = deezer_worker.client if deezer_worker else None + if not dz_client: + continue + dz_result = dz_client.search_track(artist_name, track_title) + if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name): + dz_track_id = dz_result["id"] + pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id) + dz_artist_id = dz_result.get("artist", {}).get("id") + if dz_artist_id: + pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id) + dz_details = dz_client.get_track_details(dz_track_id) + if dz_details: + bpm_val = dz_details.get("bpm") + if bpm_val and bpm_val > 0: + pp["deezer_bpm"] = bpm_val + dz_isrc = dz_details.get("isrc") + if dz_isrc: + pp["deezer_isrc"] = dz_isrc + if not pp["release_year"]: + dz_album = dz_result.get("album", {}) + dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or "" + if len(dz_release) >= 4 and dz_release[:4].isdigit(): + pp["release_year"] = dz_release[:4] + except Exception as exc: + logger_.error("Deezer lookup failed (non-fatal): %s", exc) + continue + + if source_name == "audiodb": + if cfg.get("audiodb.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + audiodb_worker = getattr(runtime, "audiodb_worker", None) + adb_client = audiodb_worker.client if audiodb_worker else None + if not adb_client: + continue + adb_result = adb_client.search_track(artist_name, track_title) + if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name): + adb_track_id = adb_result.get("idTrack") + if adb_track_id: + pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id) + adb_mb_track = adb_result.get("strMusicBrainzID") + if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track + pp["recording_mbid"] = adb_mb_track + adb_mb_artist = adb_result.get("strMusicBrainzArtistID") + if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]: + pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist + pp["artist_mbid"] = adb_mb_artist + pp["audiodb_mood"] = adb_result.get("strMood") or None + pp["audiodb_style"] = adb_result.get("strStyle") or None + pp["audiodb_genre"] = adb_result.get("strGenre") or None + except Exception as exc: + logger_.error("AudioDB lookup failed (non-fatal): %s", exc) + continue + + if source_name == "tidal": + if cfg.get("tidal.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + tidal_client = getattr(runtime, "tidal_client", None) + if not (tidal_client and tidal_client.is_authenticated()): + continue + td_result = tidal_client.search_track(artist_name, track_title) + if td_result and _names_match(td_result.get("title", ""), track_title): + td_track_id = td_result.get("id") + if td_track_id: + pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id) + td_artist = td_result.get("artist", {}) + if isinstance(td_artist, dict) and td_artist.get("id"): + pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"]) + if td_track_id: + td_details = tidal_client.get_track(str(td_track_id)) + if td_details: + pp["tidal_isrc"] = td_details.get("isrc") + td_copyright = td_details.get("copyright") + if isinstance(td_copyright, dict): + td_copyright = td_copyright.get("text", td_copyright.get("name", "")) + pp["tidal_copyright"] = td_copyright or None + if not pp["release_year"]: + td_album = td_result.get("album", {}) + td_release = "" + if isinstance(td_album, dict): + td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "") + if len(td_release) >= 4 and td_release[:4].isdigit(): + pp["release_year"] = td_release[:4] + except Exception as exc: + logger_.error("Tidal lookup failed (non-fatal): %s", exc) + continue + + if source_name == "qobuz": + if cfg.get("qobuz.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None) + qz_client = qobuz_worker.client if qobuz_worker else None + if not (qz_client and qz_client.is_authenticated()): + continue + qz_result = qz_client.search_track(artist_name, track_title) + if qz_result: + qz_performer = qz_result.get("performer") or {} + if not isinstance(qz_performer, dict): + qz_performer = {} + qz_artist_name = qz_performer.get("name", "") + if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name): + qz_track_id = qz_result.get("id") + if qz_track_id: + pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id) + if qz_performer.get("id"): + pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"]) + qz_isrc = qz_result.get("isrc") + if isinstance(qz_isrc, dict): + qz_isrc = qz_isrc.get("value", qz_isrc.get("id", "")) + if qz_isrc: + pp["qobuz_isrc"] = qz_isrc + qz_copyright = qz_result.get("copyright") + if isinstance(qz_copyright, dict): + qz_copyright = qz_copyright.get("text", qz_copyright.get("name", "")) + if isinstance(qz_copyright, str): + pp["qobuz_copyright"] = qz_copyright + qz_album = qz_result.get("album", {}) + if isinstance(qz_album, dict): + qz_label_info = qz_album.get("label", {}) + if isinstance(qz_label_info, dict) and qz_label_info.get("name"): + pp["qobuz_label"] = qz_label_info["name"] + if not pp["release_year"]: + qz_release = str(qz_album.get("release_date_original", "") or "") + if not qz_release: + qz_ts = qz_album.get("released_at") + if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: + import datetime as _dt + qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) + if len(qz_release) >= 4 and qz_release[:4].isdigit(): + pp["release_year"] = qz_release[:4] + except Exception as exc: + logger_.error("Qobuz lookup failed (non-fatal): %s", exc) + continue + + if source_name == "lastfm": + if cfg.get("lastfm.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + lastfm_worker = getattr(runtime, "lastfm_worker", None) + lf_client = lastfm_worker.client if lastfm_worker else None + if not lf_client: + continue + lf_result = lf_client.get_track_info(artist_name, track_title) + if lf_result: + lf_url = lf_result.get("url") + if lf_url: + pp["lastfm_url"] = lf_url + lf_toptags = lf_result.get("toptags", {}) + if isinstance(lf_toptags, dict): + tag_list = lf_toptags.get("tag", []) + if isinstance(tag_list, list): + pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")] + elif isinstance(tag_list, dict) and tag_list.get("name"): + pp["lastfm_tags"] = [tag_list["name"]] + except Exception as exc: + logger_.error("Last.fm lookup failed (non-fatal): %s", exc) + continue + + if source_name == "genius": + if cfg.get("genius.embed_tags", True) is False: + continue + if not track_title or not artist_name: + continue + try: + import core.genius_client as _genius_module + + if time.time() < _genius_module._rate_limit_until: + logger_.info("Genius rate-limited, skipping (non-blocking)") + continue + genius_worker = getattr(runtime, "genius_worker", None) + g_client = genius_worker.client if genius_worker else None + if not g_client: + continue + g_result = g_client.search_song(artist_name, track_title) + if g_result: + g_id = g_result.get("id") + if g_id: + pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id) + g_url = g_result.get("url") + if g_url: + pp["genius_url"] = g_url + except Exception as exc: + logger_.error("Genius lookup failed (non-fatal): %s", exc) + continue + + if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["audiodb_mood"] and not pp["audiodb_style"]: + return + + filtered_tags: Dict[str, str] = {} + for tag_name, value in pp["id_tags"].items(): + config_path = tag_config.get(tag_name) + if config_path and not _tag_enabled(config_path): + continue + filtered_tags[tag_name] = value + + written = [] + id3_tag_map = { + "MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"), + "MUSICBRAINZ_ARTIST_ID": ("TXXX", "MusicBrainz Artist Id"), + "MUSICBRAINZ_RELEASE_ID": ("TXXX", "MusicBrainz Album Id"), + "MUSICBRAINZ_RELEASEGROUPID": ("TXXX", "MusicBrainz Release Group Id"), + "MUSICBRAINZ_ALBUMARTISTID": ("TXXX", "MusicBrainz Album Artist Id"), + "MUSICBRAINZ_RELEASETRACKID": ("TXXX", "MusicBrainz Release Track Id"), + "RELEASETYPE": ("TXXX", "MusicBrainz Album Type"), + "RELEASESTATUS": ("TXXX", "MusicBrainz Album Status"), + "RELEASECOUNTRY": ("TXXX", "MusicBrainz Album Release Country"), + "ORIGINALDATE": ("TDOR", None), + "MEDIA": ("TMED", None), + } + vorbis_tag_map = { + "MUSICBRAINZ_RECORDING_ID": "MUSICBRAINZ_TRACKID", + "MUSICBRAINZ_ARTIST_ID": "MUSICBRAINZ_ARTISTID", + "MUSICBRAINZ_RELEASE_ID": "MUSICBRAINZ_ALBUMID", + "MUSICBRAINZ_RELEASEGROUPID": "MUSICBRAINZ_RELEASEGROUPID", + "MUSICBRAINZ_ALBUMARTISTID": "MUSICBRAINZ_ALBUMARTISTID", + "MUSICBRAINZ_RELEASETRACKID": "MUSICBRAINZ_RELEASETRACKID", + } + mp4_tag_map = { + "MUSICBRAINZ_RECORDING_ID": "MusicBrainz Track Id", + "MUSICBRAINZ_ARTIST_ID": "MusicBrainz Artist Id", + "MUSICBRAINZ_RELEASE_ID": "MusicBrainz Album Id", + "MUSICBRAINZ_RELEASEGROUPID": "MusicBrainz Release Group Id", + "MUSICBRAINZ_ALBUMARTISTID": "MusicBrainz Album Artist Id", + "MUSICBRAINZ_RELEASETRACKID": "MusicBrainz Release Track Id", + "RELEASETYPE": "MusicBrainz Album Type", + "RELEASESTATUS": "MusicBrainz Album Status", + "RELEASECOUNTRY": "MusicBrainz Album Release Country", + } + + if isinstance(audio_file.tags, symbols.ID3): + for tag_name, value in filtered_tags.items(): + spec = id3_tag_map.get(tag_name) + if spec: + frame_type, desc = spec + if frame_type == "UFID": + audio_file.tags.add(symbols.UFID(owner=desc, data=str(value).encode("ascii"))) + written.append(f"UFID:{desc}") + elif frame_type == "TDOR": + audio_file.tags.add(symbols.TDOR(encoding=3, text=[value])) + written.append("TDOR") + elif frame_type == "TMED": + audio_file.tags.add(symbols.TMED(encoding=3, text=[value])) + written.append("TMED") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=desc, text=[value])) + written.append(f"TXXX:{desc}") + else: + audio_file.tags.add(symbols.TXXX(encoding=3, desc=tag_name, text=[str(value)])) + written.append(f"TXXX:{tag_name}") + elif _is_vorbis_like(audio_file, symbols): + for tag_name, value in filtered_tags.items(): + audio_file[vorbis_tag_map.get(tag_name, tag_name)] = [str(value)] + written.append(vorbis_tag_map.get(tag_name, tag_name)) + elif isinstance(audio_file, symbols.MP4): + for tag_name, value in filtered_tags.items(): + key = f"----:com.apple.iTunes:{mp4_tag_map.get(tag_name, tag_name)}" + audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))] + written.append(key) + + if written: + logger_.info("Embedded IDs: %s", ", ".join(written)) + + release_year = pp["release_year"] + needs_date_tag = bool(release_year and not metadata.get("date")) + if needs_date_tag: + metadata["date"] = release_year + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["date"] = [release_year] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9day"] = [release_year] + logger_.info("Date tag: %s", release_year) + + if _tag_enabled("deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0: + bpm_int = int(pp["deezer_bpm"]) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["BPM"] = [str(bpm_int)] + elif isinstance(audio_file, symbols.MP4): + audio_file["tmpo"] = [bpm_int] + logger_.info("BPM: %s", bpm_int) + + if _tag_enabled("audiodb.tags.mood") and pp["audiodb_mood"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["MOOD"] = [pp["audiodb_mood"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))] + + if _tag_enabled("audiodb.tags.style") and pp["audiodb_style"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["STYLE"] = [pp["audiodb_style"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))] + + if _tag_enabled("metadata_enhancement.tags.genre_merge"): + enrichment_genres = [] + if _tag_enabled("musicbrainz.tags.genres"): + enrichment_genres += pp["mb_genres"] + if pp["audiodb_genre"] and _tag_enabled("audiodb.tags.genre"): + enrichment_genres.append(pp["audiodb_genre"]) + if _tag_enabled("lastfm.tags.genres"): + enrichment_genres += pp["lastfm_tags"] + if enrichment_genres: + from core.genre_filter import filter_genres as _filter_genres + + enrichment_genres = _filter_genres(enrichment_genres, cfg) + source_genres = [g.strip() for g in str(metadata.get("genre", "")).split(",") if g.strip()] + seen = set() + merged = [] + for genre in source_genres + enrichment_genres: + key = genre.strip().lower() + if key and key not in seen: + seen.add(key) + merged.append(genre.strip().title()) + if len(merged) >= 5: + break + if merged: + genre_string = ", ".join(merged) + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["GENRE"] = [genre_string] + elif isinstance(audio_file, symbols.MP4): + audio_file["\xa9gen"] = [genre_string] + logger_.info("Genres merged: %s", genre_string) + + isrc_candidates = [] + if pp["isrc"] and _tag_enabled("musicbrainz.tags.isrc"): + isrc_candidates.append(("MusicBrainz", pp["isrc"])) + if pp["deezer_isrc"] and _tag_enabled("deezer.tags.isrc"): + isrc_candidates.append(("Deezer", pp["deezer_isrc"])) + if pp["tidal_isrc"] and _tag_enabled("tidal.tags.isrc"): + isrc_candidates.append(("Tidal", pp["tidal_isrc"])) + if pp["qobuz_isrc"] and _tag_enabled("qobuz.tags.isrc"): + isrc_candidates.append(("Qobuz", pp["qobuz_isrc"])) + if isrc_candidates: + isrc_source, final_isrc = isrc_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["ISRC"] = [final_isrc] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))] + logger_.info("ISRC (%s): %s", isrc_source, final_isrc) + + copyright_candidates = [] + if pp["tidal_copyright"] and _tag_enabled("tidal.tags.copyright"): + copyright_candidates.append(("Tidal", pp["tidal_copyright"])) + if pp["qobuz_copyright"] and _tag_enabled("qobuz.tags.copyright"): + copyright_candidates.append(("Qobuz", pp["qobuz_copyright"])) + if copyright_candidates: + copyright_source, final_copyright = copyright_candidates[0] + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["COPYRIGHT"] = [final_copyright] + elif isinstance(audio_file, symbols.MP4): + audio_file["cprt"] = [final_copyright] + logger_.info("Copyright (%s): %s", copyright_source, final_copyright[:60]) + + if _tag_enabled("qobuz.tags.label") and pp["qobuz_label"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["LABEL"] = [pp["qobuz_label"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))] + + if _tag_enabled("lastfm.tags.url") and pp["lastfm_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["LASTFM_URL"] = [pp["lastfm_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))] + + if _tag_enabled("genius.tags.url") and pp["genius_url"]: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["GENIUS_URL"] = [pp["genius_url"]] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))] + + release_id = pp["release_mbid"] + if release_id: + metadata["musicbrainz_release_id"] = release_id + if db is not None: + try: + album_name_for_db = metadata.get("album", "") + album_artist_for_db = metadata.get("album_artist", "") or metadata.get("artist", "") + if album_name_for_db and album_artist_for_db: + conn = db._get_connection() + try: + cursor = conn.cursor() + cursor.execute( + """ + UPDATE albums SET year = ? + WHERE (year IS NULL OR year = 0) + AND id IN ( + SELECT al.id FROM albums al + JOIN artists ar ON ar.id = al.artist_id + WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) + ) + """, + (int(release_year), album_name_for_db, album_artist_for_db), + ) + if cursor.rowcount > 0: + conn.commit() + logger_.info("Updated album year to %s in database", release_year) + else: + conn.rollback() + finally: + conn.close() + except Exception as exc: + logger_.error("Could not update album year in DB: %s", exc) + + except Exception as exc: + logger_.error("Error embedding source IDs (non-fatal): %s", exc) + + +def download_cover_art(album_info: dict, target_dir: str, context: dict = None, runtime=None): + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + if cfg.get("metadata_enhancement.cover_art_download", True) is False: + return + + try: + cover_path = os.path.join(target_dir, "cover.jpg") + album_info = album_info or {} + release_mbid = album_info.get("musicbrainz_release_id") + prefer_caa = cfg.get("metadata_enhancement.prefer_caa_art", False) + + if os.path.exists(cover_path): + if release_mbid and prefer_caa: + try: + existing_size = os.path.getsize(cover_path) + if existing_size > 200_000: + return + is_upgrade = True + except Exception: + return + else: + return + else: + is_upgrade = False + + image_data = None + if release_mbid and prefer_caa: + try: + caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" + req = urllib.request.Request(caa_url, headers={"Accept": "image/*"}) + with urllib.request.urlopen(req, timeout=10) as response: + image_data = response.read() + if not image_data or len(image_data) <= 1000: + image_data = None + except Exception: + image_data = None + + if is_upgrade and not image_data: + logger_.error("CAA upgrade failed - keeping existing cover.jpg") + return + + if not image_data: + art_url = album_info.get("album_image_url") + if not art_url and context: + album_ctx = get_import_context_album(context) + art_url = album_ctx.get("image_url") + if not art_url and album_ctx.get("images"): + images = album_ctx.get("images", []) + if images and isinstance(images[0], dict): + art_url = images[0].get("url", "") + if art_url: + logger_.info("Using cover art URL from album context") + if art_url and "i.scdn.co" in art_url: + try: + from core.spotify_client import _upgrade_spotify_image_url + + art_url = _upgrade_spotify_image_url(art_url) + except Exception: + pass + elif art_url and "mzstatic.com" in art_url: + import re as _re + + art_url = _re.sub(r"\d+x\d+bb", "3000x3000bb", art_url) + if not art_url: + logger_.warning("No cover art URL available for download.") + return + with urllib.request.urlopen(art_url, timeout=10) as response: + image_data = response.read() + + if not image_data: + return + + with open(cover_path, "wb") as handle: + handle.write(image_data) + logger_.info("Cover art downloaded to: %s", cover_path) + except Exception as exc: + logger_.error("Error downloading cover.jpg: %s", exc) + + +def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + if cfg.get("metadata_enhancement.lrclib_enabled", True) is False: + return False + + try: + from core.lyrics_client import lyrics_client + + context = normalize_import_context(context) + original_search = get_import_original_search(context) + album_context = get_import_context_album(context) + track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track")) + + if isinstance(artist, dict): + artist_name = artist.get("name", "Unknown Artist") + elif hasattr(artist, "name"): + artist_name = artist.name + else: + artist_name = str(artist) if artist else "Unknown Artist" + + album_name = None + duration_seconds = None + if album_info and album_info.get("is_album"): + album_name = ( + get_import_clean_album(context, album_info=album_info, default="") + or album_info.get("album_name") + or album_context.get("name") + ) + + if original_search.get("duration_ms"): + duration_seconds = int(original_search["duration_ms"] / 1000) + + success = lyrics_client.create_lrc_file( + audio_file_path=file_path, + track_name=track_name, + artist_name=artist_name, + album_name=album_name, + duration_seconds=duration_seconds, + ) + + if success: + logger_.info("LRC file generated for: %s", track_name) + else: + logger_.warning("No lyrics found for: %s", track_name) + return success + except Exception as exc: + logger_.error("Error generating LRC file for %s: %s", file_path, exc) + return False + + +def wipe_source_tags(file_path: str, runtime=None) -> bool: + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + _ = cfg # keep signature parallel with other helpers + + try: + _strip_all_non_audio_tags(file_path, runtime=runtime) + symbols = _get_mutagen_symbols(runtime) + if not symbols: + return False + + audio = symbols.File(file_path) + if audio is None: + return False + if hasattr(audio, "clear_pictures"): + audio.clear_pictures() + if audio.tags is not None: + tag_count = len(audio.tags) + audio.tags.clear() + else: + audio.add_tags() + tag_count = 0 + _save_audio_file(audio, symbols) + if tag_count > 0: + logger_.info("[Tag Wipe] Stripped %s source tags from: %s", tag_count, os.path.basename(file_path)) + return True + except Exception as exc: + logger_.error("[Tag Wipe] Failed (non-fatal): %s", exc) + return False + + +def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool: + cfg = _get_config_manager(runtime) + logger_ = _get_logger(runtime) + if cfg.get("metadata_enhancement.enabled", True) is False: + logger_.warning("Metadata enhancement disabled in config.") + return True + + if album_info is None: + album_info = {} + + symbols = _get_mutagen_symbols(runtime) + if not symbols: + logger_.error("Mutagen is unavailable, cannot enhance metadata.") + return False + + file_lock = _get_file_lock(file_path) + with file_lock: + logger_.info("Enhancing metadata for: %s", os.path.basename(file_path)) + try: + _strip_all_non_audio_tags(file_path, runtime=runtime) + audio_file = symbols.File(file_path) + if audio_file is None: + logger_.error("Could not load audio file with Mutagen: %s", file_path) + return False + + if hasattr(audio_file, "clear_pictures"): + audio_file.clear_pictures() + + if audio_file.tags is not None: + if len(audio_file.tags) > 0: + tag_keys = list(audio_file.tags.keys())[:15] + logger_.info("Clearing %s existing tags: %s", len(audio_file.tags), ", ".join(str(k) for k in tag_keys)) + audio_file.tags.clear() + else: + audio_file.add_tags() + + _save_audio_file(audio_file, symbols) + + metadata = extract_source_metadata(context, artist, album_info, runtime=runtime) + if not metadata: + logger_.error("Could not extract source metadata, saving with cleared tags.") + _save_audio_file(audio_file, symbols) + return True + + track_num_str = f"{metadata.get('track_number', 1)}/{metadata.get('total_tracks', 1)}" + write_multi = cfg.get("metadata_enhancement.tags.write_multi_artist", False) + artists_list = metadata.get("_artists_list", []) + + if isinstance(audio_file.tags, symbols.ID3): + if metadata.get("title"): + audio_file.tags.add(symbols.TIT2(encoding=3, text=[metadata["title"]])) + if metadata.get("artist"): + audio_file.tags.add(symbols.TPE1(encoding=3, text=[metadata["artist"]])) + if write_multi and len(artists_list) > 1: + audio_file.tags.add(symbols.TPE1(encoding=3, text=artists_list)) + if metadata.get("album_artist"): + audio_file.tags.add(symbols.TPE2(encoding=3, text=[metadata["album_artist"]])) + if metadata.get("album"): + audio_file.tags.add(symbols.TALB(encoding=3, text=[metadata["album"]])) + if metadata.get("date"): + audio_file.tags.add(symbols.TDRC(encoding=3, text=[metadata["date"]])) + if metadata.get("genre"): + audio_file.tags.add(symbols.TCON(encoding=3, text=[metadata["genre"]])) + audio_file.tags.add(symbols.TRCK(encoding=3, text=[track_num_str])) + if metadata.get("disc_number"): + audio_file.tags.add(symbols.TPOS(encoding=3, text=[str(metadata["disc_number"])])) + elif _is_vorbis_like(audio_file, symbols): + if metadata.get("title"): + audio_file["title"] = [metadata["title"]] + if metadata.get("artist"): + audio_file["artist"] = [metadata["artist"]] + if write_multi and len(artists_list) > 1: + audio_file["artists"] = artists_list + if metadata.get("album_artist"): + audio_file["albumartist"] = [metadata["album_artist"]] + if metadata.get("album"): + audio_file["album"] = [metadata["album"]] + if metadata.get("date"): + audio_file["date"] = [metadata["date"]] + if metadata.get("genre"): + audio_file["genre"] = [metadata["genre"]] + audio_file["tracknumber"] = [track_num_str] + if metadata.get("disc_number"): + audio_file["discnumber"] = [str(metadata["disc_number"])] + elif isinstance(audio_file, symbols.MP4): + if metadata.get("title"): + audio_file["\xa9nam"] = [metadata["title"]] + if metadata.get("artist"): + audio_file["\xa9ART"] = artists_list if (write_multi and len(artists_list) > 1) else [metadata["artist"]] + if metadata.get("album_artist"): + audio_file["aART"] = [metadata["album_artist"]] + if metadata.get("album"): + audio_file["\xa9alb"] = [metadata["album"]] + if metadata.get("date"): + audio_file["\xa9day"] = [metadata["date"]] + if metadata.get("genre"): + audio_file["\xa9gen"] = [metadata["genre"]] + audio_file["trkn"] = [(metadata.get("track_number", 1), metadata.get("total_tracks", 1))] + if metadata.get("disc_number"): + audio_file["disk"] = [(metadata["disc_number"], 0)] + + embed_source_ids(audio_file, metadata, context, runtime=runtime) + + if album_info is not None and metadata.get("musicbrainz_release_id"): + album_info["musicbrainz_release_id"] = metadata["musicbrainz_release_id"] + + if cfg.get("metadata_enhancement.embed_album_art", True): + embed_album_art_metadata(audio_file, metadata, runtime=runtime) + + quality = context.get("_audio_quality", "") + if quality and cfg.get("metadata_enhancement.tags.quality_tag", True) is not False: + if isinstance(audio_file.tags, symbols.ID3): + audio_file.tags.add(symbols.TXXX(encoding=3, desc="QUALITY", text=[quality])) + elif _is_vorbis_like(audio_file, symbols): + audio_file["quality"] = [quality] + elif isinstance(audio_file, symbols.MP4): + audio_file["----:com.apple.iTunes:QUALITY"] = [symbols.MP4FreeForm(quality.encode("utf-8"))] + + _save_audio_file(audio_file, symbols) + + verified = _verify_metadata_written(file_path, runtime=runtime) + if verified: + logger_.info("Metadata enhanced successfully.") + else: + logger_.info("Metadata saved but verification found issues (see above).") + return True + except Exception as exc: + import traceback + + logger_.error("Error enhancing metadata for %s: %s", file_path, exc) + logger_.error("[Metadata Debug] Exception type: %s", type(exc).__name__) + logger_.info("[Metadata Debug] File exists: %s", os.path.exists(file_path)) + logger_.warning("[Metadata Debug] Artist: %s", artist.get("name", "MISSING") if artist else "None") + logger_.warning("[Metadata Debug] Album info: %s", album_info.get("album_name", "MISSING") if album_info else "None") + logger_.error("[Metadata Debug] Traceback:\n%s", traceback.format_exc()) + return False + + +def _normalize_album_cache_key(album_name: str) -> str: + result = _EDITION_PAREN_RE.sub("", album_name or "") + result = _EDITION_BARE_RE.sub("", result) + return result.lower().strip() diff --git a/core/metadata_service.py b/core/metadata_service.py index e2636d0f..861f0b08 100644 --- a/core/metadata_service.py +++ b/core/metadata_service.py @@ -719,6 +719,23 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti if not isinstance(images, list): images = list(images) if images else [] + artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) + if not artists and artist_name: + artists = [{'name': artist_name}] + + primary_artist = artists[0] if artists else {} + resolved_artist_name = ( + _extract_lookup_value(primary_artist, 'name', default='') + or artist_name + or _extract_lookup_value(album_data, 'artist_name', 'artist', default='') + or '' + ) + resolved_artist_id = str( + _extract_lookup_value(primary_artist, 'id', default='') + or _extract_lookup_value(album_data, 'artist_id', default='') + or '' + ).strip() + image_url = None if images: image_url = _extract_lookup_value(images[0], 'url') @@ -728,12 +745,15 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti return { 'id': _extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', 'release_id', default=album_id) or album_id, 'name': _extract_lookup_value(album_data, 'name', 'title', default=album_name or album_id) or album_name or album_id, + 'artist': resolved_artist_name or '', + 'artist_name': resolved_artist_name or '', + 'artist_id': resolved_artist_id, + 'artists': artists, 'image_url': image_url, 'images': images, 'release_date': _extract_lookup_value(album_data, 'release_date', default='') or '', 'album_type': _extract_lookup_value(album_data, 'album_type', default='album') or 'album', 'total_tracks': _extract_lookup_value(album_data, 'total_tracks', 'track_count', default=0) or 0, - 'artist_name': artist_name or _extract_lookup_value(album_data, 'artist_name', default='') or '', } @@ -754,6 +774,8 @@ def _build_album_track_entry(track_item: Any, album_info: Dict[str, Any], source 'external_urls': _extract_lookup_value(track_item, 'external_urls', default={}) or {}, 'uri': _extract_lookup_value(track_item, 'uri', default='') or '', 'album': album_info, + 'source': source, + 'provider': source, '_source': source, } @@ -767,6 +789,9 @@ def _build_album_tracks_payload( artist_name: str = '', ) -> Dict[str, Any]: album_info = _build_album_info(album_data, album_id, album_name=album_name, artist_name=artist_name) + album_info['source'] = source + album_info['_source'] = source + album_info['provider'] = source track_items = _extract_album_track_items(album_data, tracks_data) tracks = [_build_album_track_entry(track, album_info, source) for track in track_items] @@ -778,6 +803,369 @@ def _build_album_tracks_payload( } +def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]: + if not artists: + return [] + + if isinstance(artists, (str, bytes)): + artists = [artists] + elif isinstance(artists, dict): + artists = [artists] + else: + try: + artists = list(artists) + except TypeError: + artists = [artists] + + normalized: List[Dict[str, Any]] = [] + for artist in artists: + if isinstance(artist, dict): + name = _extract_lookup_value(artist, 'name', 'artist_name', 'title', default='') or '' + artist_id = _extract_lookup_value(artist, 'id', 'artist_id', default='') or '' + entry: Dict[str, Any] = {} + if name: + entry['name'] = str(name) + if artist_id: + entry['id'] = str(artist_id) + genres = _extract_lookup_value(artist, 'genres', default=None) + if genres is not None: + entry['genres'] = genres + if entry: + normalized.append(entry) + continue + + name = str(artist).strip() + if name: + normalized.append({'name': name}) + + return normalized + + +def _build_single_import_context_payload( + track_data: Any, + source: Optional[str], + source_priority: List[str], + requested_title: str = '', + requested_artist: str = '', +) -> Dict[str, Any]: + album_data = _extract_lookup_value(track_data, 'album', default=None) + + track_id = str(_extract_lookup_value(track_data, 'id', 'track_id', 'trackId', default='') or '') + track_name = _extract_lookup_value(track_data, 'name', 'title', 'trackName', default='') or requested_title or 'Unknown Track' + track_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'artists', default=[])) + if not track_artists and requested_artist: + track_artists = [{'name': requested_artist}] + + primary_track_artist = track_artists[0] if track_artists else {} + primary_artist_name = primary_track_artist.get('name') or requested_artist or 'Unknown Artist' + primary_artist_id = str(primary_track_artist.get('id', '') or _extract_lookup_value(track_data, 'artist_id', 'artistId', default='') or '') + + album_name = _extract_lookup_value(track_data, 'album_name', 'collectionName', default='') or '' + album_id = str(_extract_lookup_value(track_data, 'album_id', 'collectionId', 'albumId', default='') or '') + release_date = str(_extract_lookup_value(track_data, 'release_date', default='') or '') + album_type = str(_extract_lookup_value(track_data, 'album_type', default='album') or 'album') + total_tracks = int(_extract_lookup_value(track_data, 'total_tracks', 'track_count', default=0) or 0) + album_images: List[Dict[str, Any]] = [] + album_image_url = str(_extract_lookup_value(track_data, 'image_url', 'thumb_url', default='') or '') + album_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'album_artists', 'artists', default=[])) + + if isinstance(album_data, dict): + album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name + album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id) + release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date) + album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type) + total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks) + album_images = _extract_lookup_value(album_data, 'images', default=[]) or [] + if not album_image_url: + album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '') + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[])) + elif album_data: + album_name = album_name or str(album_data) + + if not album_artists and primary_artist_name: + album_artists = [{'name': primary_artist_name}] + + if not album_image_url and album_images: + album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '') + + track_info = { + 'id': track_id, + 'name': track_name, + 'track_number': int(_extract_lookup_value(track_data, 'track_number', 'trackNumber', default=1) or 1), + 'disc_number': int(_extract_lookup_value(track_data, 'disc_number', 'discNumber', default=1) or 1), + 'duration_ms': int(_extract_lookup_value(track_data, 'duration_ms', 'duration', 'trackTimeMillis', default=0) or 0), + 'artists': track_artists or [{'name': primary_artist_name}], + 'uri': str(_extract_lookup_value(track_data, 'uri', default='') or ''), + 'album': album_name, + 'album_id': album_id, + 'album_type': album_type, + 'release_date': release_date, + '_source': source or '', + } + + album_payload = { + 'id': album_id, + 'name': album_name, + 'release_date': release_date, + 'total_tracks': total_tracks or 1, + 'album_type': album_type, + 'image_url': album_image_url, + 'images': album_images, + 'artists': album_artists, + '_source': source or '', + } + + artist_payload = { + 'id': primary_artist_id, + 'name': primary_artist_name, + 'genres': [], + '_source': source or '', + } + + original_search = { + 'title': track_name, + 'artist': primary_artist_name, + 'album': album_name, + 'track_number': track_info['track_number'], + 'disc_number': track_info['disc_number'], + 'clean_title': track_name, + 'clean_album': album_name, + 'clean_artist': primary_artist_name, + 'artists': track_info['artists'], + 'duration_ms': track_info['duration_ms'], + 'id': track_id, + '_source': source or '', + } + + return { + 'success': bool(track_id or track_name != requested_title or album_name), + 'source': source, + 'source_priority': source_priority, + 'context': { + 'artist': artist_payload, + 'album': album_payload, + 'track_info': track_info, + 'original_search_result': original_search, + 'is_album_download': False, + 'has_clean_metadata': bool(track_id), + 'has_full_metadata': bool(track_id), + 'source': source, + 'source_priority': source_priority, + }, + } + + +def _build_single_import_fallback_context( + requested_title: str, + requested_artist: str, + source_priority: List[str], +) -> Dict[str, Any]: + artist_name = requested_artist or 'Unknown Artist' + title = requested_title or 'Unknown Track' + return { + 'success': False, + 'source': None, + 'source_priority': source_priority, + 'context': { + 'artist': { + 'id': '', + 'name': artist_name, + 'genres': [], + '_source': '', + }, + 'album': { + 'id': '', + 'name': '', + 'release_date': '', + 'total_tracks': 1, + 'album_type': 'album', + 'image_url': '', + 'images': [], + 'artists': [], + '_source': '', + }, + 'track_info': { + 'id': '', + 'name': title, + 'track_number': 1, + 'disc_number': 1, + 'duration_ms': 0, + 'artists': [{'name': artist_name}], + 'uri': '', + 'album': '', + 'album_id': '', + 'album_type': 'album', + 'release_date': '', + '_source': '', + }, + 'original_search_result': { + 'title': title, + 'artist': artist_name, + 'album': '', + 'track_number': 1, + 'disc_number': 1, + 'clean_title': title, + 'clean_album': '', + 'clean_artist': artist_name, + 'artists': [{'name': artist_name}], + 'duration_ms': 0, + 'id': '', + '_source': '', + }, + 'is_album_download': False, + 'has_clean_metadata': False, + 'has_full_metadata': False, + 'source': None, + 'source_priority': source_priority, + }, + } + + +def _build_track_search_query(source: str, title: str, artist: str) -> str: + base_query = " ".join(part for part in (title, artist) if part).strip() + if source == 'deezer' and title: + if artist: + return f'artist:"{artist}" track:"{title}"' + return f'track:"{title}"' + return base_query or title or artist + + +def _pick_best_track_match(search_results: List[Any], title: str, artist: str = '') -> Optional[Any]: + if not search_results: + return None + + target_title = str(title or '').strip().lower() + target_artist = str(artist or '').strip().lower() + + for candidate in search_results: + candidate_title = str(_extract_lookup_value(candidate, 'name', 'title', 'track_name', default='') or '').strip().lower() + if candidate_title != target_title: + continue + + if not target_artist: + return candidate + + candidate_artists = _normalize_context_artists(_extract_lookup_value(candidate, 'artists', default=[])) + candidate_artist_name = candidate_artists[0]['name'].strip().lower() if candidate_artists else '' + if candidate_artist_name == target_artist: + return candidate + + return search_results[0] + + +def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]: + if not client or not hasattr(client, 'search_tracks'): + return [] + + try: + kwargs = {'limit': limit} + if source == 'spotify': + kwargs['allow_fallback'] = False + return client.search_tracks(query, **kwargs) or [] + except Exception as exc: + logger.debug("Could not search %s for %s: %s", source, query, exc) + return [] + + +def get_single_track_import_context( + title: str, + artist: str = '', + override_id: Optional[str] = None, + override_source: str = 'spotify', + source_override: Optional[str] = None, +) -> Dict[str, Any]: + """Build an import context for singles using source-priority metadata lookup.""" + options = MetadataLookupOptions(source_override=source_override, allow_fallback=True) + source_priority = _get_source_chain_for_lookup(options) + title = (title or '').strip() + artist = (artist or '').strip() + + if override_id: + chosen_source = (override_source or 'spotify').strip().lower() or 'spotify' + client = get_client_for_source(chosen_source) + if client and hasattr(client, 'get_track_details'): + try: + track_data = client.get_track_details(str(override_id)) + if track_data: + payload = _build_single_import_context_payload( + track_data, + chosen_source, + source_priority, + requested_title=title, + requested_artist=artist, + ) + if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): + try: + artist_details = client.get_artist(payload['context']['artist']['id']) + if artist_details: + payload['context']['artist']['genres'] = _extract_lookup_value( + artist_details, + 'genres', + default=[], + ) or [] + except Exception: + pass + return payload + except Exception as exc: + logger.debug("Override track lookup failed on %s for %s: %s", chosen_source, override_id, exc) + + for source in source_priority: + client = get_client_for_source(source) + if not client: + continue + + search_query = _build_track_search_query(source, title, artist) + if not search_query: + continue + + search_results = _search_tracks_for_source(source, client, search_query, limit=5) + if not search_results and search_query != title: + search_results = _search_tracks_for_source(source, client, title, limit=5) + if not search_results and artist and search_query != artist: + search_results = _search_tracks_for_source(source, client, artist, limit=5) + + if not search_results: + continue + + best_match = _pick_best_track_match(search_results, title or search_query, artist) + if not best_match: + continue + + resolved_track_id = str(_extract_lookup_value(best_match, 'id', 'track_id', 'trackId', default='') or '') + resolved_data = best_match + if resolved_track_id and hasattr(client, 'get_track_details'): + try: + detailed = client.get_track_details(resolved_track_id) + if detailed: + resolved_data = detailed + except Exception as exc: + logger.debug("Track detail lookup failed on %s for %s: %s", source, resolved_track_id, exc) + + payload = _build_single_import_context_payload( + resolved_data, + source, + source_priority, + requested_title=title, + requested_artist=artist, + ) + if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'): + try: + artist_details = client.get_artist(payload['context']['artist']['id']) + if artist_details: + payload['context']['artist']['genres'] = _extract_lookup_value( + artist_details, + 'genres', + default=[], + ) or [] + except Exception: + pass + return payload + + return _build_single_import_fallback_context(title, artist, source_priority) + + def resolve_album_reference( album_id: str, preferred_source: Optional[str] = None, diff --git a/core/soulseek_client.py b/core/soulseek_client.py index 9b64ada6..e859fd4e 100644 --- a/core/soulseek_client.py +++ b/core/soulseek_client.py @@ -8,6 +8,7 @@ import time from pathlib import Path from utils.logging_config import get_logger from config.settings import config_manager +from core.import_filename import parse_filename_metadata logger = get_logger("soulseek_client") @@ -87,60 +88,17 @@ class TrackResult(SearchResult): def _parse_filename_metadata(self): """Extract artist, title, album from filename patterns""" - import re - import os - - # Get just the filename without extension and path - base_name = os.path.splitext(os.path.basename(self.filename))[0] - - # Common patterns for track naming - patterns = [ - r'^(\d+)\s*[-\.]\s*(.+?)\s*[-–]\s*(.+)$', # "01 - Artist - Title" or "01. Artist - Title" - r'^(.+?)\s*[-–]\s*(.+)$', # "Artist - Title" - r'^(\d+)\s*[-\.]\s*(.+)$', # "01 - Title" or "01. Title" - ] - - for pattern in patterns: - match = re.match(pattern, base_name) - if match: - groups = match.groups() - if len(groups) == 3: # Track number, artist, title - try: - self.track_number = int(groups[0]) - self.artist = self.artist or groups[1].strip() - self.title = self.title or groups[2].strip() - except ValueError: - # First group might not be a number - self.artist = self.artist or groups[0].strip() - self.title = self.title or f"{groups[1]} - {groups[2]}".strip() - elif len(groups) == 2: - if groups[0].isdigit(): # Track number and title - try: - self.track_number = int(groups[0]) - self.title = self.title or groups[1].strip() - except ValueError: - pass - else: # Artist and title - self.artist = self.artist or groups[0].strip() - self.title = self.title or groups[1].strip() - break - - # Fallback: use filename as title if nothing was extracted - if not self.title: - self.title = base_name - - # Try to extract album from directory path - if not self.album and '/' in self.filename: - path_parts = self.filename.split('/') - if len(path_parts) >= 2: - # Look for album-like directory names - for part in reversed(path_parts[:-1]): # Exclude filename - if part and not part.startswith('@'): # Skip system directories - # Clean up common patterns - cleaned = re.sub(r'^\d+\s*[-\.]\s*', '', part) # Remove leading numbers - if len(cleaned) > 3: # Must be substantial - self.album = cleaned - break + parsed = parse_filename_metadata(self.filename) + if not self.artist and parsed.get("artist"): + self.artist = parsed["artist"] + if not self.title and parsed.get("title"): + self.title = parsed["title"] + if not self.album and parsed.get("album"): + self.album = parsed["album"] + if self.track_number is None: + track_number = parsed.get("track_number") + if track_number is not None: + self.track_number = track_number @dataclass class AlbumResult: @@ -1829,4 +1787,4 @@ class SoulseekClient: def __del__(self): # No persistent session to clean up - pass \ No newline at end of file + pass diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index a0cdd66b..c1d9aa86 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -2955,9 +2955,6 @@ class WatchlistScanner: } track_data = { - 'spotify_track_id': track['id'], - 'spotify_album_id': album_data['id'], - 'spotify_artist_id': artist.spotify_artist_id, 'track_name': track['name'], 'artist_name': artist.artist_name, 'album_name': album_data.get('name', 'Unknown Album'), diff --git a/tests/test_downloads_pagination.py b/tests/test_downloads_pagination.py index af2c7b9a..f85dd96d 100644 --- a/tests/test_downloads_pagination.py +++ b/tests/test_downloads_pagination.py @@ -7,7 +7,6 @@ and `status` query params and includes a `total` count. """ import sys -import threading import types from unittest.mock import patch @@ -45,6 +44,7 @@ _install_flask_limiter_stub() from flask import Flask, Blueprint # noqa: E402 from api import downloads as downloads_mod # noqa: E402 +import core.import_runtime_state as runtime_state # noqa: E402 def _make_task(status="downloading", when=None): @@ -63,11 +63,10 @@ def _make_task(status="downloading", when=None): def _make_app_with_tasks(tasks_dict): """Create a minimal Flask app with the downloads blueprint mounted and - a fake `web_server` module exposing the given download_tasks dict.""" - fake_ws = types.ModuleType("web_server") - fake_ws.download_tasks = tasks_dict - fake_ws.tasks_lock = threading.RLock() - sys.modules["web_server"] = fake_ws + the shared import runtime state populated with the given download_tasks dict.""" + original_tasks = dict(runtime_state.download_tasks) + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(tasks_dict) # Bypass API key auth for tests. def _passthrough(f): @@ -83,6 +82,7 @@ def _make_app_with_tasks(tasks_dict): downloads_mod.register_routes(bp) app.register_blueprint(bp) + app._original_download_tasks = original_tasks return app @@ -96,8 +96,12 @@ def client(): for i in range(25) } app = _make_app_with_tasks(tasks) - with app.test_client() as c: - yield c + try: + with app.test_client() as c: + yield c + finally: + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(app._original_download_tasks) def test_default_limit_applied(client): diff --git a/tests/test_import_album.py b/tests/test_import_album.py new file mode 100644 index 00000000..81ab6fb6 --- /dev/null +++ b/tests/test_import_album.py @@ -0,0 +1,186 @@ +import core.import_album as import_album + + +class _FakeEngine: + def normalize_string(self, text): + return str(text or "").strip().lower() + + def similarity_score(self, left, right): + return 1.0 if left == right else 0.0 + + +def test_resolve_album_artist_context_uses_provider_genres(monkeypatch): + class _FakeClient: + def get_artist(self, artist_id): + assert artist_id == "artist-1" + return {"genres": ["rock", "indie"]} + + monkeypatch.setattr(import_album, "get_client_for_source", lambda source: _FakeClient() if source == "spotify" else None) + + context = import_album.resolve_album_artist_context( + { + "id": "album-1", + "name": "Album One", + "artist_id": "artist-1", + "artists": [{"name": "Artist One", "id": "artist-1"}], + }, + source="spotify", + ) + + assert context == { + "id": "artist-1", + "name": "Artist One", + "genres": ["rock", "indie"], + "source": "spotify", + } + + +def test_build_album_import_context_is_neutral(): + context = import_album.build_album_import_context( + { + "id": "album-1", + "name": "Album One", + "artist": "Artist One", + "artist_id": "artist-1", + "artists": [{"name": "Artist One", "id": "artist-1"}], + "release_date": "2024-01-01", + "total_tracks": 12, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + "source": "deezer", + }, + { + "id": "track-1", + "name": "Song One", + "track_number": 3, + "disc_number": 2, + "duration_ms": 180000, + "artists": [{"name": "Artist One"}], + "uri": "deezer:track:track-1", + "album": "Album One", + "source": "deezer", + }, + artist_context={ + "id": "artist-1", + "name": "Artist One", + "genres": ["rock"], + "source": "deezer", + }, + total_discs=2, + source="deezer", + ) + + assert context["artist"]["name"] == "Artist One" + assert context["artist"]["genres"] == ["rock"] + assert context["album"]["name"] == "Album One" + assert context["album"]["total_discs"] == 2 + assert context["track_info"]["name"] == "Song One" + assert context["track_info"]["track_number"] == 3 + assert context["original_search_result"]["clean_title"] == "Song One" + assert context["source"] == "deezer" + assert "spotify_artist" not in context + assert "spotify_album" not in context + + +def test_build_album_import_match_payload_uses_generic_track_keys(monkeypatch, tmp_path): + staging_root = tmp_path / "Staging" + staging_root.mkdir() + (staging_root / "Song One.flac").write_text("fake") + + monkeypatch.setattr(import_album, "get_staging_path", lambda: str(staging_root)) + monkeypatch.setattr(import_album, "_get_matching_engine", lambda: _FakeEngine()) + monkeypatch.setattr( + import_album, + "read_staging_file_metadata", + lambda file_path, filename=None: { + "title": "Song One", + "artist": "Artist One", + "albumartist": "Artist One", + "album": "Album One", + "track_number": 1, + "disc_number": 1, + }, + ) + monkeypatch.setattr( + import_album, + "get_artist_album_tracks", + lambda album_id, artist_name="", album_name="", source=None: { + "success": True, + "album": { + "id": album_id, + "name": "Album One", + "artist": "Artist One", + "artist_name": "Artist One", + "artist_id": "artist-1", + "artists": [{"name": "Artist One", "id": "artist-1"}], + "release_date": "2024-01-01", + "total_tracks": 1, + "total_discs": 1, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + "images": [{"url": "https://img.example/album.jpg"}], + "source": "spotify", + }, + "tracks": [ + { + "id": "track-1", + "name": "Song One", + "track_number": 1, + "disc_number": 1, + "duration_ms": 180000, + "artists": [{"name": "Artist One"}], + "uri": "spotify:track:track-1", + "album": { + "id": album_id, + "name": "Album One", + "artist": "Artist One", + }, + "source": "spotify", + } + ], + "source": "spotify", + "source_priority": ["spotify"], + "resolved_album_id": album_id, + }, + ) + + result = import_album.build_album_import_match_payload( + "album-1", + album_name="Album One", + album_artist="Artist One", + source="spotify", + ) + + assert result["success"] is True + assert result["album"]["artist"] == "Artist One" + assert result["source"] == "spotify" + assert result["matches"] == [ + { + "track": { + "id": "track-1", + "name": "Song One", + "track_number": 1, + "disc_number": 1, + "duration_ms": 180000, + "artists": [{"name": "Artist One"}], + "uri": "spotify:track:track-1", + "album": { + "id": "album-1", + "name": "Album One", + "artist": "Artist One", + }, + "source": "spotify", + }, + "staging_file": { + "filename": "Song One.flac", + "full_path": str(staging_root / "Song One.flac"), + "title": "Song One", + "artist": "Artist One", + "album": "Album One", + "albumartist": "Artist One", + "track_number": 1, + "disc_number": 1, + }, + "confidence": 1.0, + } + ] diff --git a/tests/test_import_context.py b/tests/test_import_context.py new file mode 100644 index 00000000..f91acf19 --- /dev/null +++ b/tests/test_import_context.py @@ -0,0 +1,202 @@ +import pytest + +from core.import_context import ( + build_import_album_info, + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_has_clean_metadata, + get_import_has_full_metadata, + get_import_source, + get_import_source_ids, + get_library_source_id_columns, + get_source_tag_names, + normalize_import_context, +) + + +def test_normalize_import_context_promotes_neutral_fields_without_legacy_aliases(): + context = { + "source": "deezer", + "spotify_artist": {"name": "Artist One", "id": "artist-1"}, + "spotify_album": { + "name": "Album One", + "id": "album-1", + "release_date": "2024-01-01", + "total_tracks": 12, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + }, + "track_info": { + "name": "Song One", + "id": "track-1", + "track_number": 3, + "disc_number": 2, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "spotify_clean_title": "Song One", + "spotify_clean_album": "Album One", + "spotify_clean_artist": "Artist One", + }, + "has_clean_spotify_data": True, + "has_full_spotify_metadata": False, + } + + normalized = normalize_import_context(context) + + assert normalized["artist"]["name"] == "Artist One" + assert normalized["album"]["name"] == "Album One" + assert "spotify_artist" not in normalized + assert "spotify_album" not in normalized + assert normalized["original_search_result"]["clean_title"] == "Song One" + assert normalized["original_search_result"]["clean_album"] == "Album One" + assert normalized["original_search_result"]["clean_artist"] == "Artist One" + assert "spotify_clean_title" not in normalized["original_search_result"] + assert "spotify_clean_album" not in normalized["original_search_result"] + assert "spotify_clean_artist" not in normalized["original_search_result"] + assert get_import_clean_title(normalized) == "Song One" + assert get_import_clean_album(normalized) == "Album One" + assert get_import_clean_artist(normalized) == "Artist One" + assert get_import_source(normalized) == "deezer" + assert get_import_has_clean_metadata(normalized) is True + assert get_import_has_full_metadata(normalized) is False + + +def test_neutral_import_context_helpers_work_without_legacy_aliases(): + context = { + "source": "deezer", + "artist": {"name": "Artist One", "id": "artist-1"}, + "album": { + "name": "Album One", + "id": "album-1", + "release_date": "2024-01-01", + "total_tracks": 12, + "album_type": "album", + "image_url": "https://img.example/album.jpg", + }, + "track_info": { + "name": "Song One", + "id": "track-1", + "track_number": 3, + "disc_number": 2, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "title": "Song One", + "artist": "Artist One", + "album": "Album One", + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + }, + "has_clean_metadata": True, + "has_full_metadata": True, + } + + assert get_import_clean_title(context) == "Song One" + assert get_import_clean_album(context) == "Album One" + assert get_import_clean_artist(context) == "Artist One" + assert get_import_source(context) == "deezer" + + normalized = normalize_import_context(context) + assert normalized["artist"]["name"] == "Artist One" + assert normalized["album"]["name"] == "Album One" + assert normalized["original_search_result"]["clean_title"] == "Song One" + assert "spotify_artist" not in normalized + assert "spotify_album" not in normalized + assert get_import_has_clean_metadata(normalized) is True + assert get_import_has_full_metadata(normalized) is True + + +@pytest.mark.parametrize( + "source,expected_tags,expected_columns", + [ + ( + "spotify", + {"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"}, + {"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"}, + ), + ( + "itunes", + {"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"}, + {"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"}, + ), + ( + "deezer", + {"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None}, + {"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"}, + ), + ( + "hydrabase", + {"track": None, "artist": None, "album": None}, + {"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"}, + ), + ( + "discogs", + {"track": None, "artist": None, "album": None}, + {"artist": "discogs_id", "album": "discogs_id", "track": None}, + ), + ], +) +def test_source_tag_and_library_column_mappings(source, expected_tags, expected_columns): + assert get_source_tag_names(source) == expected_tags + assert get_library_source_id_columns(source) == expected_columns + + +def test_get_import_source_ids_prefers_nested_source_specific_ids(): + context = normalize_import_context( + { + "source": "deezer", + "artist": {"deezer_id": "deezer-artist-1"}, + "album": {"deezer_id": "deezer-album-1"}, + "track_info": {"deezer_id": "deezer-track-1"}, + "original_search_result": {"source_artist_id": "deezer-artist-1"}, + } + ) + + assert get_import_source_ids(context) == { + "track_id": "deezer-track-1", + "artist_id": "deezer-artist-1", + "album_id": "deezer-album-1", + } + + +def test_build_import_album_info_uses_normalized_album_context(): + context = normalize_import_context( + { + "source": "deezer", + "artist": {"name": "Artist One"}, + "album": { + "name": "Album One", + "image_url": "https://img.example/album.jpg", + "release_date": "2024-05-01", + "total_tracks": 8, + "album_type": "album", + }, + "track_info": { + "name": "Song One", + "track_number": 4, + "disc_number": 2, + "duration_ms": 240000, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "title": "Song One", + "album": "Album One", + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + }, + } + ) + + album_info = build_import_album_info(context) + + assert album_info["is_album"] is True + assert album_info["album_name"] == "Album One" + assert album_info["track_number"] == 4 + assert album_info["disc_number"] == 2 + assert album_info["clean_track_name"] == "Song One" + assert album_info["album_image_url"] == "https://img.example/album.jpg" + assert album_info["source"] == "deezer" diff --git a/tests/test_import_file_ops.py b/tests/test_import_file_ops.py new file mode 100644 index 00000000..e9d72901 --- /dev/null +++ b/tests/test_import_file_ops.py @@ -0,0 +1,92 @@ +import sys +import types + +from core.import_file_ops import ( + cleanup_empty_directories, + extract_track_number_from_filename, + safe_move_file, + read_staging_file_metadata, +) + + +def test_extract_track_number_from_filename_handles_common_patterns(): + assert extract_track_number_from_filename("01 - Song.mp3") == 1 + assert extract_track_number_from_filename("1-03 - Song.mp3") == 3 + assert extract_track_number_from_filename("Artist - Song.mp3") == 1 + + +def test_safe_move_file_replaces_existing_destination(tmp_path): + src = tmp_path / "source.flac" + dst_dir = tmp_path / "dest" + dst_dir.mkdir() + dst = dst_dir / "track.flac" + + src.write_text("new") + dst.write_text("old") + + safe_move_file(src, dst) + + assert not src.exists() + assert dst.read_text() == "new" + + +def test_cleanup_empty_directories_removes_nested_empty_paths(tmp_path): + download_root = tmp_path / "downloads" + nested_dir = download_root / "Artist" / "Album" + nested_dir.mkdir(parents=True) + moved_file_path = nested_dir / "track.flac" + + cleanup_empty_directories(str(download_root), str(moved_file_path)) + + assert not nested_dir.exists() + assert not (download_root / "Artist").exists() + assert download_root.exists() + + +def test_read_staging_file_metadata_reads_tags(monkeypatch, tmp_path): + file_path = tmp_path / "Song One.flac" + file_path.write_text("fake") + + class DummyTags: + def __init__(self): + self.values = { + "title": ["Song One"], + "artist": ["Artist One"], + "albumartist": ["Album Artist"], + "album": ["Album One"], + "tracknumber": ["03/12"], + "discnumber": ["2/3"], + } + + def get(self, key, default=None): + return self.values.get(key, default) + + fake_mutagen = types.ModuleType("mutagen") + fake_mutagen.File = lambda path, easy=True: DummyTags() + monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen) + + metadata = read_staging_file_metadata(str(file_path), file_path.name) + + assert metadata == { + "title": "Song One", + "artist": "Artist One", + "albumartist": "Album Artist", + "album": "Album One", + "track_number": 3, + "disc_number": 2, + } + + +def test_read_staging_file_metadata_falls_back_to_filename_track_number(monkeypatch, tmp_path): + file_path = tmp_path / "07 - Song Two.flac" + file_path.write_text("fake") + + fake_mutagen = types.ModuleType("mutagen") + fake_mutagen.File = lambda path, easy=True: None + monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen) + + metadata = read_staging_file_metadata(str(file_path), file_path.name) + + assert metadata["title"] == "07 - Song Two" + assert metadata["track_number"] == 7 + assert metadata["disc_number"] == 1 diff --git a/tests/test_import_filename.py b/tests/test_import_filename.py new file mode 100644 index 00000000..a5b32ff2 --- /dev/null +++ b/tests/test_import_filename.py @@ -0,0 +1,33 @@ +import pytest + +from core.import_filename import parse_filename_metadata + + +@pytest.mark.parametrize( + "filename,expected", + [ + ( + "01 - Artist One - Title One.mp3", + {"artist": "Artist One", "title": "Title One", "album": "", "track_number": 1}, + ), + ( + "Artist Two - Title Two.flac", + {"artist": "Artist Two", "title": "Title Two", "album": "", "track_number": None}, + ), + ( + r"Artist Three\Album Three\03 - Title Three.ogg", + {"artist": "", "title": "Title Three", "album": "Album Three", "track_number": 3}, + ), + ( + "Loose Song.wav", + {"artist": "", "title": "Loose Song", "album": "", "track_number": None}, + ), + ], +) +def test_parse_filename_metadata_handles_common_patterns(filename, expected): + parsed = parse_filename_metadata(filename) + + assert parsed["artist"] == expected["artist"] + assert parsed["title"] == expected["title"] + assert parsed["album"] == expected["album"] + assert parsed["track_number"] == expected["track_number"] diff --git a/tests/test_import_guards.py b/tests/test_import_guards.py new file mode 100644 index 00000000..a7b95372 --- /dev/null +++ b/tests/test_import_guards.py @@ -0,0 +1,47 @@ +from types import SimpleNamespace + +from core import import_guards as guards + + +class _FakeDB: + def __init__(self, quality_profile): + self._quality_profile = quality_profile + + def get_quality_profile(self): + return self._quality_profile + + +def test_check_flac_bit_depth_rejects_strict_mismatch(monkeypatch): + monkeypatch.setattr( + guards, + "MusicDatabase", + lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": False}}}), + ) + monkeypatch.setattr( + guards, + "_get_config_manager", + lambda: SimpleNamespace(get=lambda _key, default=None: False), + ) + + context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}} + + assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) == ( + "FLAC bit depth mismatch: file is 24-bit, preference is 16-bit" + ) + + +def test_check_flac_bit_depth_allows_fallback_when_enabled(monkeypatch): + monkeypatch.setattr( + guards, + "MusicDatabase", + lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": True}}}), + ) + monkeypatch.setattr( + guards, + "_get_config_manager", + lambda: SimpleNamespace(get=lambda _key, default=None: False), + ) + + context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}} + + assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) is None diff --git a/tests/test_import_paths.py b/tests/test_import_paths.py new file mode 100644 index 00000000..f9a39fab --- /dev/null +++ b/tests/test_import_paths.py @@ -0,0 +1,161 @@ +import core.import_paths as import_paths + + +class _Config: + def __init__(self, values): + self._values = values + + def get(self, key, default=None): + return self._values.get(key, default) + + +def test_sanitize_filename_replaces_illegal_characters(): + assert import_paths.sanitize_filename("AC/DC: Song?") == "AC_DC_ Song_" + assert import_paths.sanitize_filename("AUX.txt").startswith("_") + + +def test_build_simple_download_destination_uses_album_folder(monkeypatch, tmp_path): + config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")}) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + + destination, album_name, filename = import_paths.build_simple_download_destination( + { + "search_result": { + "filename": "Album Folder/source.flac", + "album": "Album Folder", + } + }, + str(tmp_path / "source.flac"), + ) + + assert destination == tmp_path / "Transfer" / "Album Folder" / "source.flac" + assert album_name == "Album Folder" + assert filename == "source.flac" + assert destination.parent.exists() + + +def test_build_simple_download_destination_falls_back_to_transfer_root(monkeypatch, tmp_path): + config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")}) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + + destination, album_name, filename = import_paths.build_simple_download_destination( + { + "search_result": { + "filename": "source.flac", + "album": "Unknown Album", + } + }, + str(tmp_path / "source.flac"), + ) + + assert destination == tmp_path / "Transfer" / "source.flac" + assert album_name == "" + assert filename == "source.flac" + assert destination.parent.exists() + + +def test_get_file_path_from_template_raw_handles_quality_and_disc_placeholders(monkeypatch): + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config({})) + + folder_path, filename = import_paths.get_file_path_from_template_raw( + "$artist/$album/$discnum - $title [$quality]", + { + "artist": "Artist One", + "album": "Album One", + "title": "Song One", + "quality": "FLAC 16bit", + "disc_number": 3, + }, + ) + + assert folder_path == "Artist One/Album One" + assert filename == "3 - Song One [FLAC 16bit]" + + +def test_resolve_album_group_upgrades_standard_to_deluxe(): + import_paths._album_name_cache.clear() + import_paths._album_editions.clear() + + artist_context = {"name": "Cache Artist"} + standard_album = {"album_name": "Cache Album"} + deluxe_album = {"album_name": "Cache Album (Deluxe Edition)"} + + assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album" + assert import_paths.resolve_album_group(artist_context, deluxe_album) == "Cache Album (Deluxe Edition)" + assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album (Deluxe Edition)" + + +def test_build_final_path_for_track_uses_template_and_disc_folder(monkeypatch, tmp_path): + config = _Config( + { + "soulseek.transfer_path": str(tmp_path / "Transfer"), + "file_organization.enabled": True, + "file_organization.templates": { + "album_path": "$albumartist/$albumartist - $album/$track - $title", + "single_path": "$artist/$artist - $title", + }, + "file_organization.collab_artist_mode": "first", + "file_organization.disc_label": "Disc", + } + ) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config) + + calls = [] + + def fake_get_album_tracks_for_source(source, album_id): + calls.append((source, album_id)) + return { + "items": [ + {"disc_number": 1}, + {"disc_number": 2}, + ] + } + + monkeypatch.setattr(import_paths, "_get_album_tracks_for_source", fake_get_album_tracks_for_source) + + context = { + "artist": {"name": "Artist One"}, + "album": { + "name": "Album One", + "id": "album-1", + "release_date": "2026-01-01", + "total_tracks": 12, + "album_type": "album", + "artists": [{"name": "Artist One"}], + }, + "track_info": { + "name": "Song One", + "id": "track-1", + "track_number": 4, + "disc_number": 1, + "artists": [{"name": "Artist One"}], + }, + "original_search_result": { + "title": "Song One", + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + "artists": [{"name": "Artist One"}], + }, + "source": "deezer", + "is_album_download": False, + } + + final_path, created = import_paths.build_final_path_for_track( + context, + {"name": "Artist One"}, + { + "is_album": True, + "album_name": "Album One", + "track_number": 4, + "disc_number": 1, + }, + ".flac", + ) + + assert created is True + assert calls == [("deezer", "album-1")] + assert final_path == str( + tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1" / "04 - Song One.flac" + ) + assert (tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1").is_dir() diff --git a/tests/test_import_pipeline.py b/tests/test_import_pipeline.py new file mode 100644 index 00000000..768a8788 --- /dev/null +++ b/tests/test_import_pipeline.py @@ -0,0 +1,128 @@ +import logging +import sys +import types + +import core.import_pipeline as import_pipeline +import core.import_paths as import_paths +import core.import_runtime_state as runtime_state + + +class _Config: + def __init__(self, transfer_path): + self.transfer_path = transfer_path + + def get(self, key, default=None): + if key == "soulseek.transfer_path": + return self.transfer_path + if key in {"post_processing.replaygain_enabled", "lossy_copy.enabled", "lossy_copy.delete_original", "import.replace_lower_quality"}: + return False + return default + + +class _FakeAcoustidVerifier: + def quick_check_available(self): + return False, "disabled" + + +class _ImmediateThread: + def __init__(self, target=None, daemon=None): + self._target = target + + def start(self): + if self._target: + self._target() + + +def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch): + transfer_root = tmp_path / "Transfer" + transfer_root.mkdir() + source_path = tmp_path / "source.flac" + source_path.write_bytes(b"audio") + + context_key = "ctx-1" + task_id = "task-1" + batch_id = "batch-1" + context = { + "search_result": { + "is_simple_download": True, + "filename": "Album Folder/source.flac", + "album": "Album Folder", + }, + "track_info": {}, + "original_search_result": {}, + "is_album_download": False, + "task_id": task_id, + "batch_id": batch_id, + } + + mark_calls = [] + completion_calls = [] + scan_calls = [] + activity_calls = [] + + original_matched_context = dict(runtime_state.matched_downloads_context) + original_download_tasks = dict(runtime_state.download_tasks) + original_download_batches = dict(runtime_state.download_batches) + original_processed_ids = set(runtime_state._processed_download_ids) + original_post_process_locks = dict(runtime_state._post_process_locks) + + runtime_state.matched_downloads_context.clear() + runtime_state.download_tasks.clear() + runtime_state.download_batches.clear() + runtime_state._processed_download_ids.clear() + runtime_state._post_process_locks.clear() + + runtime = types.SimpleNamespace( + automation_engine=None, + on_download_completed=lambda batch, task, success: completion_calls.append((batch, task, success)), + web_scan_manager=types.SimpleNamespace(request_scan=lambda reason: scan_calls.append(reason)), + repair_worker=None, + ) + + fake_acoustid = types.ModuleType("core.acoustid_verification") + fake_acoustid.AcoustIDVerification = _FakeAcoustidVerifier + fake_acoustid.VerificationResult = types.SimpleNamespace(FAIL="FAIL") + + monkeypatch.setitem(sys.modules, "core.acoustid_verification", fake_acoustid) + monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config(str(transfer_root))) + monkeypatch.setattr(import_pipeline, "add_activity_item", lambda *args, **kwargs: activity_calls.append((args, kwargs))) + monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "record_library_history_download", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "record_download_provenance", lambda *args, **kwargs: None) + monkeypatch.setattr(import_pipeline, "_mark_task_completed", lambda task, track_info: mark_calls.append((task, track_info))) + monkeypatch.setattr(import_pipeline.threading, "Thread", _ImmediateThread) + + runtime_state.matched_downloads_context[context_key] = context + runtime_state.download_tasks[task_id] = {"track_info": {}, "status": "running"} + + try: + import_pipeline.post_process_matched_download_with_verification( + context_key, + context, + str(source_path), + task_id, + batch_id, + runtime, + ) + + expected_path = transfer_root / "Album Folder" / "source.flac" + assert expected_path.exists() + assert not source_path.exists() + assert context["_simple_download_completed"] is True + assert context["_final_path"] == str(expected_path) + assert mark_calls == [(task_id, {})] + assert completion_calls == [(batch_id, task_id, True)] + assert context_key not in runtime_state.matched_downloads_context + assert scan_calls == ["Simple download completed"] + assert activity_calls + finally: + runtime_state.matched_downloads_context.clear() + runtime_state.matched_downloads_context.update(original_matched_context) + runtime_state.download_tasks.clear() + runtime_state.download_tasks.update(original_download_tasks) + runtime_state.download_batches.clear() + runtime_state.download_batches.update(original_download_batches) + runtime_state._processed_download_ids.clear() + runtime_state._processed_download_ids.update(original_processed_ids) + runtime_state._post_process_locks.clear() + runtime_state._post_process_locks.update(original_post_process_locks) diff --git a/tests/test_import_side_effects.py b/tests/test_import_side_effects.py new file mode 100644 index 00000000..510a9ab0 --- /dev/null +++ b/tests/test_import_side_effects.py @@ -0,0 +1,144 @@ +import sqlite3 +from types import SimpleNamespace + +from core import import_side_effects as side_effects + + +class _FakeDB: + def __init__(self, conn): + self._conn = conn + + def _get_connection(self): + return self._conn + + +def _make_soulsync_db(): + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.execute( + """ + CREATE TABLE artists ( + id TEXT PRIMARY KEY, + name TEXT, + genres TEXT, + thumb_url TEXT, + server_source TEXT, + created_at TEXT, + updated_at TEXT, + spotify_artist_id TEXT + ) + """ + ) + conn.execute( + """ + CREATE TABLE albums ( + id TEXT PRIMARY KEY, + artist_id TEXT, + title TEXT, + year INTEGER, + thumb_url TEXT, + genres TEXT, + track_count INTEGER, + duration INTEGER, + server_source TEXT, + created_at TEXT, + updated_at TEXT, + spotify_album_id TEXT + ) + """ + ) + conn.execute( + """ + CREATE TABLE tracks ( + id TEXT PRIMARY KEY, + album_id TEXT, + artist_id TEXT, + title TEXT, + track_number INTEGER, + duration INTEGER, + file_path TEXT, + bitrate INTEGER, + track_artist TEXT, + server_source TEXT, + created_at TEXT, + updated_at TEXT, + spotify_track_id TEXT + ) + """ + ) + return conn + + +def test_record_soulsync_library_entry_writes_artist_album_and_track(tmp_path, monkeypatch): + conn = _make_soulsync_db() + fake_db = _FakeDB(conn) + final_path = tmp_path / "track.flac" + final_path.write_bytes(b"audio") + + monkeypatch.setattr(side_effects, "get_database", lambda: fake_db) + monkeypatch.setattr( + side_effects, + "_get_config_manager", + lambda: SimpleNamespace(get_active_media_server=lambda: "soulsync"), + ) + + import core.genre_filter as genre_filter + + monkeypatch.setattr(genre_filter, "filter_genres", lambda genres, _cfg: [genre.upper() for genre in genres]) + + context = { + "source": "spotify", + "artist": {"id": "sp-artist", "name": "Artist One"}, + "album": { + "id": "sp-album", + "name": "Album One", + "release_date": "2024-02-03", + "total_tracks": 12, + "image_url": "https://img.example/album.jpg", + }, + "track_info": { + "id": "sp-track", + "name": "Song One", + "track_number": 7, + "duration_ms": 210000, + "artists": [{"name": "Guest Artist"}], + "_source": "spotify", + }, + "original_search_result": { + "title": "Song One", + "artists": [{"name": "Guest Artist"}], + "_source": "spotify", + }, + "_final_processed_path": str(final_path), + } + + artist_context = {"name": "Artist One", "genres": ["rock", "indie"]} + album_info = {"is_album": True, "album_name": "Album One", "track_number": 7} + + side_effects.record_soulsync_library_entry(context, artist_context, album_info) + + artist_row = conn.execute("SELECT * FROM artists").fetchone() + album_row = conn.execute("SELECT * FROM albums").fetchone() + track_row = conn.execute("SELECT * FROM tracks").fetchone() + + assert artist_row["name"] == "Artist One" + assert artist_row["server_source"] == "soulsync" + assert artist_row["spotify_artist_id"] == "sp-artist" + assert artist_row["genres"] == '["ROCK", "INDIE"]' + + assert album_row["title"] == "Album One" + assert album_row["server_source"] == "soulsync" + assert album_row["spotify_album_id"] == "sp-album" + assert album_row["year"] == 2024 + assert album_row["track_count"] == 12 + assert album_row["duration"] == 210000 + assert album_row["artist_id"] == artist_row["id"] + + assert track_row["title"] == "Song One" + assert track_row["server_source"] == "soulsync" + assert track_row["spotify_track_id"] == "sp-track" + assert track_row["track_number"] == 7 + assert track_row["duration"] == 210000 + assert track_row["track_artist"] == "Guest Artist" + assert track_row["album_id"] == album_row["id"] + assert track_row["file_path"] == str(final_path) diff --git a/tests/test_import_staging.py b/tests/test_import_staging.py new file mode 100644 index 00000000..1cd5e15b --- /dev/null +++ b/tests/test_import_staging.py @@ -0,0 +1,265 @@ +from types import SimpleNamespace + +import core.import_staging as import_staging + + +class FakeClient: + def __init__(self, results=None): + self.results = results or [] + self.calls = [] + + def search_albums(self, query, **kwargs): + self.calls.append((query, kwargs)) + return self.results + + def search_tracks(self, query, **kwargs): + self.calls.append((query, kwargs)) + return self.results + + +def _album_result(album_id, name, artist, release_date="2024-01-01", total_tracks=10, image_url="https://img.example/album.jpg", album_type="album"): + return SimpleNamespace( + id=album_id, + name=name, + artists=[artist], + release_date=release_date, + total_tracks=total_tracks, + image_url=image_url, + album_type=album_type, + ) + + +def test_search_import_albums_prefers_primary_source(monkeypatch): + deezer_client = FakeClient([ + _album_result("deezer-1", "Album One", "Artist One"), + ]) + spotify_client = FakeClient([ + _album_result("spotify-1", "Album One", "Artist One"), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_albums_for_source", + lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit), + ) + + results = import_staging.search_import_albums("Album One", limit=2) + + assert results == [ + { + "id": "deezer-1", + "name": "Album One", + "artist": "Artist One", + "release_date": "2024-01-01", + "total_tracks": 10, + "image_url": "https://img.example/album.jpg", + "album_type": "album", + "source": "deezer", + } + ] + assert deezer_client.calls == [("Album One", {"limit": 2})] + assert spotify_client.calls == [] + + +def test_search_import_albums_falls_back_when_primary_has_no_results(monkeypatch): + deezer_client = FakeClient([]) + spotify_client = FakeClient([ + _album_result("spotify-1", "Album Two", "Artist Two"), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_albums_for_source", + lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit), + ) + + results = import_staging.search_import_albums("Album Two", limit=2) + + assert results == [ + { + "id": "spotify-1", + "name": "Album Two", + "artist": "Artist Two", + "release_date": "2024-01-01", + "total_tracks": 10, + "image_url": "https://img.example/album.jpg", + "album_type": "album", + "source": "spotify", + } + ] + assert deezer_client.calls == [("Album Two", {"limit": 2})] + assert spotify_client.calls == [("Album Two", {"limit": 2, "allow_fallback": False})] + + +def test_search_import_tracks_prefers_primary_source(monkeypatch): + deezer_client = FakeClient([ + SimpleNamespace( + id="deezer-track-1", + name="Song One", + artists=[{"name": "Artist One"}], + album={"id": "deezer-album-1", "name": "Album One"}, + duration_ms=210000, + image_url="https://img.example/track.jpg", + track_number=7, + ), + ]) + spotify_client = FakeClient([ + SimpleNamespace( + id="spotify-track-1", + name="Song One", + artists=["Artist One"], + album="Album One", + duration_ms=210000, + image_url="https://img.example/track.jpg", + track_number=7, + ), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_tracks_for_source", + lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit), + ) + + results = import_staging.search_import_tracks("Song One", limit=2) + + assert results == [ + { + "id": "deezer-track-1", + "name": "Song One", + "artist": "Artist One", + "album": "Album One", + "album_id": "deezer-album-1", + "duration_ms": 210000, + "image_url": "https://img.example/track.jpg", + "track_number": 7, + "source": "deezer", + } + ] + assert deezer_client.calls == [("Song One", {"limit": 2})] + assert spotify_client.calls == [] + + +def test_search_import_tracks_falls_back_when_primary_has_no_results(monkeypatch): + deezer_client = FakeClient([]) + spotify_client = FakeClient([ + SimpleNamespace( + id="spotify-track-1", + name="Song Two", + artists=["Artist Two"], + album="Album Two", + duration_ms=180000, + image_url="", + track_number=3, + ), + ]) + + monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"]) + monkeypatch.setattr( + import_staging, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source), + ) + monkeypatch.setattr( + import_staging, + "_search_tracks_for_source", + lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit), + ) + + results = import_staging.search_import_tracks("Song Two", limit=2) + + assert results == [ + { + "id": "spotify-track-1", + "name": "Song Two", + "artist": "Artist Two", + "album": "Album Two", + "album_id": "", + "duration_ms": 180000, + "image_url": "", + "track_number": 3, + "source": "spotify", + } + ] + assert deezer_client.calls == [("Song Two", {"limit": 2})] + assert spotify_client.calls == [("Song Two", {"limit": 2, "allow_fallback": False})] + + +def test_build_import_suggestions_background_uses_collected_queries(monkeypatch, tmp_path): + staging_root = tmp_path / "Staging" + staging_root.mkdir() + + cache = import_staging.get_import_suggestions_cache() + original_cache = dict(cache) + cache["suggestions"] = [] + cache["building"] = False + cache["built"] = False + + monkeypatch.setattr(import_staging, "get_staging_path", lambda: str(staging_root)) + monkeypatch.setattr(import_staging, "_collect_import_suggestion_queries", lambda staging_path: ["Album One", "Folder Hint"]) + monkeypatch.setattr( + import_staging, + "search_import_albums", + lambda query, limit=2: [ + { + "id": query.lower().replace(" ", "-"), + "name": query, + "artist": f"{query} Artist", + "release_date": "2024-02-01", + "total_tracks": 12, + "image_url": "", + "album_type": "album", + } + ], + ) + + try: + import_staging._build_import_suggestions_background() + + assert cache["built"] is True + assert cache["building"] is False + assert cache["suggestions"] == [ + { + "id": "album-one", + "name": "Album One", + "artist": "Album One Artist", + "release_date": "2024-02-01", + "total_tracks": 12, + "image_url": "", + "album_type": "album", + }, + { + "id": "folder-hint", + "name": "Folder Hint", + "artist": "Folder Hint Artist", + "release_date": "2024-02-01", + "total_tracks": 12, + "image_url": "", + "album_type": "album", + }, + ] + finally: + cache.clear() + cache.update(original_cache) diff --git a/tests/test_metadata_enrichment.py b/tests/test_metadata_enrichment.py new file mode 100644 index 00000000..49cbe630 --- /dev/null +++ b/tests/test_metadata_enrichment.py @@ -0,0 +1,333 @@ +import logging +import types + +import pytest + +from core import metadata_enrichment as me + + +class _Config: + def __init__(self, values=None): + self.values = values or {} + + def get(self, key, default=None): + return self.values.get(key, default) + + +class _FakeTag: + def __init__(self, kind, **kwargs): + self.kind = kind + self.kwargs = kwargs + + +class _FakeID3Tags: + def __init__(self): + self.added = [] + + def add(self, frame): + self.added.append(frame) + + def clear(self): + self.added.clear() + + def getall(self, _key): + return [] + + def keys(self): + return [frame.kind for frame in self.added] + + def __len__(self): + return len(self.added) + + +class _FakeAudio: + def __init__(self): + self.tags = _FakeID3Tags() + self.save_calls = [] + self.clear_pictures_calls = 0 + + def clear_pictures(self): + self.clear_pictures_calls += 1 + + def add_tags(self): + self.tags = _FakeID3Tags() + + def save(self, **kwargs): + self.save_calls.append(kwargs) + + +class _FakeResponse: + def __init__(self, payload, content_type="image/jpeg"): + self._payload = payload + self._content_type = content_type + + def read(self): + return self._payload + + def info(self): + return types.SimpleNamespace(get_content_type=lambda: self._content_type) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + +def _fake_symbols(audio): + def _tag_factory(kind): + return lambda **kwargs: _FakeTag(kind, **kwargs) + + return types.SimpleNamespace( + File=lambda _path: audio, + APEv2=type("FakeAPEv2", (), {}), + APENoHeaderError=Exception, + FLAC=type("FakeFLAC", (), {}), + Picture=type("FakePicture", (), {"__init__": lambda self: None}), + ID3=_FakeID3Tags, + APIC=_tag_factory("APIC"), + TBPM=_tag_factory("TBPM"), + TCOP=_tag_factory("TCOP"), + TDOR=_tag_factory("TDOR"), + TDRC=_tag_factory("TDRC"), + TCON=_tag_factory("TCON"), + TIT2=_tag_factory("TIT2"), + TALB=_tag_factory("TALB"), + TPE1=_tag_factory("TPE1"), + TPE2=_tag_factory("TPE2"), + TPOS=_tag_factory("TPOS"), + TPUB=_tag_factory("TPUB"), + TRCK=_tag_factory("TRCK"), + TSRC=_tag_factory("TSRC"), + TXXX=_tag_factory("TXXX"), + UFID=_tag_factory("UFID"), + TMED=_tag_factory("TMED"), + MP4=type("FakeMP4", (), {}), + MP4Cover=types.SimpleNamespace(FORMAT_JPEG=1, FORMAT_PNG=2, __call__=lambda *args, **kwargs: ("cover", args, kwargs)), + MP4FreeForm=lambda data: ("freeform", data), + OggVorbis=type("FakeOggVorbis", (), {}), + OggOpus=None, + ) + + +def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_for_non_itunes_sources(): + class _ItunesClient: + def __init__(self): + self.called = False + + def resolve_primary_artist(self, artist_id): + self.called = True + raise AssertionError("itunes fallback should not run for non-itunes sources") + + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config({"file_organization.collab_artist_mode": "first"}), + itunes_enrichment_worker=types.SimpleNamespace(client=_ItunesClient()), + ) + + context = { + "source": "spotify", + "artist": {"name": "Artist One & Artist Two", "id": "123", "genres": ["rock", "indie"]}, + "album": { + "name": "Album One", + "total_tracks": 12, + "release_date": "2024-01-02", + "images": [{"url": "https://img.example/album.jpg"}], + }, + "track_info": { + "artists": [{"name": "Artist One"}], + "_source": "spotify", + "track_number": 3, + "disc_number": 2, + "total_tracks": 12, + }, + "original_search_result": { + "title": "Song One", + "artists": [{"name": "Artist One"}], + "clean_title": "Song One", + "clean_album": "Album One", + "clean_artist": "Artist One", + "disc_number": 2, + "duration_ms": 180000, + }, + } + + metadata = me.extract_source_metadata( + context, + context["artist"], + {"is_album": True, "album_name": "Album One", "track_number": 3, "disc_number": 2, "album_image_url": "https://img.example/album.jpg"}, + runtime=runtime, + ) + + assert metadata["source"] == "spotify" + assert metadata["title"] == "Song One" + assert metadata["artist"] == "Artist One" + assert metadata["album_artist"] == "Artist One & Artist Two" + assert metadata["album"] == "Album One" + assert metadata["track_number"] == 3 + assert metadata["total_tracks"] == 12 + assert metadata["disc_number"] == 2 + assert metadata["album_art_url"] == "https://img.example/album.jpg" + assert runtime.itunes_enrichment_worker.client.called is False + + +def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatch): + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config(), + mb_worker=None, + deezer_worker=None, + audiodb_worker=None, + tidal_client=None, + qobuz_enrichment_worker=None, + lastfm_worker=None, + genius_worker=None, + itunes_enrichment_worker=None, + get_database=lambda: None, + ) + + audio = _FakeAudio() + symbols = _fake_symbols(audio) + monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + + current_metadata = { + "source": "deezer", + "source_track_id": "dz-track", + "source_artist_id": "dz-artist", + "source_album_id": "dz-album", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + } + me.embed_source_ids(audio, current_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime) + + current_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"] + assert "DEEZER_TRACK_ID" in current_descs + assert "DEEZER_ARTIST_ID" in current_descs + + audio = _FakeAudio() + symbols = _fake_symbols(audio) + monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + + legacy_metadata = { + "source": "", + "spotify_track_id": "sp-track", + "spotify_artist_id": "sp-artist", + "spotify_album_id": "sp-album", + "itunes_track_id": "it-track", + "itunes_artist_id": "it-artist", + "itunes_album_id": "it-album", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + } + me.embed_source_ids(audio, legacy_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime) + + legacy_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"] + assert "SPOTIFY_TRACK_ID" in legacy_descs + assert "SPOTIFY_ARTIST_ID" in legacy_descs + assert "SPOTIFY_ALBUM_ID" in legacy_descs + assert "ITUNES_TRACK_ID" in legacy_descs + assert "ITUNES_ARTIST_ID" in legacy_descs + assert "ITUNES_ALBUM_ID" in legacy_descs + + +def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch): + audio = _FakeAudio() + symbols = _fake_symbols(audio) + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config( + { + "metadata_enhancement.enabled": True, + "metadata_enhancement.embed_album_art": False, + "metadata_enhancement.tags.write_multi_artist": False, + } + ), + mb_worker=None, + deezer_worker=None, + audiodb_worker=None, + tidal_client=None, + qobuz_enrichment_worker=None, + lastfm_worker=None, + genius_worker=None, + itunes_enrichment_worker=None, + get_database=lambda: None, + ) + + strip_calls = [] + verify_calls = [] + + monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols) + monkeypatch.setattr(me, "_strip_all_non_audio_tags", lambda file_path, runtime=None: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0}) + monkeypatch.setattr( + me, + "extract_source_metadata", + lambda context, artist, album_info, runtime=None: { + "source": "deezer", + "source_track_id": "dz-track", + "source_artist_id": "dz-artist", + "source_album_id": "dz-album", + "title": "Song One", + "artist": "Artist One", + "album_artist": "Artist One", + "album": "Album One", + "track_number": 3, + "total_tracks": 12, + "disc_number": 2, + "date": "2024", + "genre": "Rock", + "musicbrainz_release_id": "mb-release-1", + }, + ) + monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None) + monkeypatch.setattr(me, "_verify_metadata_written", lambda file_path, runtime=None: verify_calls.append(file_path) or True) + + album_info = {} + result = me.enhance_file_metadata( + "song.flac", + {"_audio_quality": ""}, + {"name": "Artist One"}, + album_info, + runtime=runtime, + ) + + assert result is True + assert strip_calls == ["song.flac"] + assert verify_calls == ["song.flac"] + assert audio.clear_pictures_calls == 1 + assert len(audio.save_calls) == 2 + assert album_info["musicbrainz_release_id"] == "mb-release-1" + assert any(frame.kind == "TIT2" for frame in audio.tags.added) + assert any(frame.kind == "TPE1" for frame in audio.tags.added) + assert any(frame.kind == "TXXX" and frame.kwargs.get("desc") == "DEEZER_TRACK_ID" for frame in audio.tags.added) + + +def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch): + runtime = types.SimpleNamespace( + logger=logging.getLogger("test.metadata_enrichment"), + config_manager=_Config( + { + "metadata_enhancement.cover_art_download": True, + "metadata_enhancement.prefer_caa_art": False, + } + ), + ) + + monkeypatch.setattr(me.urllib.request, "urlopen", lambda *args, **kwargs: _FakeResponse(b"cover-bytes")) + + target_dir = tmp_path / "Album One" + target_dir.mkdir() + + me.download_cover_art( + {}, + str(target_dir), + {"album": {"image_url": "https://img.example/album.jpg"}}, + runtime=runtime, + ) + + cover_path = target_dir / "cover.jpg" + assert cover_path.exists() + assert cover_path.read_bytes() == b"cover-bytes" diff --git a/tests/test_metadata_service_single_import_context.py b/tests/test_metadata_service_single_import_context.py new file mode 100644 index 00000000..ed7ae567 --- /dev/null +++ b/tests/test_metadata_service_single_import_context.py @@ -0,0 +1,193 @@ +from types import SimpleNamespace + +from core import metadata_service + + +class FakeClient: + def __init__(self, search_results=None, details=None, artist_details=None): + self.search_results = search_results or [] + self.details = details or {} + self.artist_details = artist_details or {} + self.calls = [] + + def search_tracks(self, query, limit=1, allow_fallback=True): + self.calls.append(("search_tracks", query, limit, allow_fallback)) + return self.search_results + + def get_track_details(self, track_id): + self.calls.append(("get_track_details", track_id)) + return self.details.get(str(track_id)) + + def get_artist(self, artist_id): + self.calls.append(("get_artist", artist_id)) + return self.artist_details.get(str(artist_id)) + + +def _track_result(track_id="track-1", name="Song One", artist="Artist One"): + return SimpleNamespace( + id=track_id, + name=name, + artists=[artist], + album="Album One", + duration_ms=123000, + track_number=1, + disc_number=1, + image_url="https://img.example/track.jpg", + ) + + +def _track_details(source, track_id="track-1", name="Song One", artist_name="Artist One", artist_id="artist-1"): + return { + "id": track_id, + "name": name, + "track_number": 7, + "disc_number": 1, + "duration_ms": 210000, + "explicit": True, + "uri": f"{source}:track:{track_id}", + "artists": [{"name": artist_name, "id": artist_id}], + "album": { + "id": f"{source}-album-1", + "name": "Album One", + "release_date": "2024-01-01", + "album_type": "album", + "total_tracks": 10, + "images": [{"url": f"https://img.example/{source}-album.jpg"}], + "artists": [{"name": artist_name, "id": artist_id}], + }, + } + + +def test_get_single_track_import_context_uses_primary_source_priority(monkeypatch): + deezer_client = FakeClient( + search_results=[_track_result(track_id="deezer-track-1")], + details={"deezer-track-1": _track_details("deezer", track_id="deezer-track-1", artist_name="Artist One", artist_id="deezer-artist-1")}, + artist_details={"deezer-artist-1": {"id": "deezer-artist-1", "genres": ["electronic"]}}, + ) + spotify_client = FakeClient() + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), + ) + + result = metadata_service.get_single_track_import_context("Song One", "Artist One") + + assert result["success"] is True + assert result["source"] == "deezer" + assert result["source_priority"] == ["deezer", "spotify", "itunes"] + assert result["context"]["track_info"]["name"] == "Song One" + assert result["context"]["track_info"]["album_id"] == "deezer-album-1" + assert result["context"]["album"]["image_url"] == "https://img.example/deezer-album.jpg" + assert result["context"]["artist"]["genres"] == ["electronic"] + assert result["context"]["original_search_result"]["clean_title"] == "Song One" + assert deezer_client.calls == [ + ('search_tracks', 'artist:"Artist One" track:"Song One"', 5, True), + ("get_track_details", "deezer-track-1"), + ("get_artist", "deezer-artist-1"), + ] + assert spotify_client.calls == [] + + +def test_get_single_track_import_context_falls_back_to_next_source(monkeypatch): + deezer_client = FakeClient(search_results=[]) + spotify_client = FakeClient( + search_results=[_track_result(track_id="spotify-track-1")], + details={"spotify-track-1": _track_details("spotify", track_id="spotify-track-1", artist_name="Artist Two", artist_id="spotify-artist-1")}, + artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["indie"]}}, + ) + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), + ) + + result = metadata_service.get_single_track_import_context("Song Two", "Artist Two") + + assert result["success"] is True + assert result["source"] == "spotify" + assert result["context"]["track_info"]["id"] == "spotify-track-1" + assert result["context"]["album"]["name"] == "Album One" + assert deezer_client.calls == [ + ('search_tracks', 'artist:"Artist Two" track:"Song Two"', 5, True), + ('search_tracks', 'Song Two', 5, True), + ('search_tracks', 'Artist Two', 5, True), + ] + assert spotify_client.calls == [ + ("search_tracks", "Song Two Artist Two", 5, False), + ("get_track_details", "spotify-track-1"), + ("get_artist", "spotify-artist-1"), + ] + + +def test_get_single_track_import_context_uses_explicit_override_first(monkeypatch): + spotify_client = FakeClient( + details={"override-track-1": _track_details("spotify", track_id="override-track-1", artist_name="Override Artist", artist_id="spotify-artist-1")}, + artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["pop"]}}, + ) + deezer_client = FakeClient() + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source), + ) + + result = metadata_service.get_single_track_import_context( + "Ignored Title", + "Ignored Artist", + override_id="override-track-1", + ) + + assert result["success"] is True + assert result["source"] == "spotify" + assert result["context"]["track_info"]["id"] == "override-track-1" + assert result["context"]["artist"]["genres"] == ["pop"] + assert spotify_client.calls == [ + ("get_track_details", "override-track-1"), + ("get_artist", "spotify-artist-1"), + ] + assert deezer_client.calls == [] + + +def test_get_single_track_import_context_uses_explicit_override_source(monkeypatch): + itunes_client = FakeClient( + details={"override-track-2": _track_details("itunes", track_id="override-track-2", artist_name="Override Artist Two", artist_id="itunes-artist-1")}, + artist_details={"itunes-artist-1": {"id": "itunes-artist-1", "genres": ["singer-songwriter"]}}, + ) + spotify_client = FakeClient() + deezer_client = FakeClient() + + monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer") + monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"]) + monkeypatch.setattr( + metadata_service, + "get_client_for_source", + lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": itunes_client}.get(source), + ) + + result = metadata_service.get_single_track_import_context( + "Ignored Title", + "Ignored Artist", + override_id="override-track-2", + override_source="itunes", + ) + + assert result["success"] is True + assert result["source"] == "itunes" + assert result["context"]["track_info"]["id"] == "override-track-2" + assert result["context"]["artist"]["genres"] == ["singer-songwriter"] + assert itunes_client.calls == [ + ("get_track_details", "override-track-2"), + ("get_artist", "itunes-artist-1"), + ] + assert deezer_client.calls == [] + assert spotify_client.calls == [] diff --git a/tests/test_watchlist_scanner_scan.py b/tests/test_watchlist_scanner_scan.py index dcddd3df..6c08efa0 100644 --- a/tests/test_watchlist_scanner_scan.py +++ b/tests/test_watchlist_scanner_scan.py @@ -2,10 +2,8 @@ import sys import types from datetime import datetime, timedelta - _RECENT_RELEASE_DATE = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d") - if "spotipy" not in sys.modules: spotipy = types.ModuleType("spotipy") @@ -1045,6 +1043,10 @@ def test_update_discovery_pool_incremental_uses_source_priority(monkeypatch): assert scanner.database.discovery_pool_calls assert scanner.database.discovery_pool_calls[0][1] == "deezer" + assert scanner.database.discovery_pool_calls[0][0]["deezer_track_id"] == "dz-track-1" + assert scanner.database.discovery_pool_calls[0][0]["deezer_album_id"] == "dz-release-1" + assert scanner.database.discovery_pool_calls[0][0]["deezer_artist_id"] == "dz-artist" + assert scanner.database.discovery_pool_calls[0][0].get("spotify_track_id") is None assert deezer_client.search_calls == [("Incremental Artist", 1, {})] assert deezer_client.album_calls assert spotify_client.search_calls == [] diff --git a/web_server.py b/web_server.py index 8d2a06eb..504ba7b5 100644 --- a/web_server.py +++ b/web_server.py @@ -26,6 +26,9 @@ from flask import Flask, render_template, request, jsonify, redirect, send_file, from flask_socketio import SocketIO, emit, join_room, leave_room from utils.logging_config import get_logger, setup_logging from utils.async_helpers import run_async +from mutagen.flac import FLAC +from mutagen.mp4 import MP4 +from mutagen.oggvorbis import OggVorbis # --- Core Application Imports --- # Import the same core clients and config manager used by the GUI app @@ -94,6 +97,56 @@ from core.database_update_worker import DatabaseUpdateWorker from core.web_scan_manager import WebScanManager from core.lyrics_client import lyrics_client from core.metadata_cache import get_metadata_cache +from core.import_context import ( + build_import_album_info, + get_import_clean_album, + get_import_clean_artist, + get_import_clean_title, + get_import_context_album, + get_import_context_artist, + get_import_has_clean_metadata, + get_import_has_full_metadata, + get_import_original_search, + get_import_source, + get_import_source_ids, + get_import_track_info, + get_library_source_id_columns, + get_source_tag_names, + normalize_import_context, +) +from core.import_album import ( + build_album_import_context, + build_album_import_match_payload, + resolve_album_artist_context, +) +from core.import_album_naming import resolve_album_group as _resolve_album_group +from core.import_filename import extract_track_number_from_filename, parse_filename_metadata +from core.import_staging import ( + get_import_suggestions_cache, + get_primary_source, + get_staging_path, + read_staging_file_metadata, + refresh_import_suggestions_cache, + search_import_albums, + search_import_tracks, + start_import_suggestions_cache, +) +from core.import_paths import build_final_path_for_track as _build_final_path_for_track +from core.metadata_common import get_file_lock +from core.import_runtime_state import ( + activity_feed, + activity_feed_lock, + add_activity_item, + download_batches, + download_tasks, + matched_context_lock, + matched_downloads_context, + mark_task_completed as _core_mark_task_completed, + set_activity_toast_emitter, + tasks_lock, +) +from core import metadata_enrichment +from core.metadata_source import normalize_album_cache_key from database.music_database import get_database, MusicDatabase from services.sync_service import PlaylistSyncService @@ -225,6 +278,7 @@ _socketio_cors_origins = _resolve_socketio_cors_origins(config_manager) socketio = SocketIO(app, async_mode='threading', cors_allowed_origins=_socketio_cors_origins) _log_socketio_startup_status(_socketio_cors_origins, logger) _socketio_rejection_logger = _SocketIORejectionLogger(logger) +set_activity_toast_emitter(socketio.emit) # Plex PIN auth requests stored in memory for polling _plex_pin_requests = {} @@ -1883,7 +1937,7 @@ def _register_automation_handlers(): # --- 4. Sweep empty staging directories --- _update_automation_progress(automation_id, phase='Sweeping import folder...', progress=60) - staging_path = _get_staging_path() + staging_path = get_staging_path() s_removed = 0 if os.path.isdir(staging_path): for dirpath, _dirnames, _filenames in os.walk(staging_path, topdown=False): @@ -2162,346 +2216,6 @@ def _register_automation_handlers(): logger.info("Automation action handlers registered") - -def _emit_track_downloaded(context): - """Emit track_downloaded event for automation engine. Safe to call anywhere.""" - try: - if not automation_engine: - return - ti = context.get('track_info') or context.get('search_result') or {} - artist_name = '' - artists = ti.get('artists', []) - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - automation_engine.emit('track_downloaded', { - 'artist': artist_name, - 'title': ti.get('name', ti.get('title', '')), - 'album': ti.get('album', ''), - 'quality': context.get('_audio_quality', 'Unknown'), - }) - except Exception: - pass - - -def _record_library_history_download(context): - """Record a completed download to the library_history table. Non-blocking.""" - try: - # Determine download source - search_result = context.get('original_search_result') or context.get('search_result') or {} - username = search_result.get('username', context.get('_download_username', '')) - _svc_map = {'youtube': 'YouTube', 'tidal': 'Tidal', 'qobuz': 'Qobuz', 'hifi': 'HiFi', 'deezer_dl': 'Deezer', 'lidarr': 'Lidarr'} - download_source = _svc_map.get(username, 'Soulseek') - - ti = context.get('track_info') or context.get('search_result') or {} - artist_name = '' - artists = ti.get('artists', []) - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - if not artist_name: - artist_name = ti.get('artist', '') - - album_raw = ti.get('album', '') - album_name = album_raw.get('name', '') if isinstance(album_raw, dict) else str(album_raw or '') - - title = ti.get('name', ti.get('title', '')) - quality = context.get('_audio_quality', '') - file_path = context.get('_final_processed_path', context.get('_final_path', '')) - - # Try to get album art URL - thumb_url = '' - spotify_album = context.get('spotify_album') - if spotify_album and isinstance(spotify_album, dict): - thumb_url = spotify_album.get('image_url', '') - if not thumb_url: - images = spotify_album.get('images', []) - if images: - thumb_url = images[0].get('url', '') - if not thumb_url: - album_info = context.get('album_info', {}) - if isinstance(album_info, dict): - thumb_url = album_info.get('album_image_url', '') - - # Source provenance — what file/track was actually downloaded - source_filename = search_result.get('filename', '') - # Track ID: try search result first, then track_info (Spotify ID used for streaming lookups) - source_track_id = (search_result.get('track_id', '') - or search_result.get('id', '') - or ti.get('id', '')) - - # Source title/artist — what the download source said the track was. - # For Soulseek: parsed from the peer's filename by TrackResult._parse_filename_metadata() - # For Tidal/YouTube/Qobuz: from the streaming API's own metadata - # These live on the candidate's original fields, NOT the spotify_clean_* fields - source_track_title = search_result.get('title', '') or search_result.get('name', '') - source_artist = search_result.get('artist', '') - # For streaming sources, track ID is encoded in filename as "id||display_name" - if source_filename and '||' in source_filename and username in ('tidal', 'youtube', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): - _stream_id = source_filename.split('||')[0] - if _stream_id and not source_track_id: - source_track_id = _stream_id - - # AcoustID verification result - acoustid_result = context.get('_acoustid_result', '') - - db = get_database() - db.add_library_history_entry( - event_type='download', - title=title, - artist_name=artist_name, - album_name=album_name, - quality=quality, - file_path=file_path, - thumb_url=thumb_url, - download_source=download_source, - source_track_id=source_track_id, - source_track_title=source_track_title, - source_filename=source_filename, - acoustid_result=acoustid_result, - source_artist=source_artist - ) - except Exception: - pass # Non-critical, never block download flow - - -def _record_download_provenance(context): - """Record download source provenance for track lineage tracking. Non-blocking.""" - try: - # Extract source info - search_result = context.get('original_search_result') or context.get('search_result') or {} - username = search_result.get('username', context.get('_download_username', '')) - filename = search_result.get('filename', '') - - # Determine source service from username - service_map = {'youtube': 'youtube', 'tidal': 'tidal', 'qobuz': 'qobuz', 'hifi': 'hifi', 'deezer_dl': 'deezer', 'lidarr': 'lidarr'} - source_service = service_map.get(username, 'soulseek') - - # Track metadata - ti = context.get('track_info') or context.get('search_result') or {} - artist_name = '' - artists = ti.get('artists', []) - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - if not artist_name: - artist_name = ti.get('artist', '') - - album_raw = ti.get('album', '') - album_name = album_raw.get('name', '') if isinstance(album_raw, dict) else str(album_raw or '') - title = ti.get('name', ti.get('title', '')) - - file_path = context.get('_final_processed_path', context.get('_final_path', '')) - quality = context.get('_audio_quality', '') - size = search_result.get('size', 0) - - # Read audio details from the file for provenance (survives transcoding) - bit_depth = None - sample_rate = None - bitrate = None - try: - if file_path and os.path.isfile(file_path): - from mutagen import File as MutagenFile - audio = MutagenFile(file_path) - if audio and audio.info: - sample_rate = getattr(audio.info, 'sample_rate', None) - bitrate = getattr(audio.info, 'bitrate', None) - bit_depth = getattr(audio.info, 'bits_per_sample', None) - except Exception: - pass - - db = get_database() - db.record_track_download( - file_path=file_path, - source_service=source_service, - source_username=username, - source_filename=filename, - source_size=size or 0, - audio_quality=quality, - track_title=title, - track_artist=artist_name, - track_album=album_name, - bit_depth=bit_depth, - sample_rate=sample_rate, - bitrate=bitrate, - ) - except Exception: - pass # Non-critical, never block download flow - - -def _record_soulsync_library_entry(context, spotify_artist, album_info): - """Write artist/album/track to library DB after successful download/import. - - Only runs when active server is 'soulsync' (standalone mode). Creates - DB records with server_source='soulsync' and pre-populates enrichment - IDs so enrichment workers don't need to re-discover them. - """ - try: - active_server = config_manager.get_active_media_server() - if active_server != 'soulsync': - return - - final_path = context.get('_final_processed_path') - if not final_path: - return - - spotify_album = context.get('spotify_album', {}) or {} - track_info = context.get('track_info', {}) or {} - original_search = context.get('original_search_result', {}) or {} - - artist_name = (spotify_artist or {}).get('name', '') - if not artist_name: - artist_name = original_search.get('spotify_clean_artist', '') or original_search.get('artist', '') - if not artist_name or artist_name in ('Unknown', 'Unknown Artist'): - return - - album_name = '' - if album_info and isinstance(album_info, dict): - album_name = album_info.get('album_name', '') - if not album_name: - album_name = spotify_album.get('name', '') or original_search.get('album', '') - if not album_name: - album_name = track_info.get('name', 'Unknown') - - track_name = original_search.get('spotify_clean_title', '') or track_info.get('name', '') or original_search.get('title', '') - track_number = (track_info.get('track_number') or (album_info.get('track_number') if isinstance(album_info, dict) else None)) or 1 - duration_ms = track_info.get('duration_ms', 0) or 0 - - year = None - release_date = spotify_album.get('release_date', '') - if release_date and len(release_date) >= 4: - try: - year = int(release_date[:4]) - except ValueError: - pass - - image_url = spotify_album.get('image_url', '') - if not image_url: - images = spotify_album.get('images', []) - if images and isinstance(images, list) and len(images) > 0: - img = images[0] - image_url = img.get('url', '') if isinstance(img, dict) else str(img) - - # Enrichment IDs from context — saves enrichment workers from re-discovering - spotify_artist_id = (spotify_artist or {}).get('id', '') - if spotify_artist_id in ('auto_import', 'from_sync_modal', 'explicit_artist', ''): - spotify_artist_id = '' - spotify_album_id = spotify_album.get('id', '') - if spotify_album_id in ('from_sync_modal', 'explicit_album', ''): - spotify_album_id = '' - spotify_track_id = track_info.get('id', '') or original_search.get('id', '') - - genres = (spotify_artist or {}).get('genres', []) - if genres: - from core.genre_filter import filter_genres as _gf2 - genres = _gf2(genres, config_manager) - genres_json = json.dumps(genres) if genres else '' - - bitrate = 0 - try: - from mutagen import File as MutagenFile - audio = MutagenFile(final_path) - if audio and hasattr(audio, 'info') and audio.info and hasattr(audio.info, 'bitrate'): - bitrate = int(audio.info.bitrate / 1000) if audio.info.bitrate else 0 - except Exception: - pass - - import hashlib - def _sid(text): - return str(abs(int(hashlib.md5(text.encode('utf-8', errors='replace')).hexdigest(), 16)) % (10 ** 9)) - - artist_id = _sid(artist_name.lower().strip()) - album_id = _sid(f"{artist_name}::{album_name}".lower().strip()) - track_id = _sid(final_path) - total_tracks = spotify_album.get('total_tracks', 0) or 0 - - db = get_database() - with db._get_connection() as conn: - cursor = conn.cursor() - - # ── Artist: find existing soulsync record or create ── - cursor.execute("SELECT id FROM artists WHERE id = ? AND server_source = 'soulsync'", (artist_id,)) - if not cursor.fetchone(): - # Check if soulsync artist exists by name - cursor.execute("SELECT id FROM artists WHERE name COLLATE NOCASE = ? AND server_source = 'soulsync' LIMIT 1", (artist_name,)) - existing_by_name = cursor.fetchone() - if existing_by_name: - artist_id = existing_by_name[0] - else: - # Avoid PK collision with other server sources - cursor.execute("SELECT id FROM artists WHERE id = ?", (artist_id,)) - if cursor.fetchone(): - # ID taken by another source — append suffix - artist_id = _sid(artist_name.lower().strip() + '::soulsync') - cursor.execute(""" - INSERT INTO artists (id, name, genres, thumb_url, server_source, created_at, updated_at) - VALUES (?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - """, (artist_id, artist_name, genres_json, image_url)) - if spotify_artist_id: - try: - cursor.execute("UPDATE artists SET spotify_artist_id = ? WHERE id = ?", (spotify_artist_id, artist_id)) - except Exception: - pass - - # ── Album: find existing soulsync record or create ── - cursor.execute("SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'", (album_id,)) - if not cursor.fetchone(): - cursor.execute("SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? AND server_source = 'soulsync' LIMIT 1", - (album_name, artist_id)) - existing_album_by_name = cursor.fetchone() - if existing_album_by_name: - album_id = existing_album_by_name[0] - else: - # Avoid PK collision - cursor.execute("SELECT id FROM albums WHERE id = ?", (album_id,)) - if cursor.fetchone(): - album_id = _sid(f"{artist_name}::{album_name}::soulsync".lower().strip()) - cursor.execute(""" - INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, track_count, - duration, server_source, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - """, (album_id, artist_id, album_name, year, image_url, genres_json, total_tracks, duration_ms)) - if spotify_album_id: - try: - cursor.execute("UPDATE albums SET spotify_album_id = ? WHERE id = ?", (spotify_album_id, album_id)) - except Exception: - pass - - # ── Track ── - # Determine per-track artist (for compilations/features where track artist != album artist) - track_artist = None - track_artists_list = track_info.get('artists', []) or original_search.get('artists', []) - if track_artists_list: - first_track_artist = track_artists_list[0] - if isinstance(first_track_artist, dict): - ta_name = first_track_artist.get('name', '') - else: - ta_name = str(first_track_artist) - if ta_name and ta_name.lower() != artist_name.lower(): - track_artist = ta_name # Only store when different from album artist - - cursor.execute("SELECT id FROM tracks WHERE file_path = ?", (final_path,)) - if not cursor.fetchone(): - cursor.execute(""" - INSERT INTO tracks (id, album_id, artist_id, title, track_number, - duration, file_path, bitrate, track_artist, server_source, - created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - """, (track_id, album_id, artist_id, track_name, track_number, - duration_ms, final_path, bitrate, track_artist)) - if spotify_track_id: - try: - cursor.execute("UPDATE tracks SET spotify_track_id = ? WHERE id = ?", (spotify_track_id, track_id)) - except Exception: - pass - - conn.commit() - logger.info(f"[SoulSync Library] Added: {artist_name} / {album_name} / {track_name}") - - except Exception as e: - logger.debug(f"[SoulSync Library] Non-critical error: {e}") - - # --- Register Public REST API Blueprint (v1) --- try: from api import create_api_blueprint, limiter @@ -2652,30 +2366,13 @@ def _update_automation_progress(automation_id, **kwargs): pass # --- Global Matched Downloads Context Management --- -# Thread-safe storage for matched download contexts -# Key: slskd download ID, Value: dict containing Spotify artist/album data -matched_downloads_context = {} -matched_context_lock = threading.Lock() +# Shared with core.import_runtime_state so the refactored pipeline and web +# server operate on the same context registry. _orphaned_download_keys = set() # Context keys of downloads abandoned during retry -# --- File-Level Metadata Write Locking --- -# Prevents concurrent threads from writing metadata to the same file simultaneously -_metadata_write_locks = {} # file_path -> threading.Lock() -_metadata_locks_lock = threading.Lock() # Lock for the locks dict - -def _get_file_lock(file_path): - """Get or create a lock for a specific file path to prevent concurrent metadata writes.""" - with _metadata_locks_lock: - if file_path not in _metadata_write_locks: - _metadata_write_locks[file_path] = threading.Lock() - return _metadata_write_locks[file_path] - # --- Download Missing Tracks Modal State Management --- # Thread-safe state tracking for modal download functionality with batch management missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker") -download_tasks = {} # task_id -> task state dict -download_batches = {} # batch_id -> {queue, active_count, max_concurrent} -tasks_lock = threading.Lock() batch_locks = {} # batch_id -> Lock() for atomic batch operations def _get_max_concurrent(): @@ -2707,8 +2404,7 @@ def _mark_task_completed(task_id, track_info=None): Assumes task_id exists in download_tasks (should be called within tasks_lock). """ global session_completed_downloads - - download_tasks[task_id]['status'] = 'completed' + _core_mark_task_completed(task_id, track_info) # Increment session counter (matches dashboard.py behavior) with session_stats_lock: @@ -4159,12 +3855,6 @@ _EDITION_BARE_RE = _re.compile( _re.IGNORECASE ) -def _normalize_album_cache_key(album_name): - """Normalize album name for cache key: strip edition suffixes, lowercase, strip whitespace.""" - result = _EDITION_PAREN_RE.sub('', album_name) - result = _EDITION_BARE_RE.sub('', result) - return result.lower().strip() - def _prepare_stream_task(track_data): """ Background streaming task that downloads track to Stream folder and updates global state. @@ -6225,10 +5915,6 @@ def get_debug_info(): return jsonify(info) -# Global activity tracking storage -activity_feed = [] -activity_feed_lock = threading.Lock() - @app.route('/api/activity/feed') def get_activity_feed(): """Get recent activity feed for dashboard""" @@ -6297,37 +5983,6 @@ def get_activity_logs(): except Exception as e: return jsonify({'logs': [f'Error reading activity feed: {str(e)}']}) -def add_activity_item(icon: str, title: str, subtitle: str, time_ago: str = "Now", show_toast: bool = True): - """Add activity item to the feed (replicates dashboard.py functionality)""" - try: - import time - from datetime import datetime, timezone - activity_item = { - 'icon': icon, - 'title': title, - 'subtitle': subtitle, - 'time': datetime.now(timezone.utc).isoformat(), - 'timestamp': time.time(), - 'show_toast': show_toast - } - - with activity_feed_lock: - activity_feed.append(activity_item) - # Keep only last 20 items to prevent memory growth - if len(activity_feed) > 20: - activity_feed.pop(0) - - # Instant toast push via WebSocket (replaces 3-second polling) - if show_toast: - try: - socketio.emit('dashboard:toast', activity_item) - except Exception: - pass - - logger.info(f"Activity: {icon} {title} - {subtitle}") - except Exception as e: - logger.error(f"Error adding activity item: {e}") - # --- Internal API Key Management (browser-only, no auth) --- @app.route('/api/v1/api-keys-internal', methods=['GET']) def list_api_keys_internal(): @@ -13238,7 +12893,7 @@ def write_track_tags(track_id): cover_url = thumb # Use file lock for thread safety - file_lock = _get_file_lock(resolved_path) + file_lock = get_file_lock(resolved_path) with file_lock: result = write_tags_to_file(resolved_path, db_data, embed_cover=embed_cover, cover_url=cover_url) @@ -13396,7 +13051,7 @@ def write_tracks_tags_batch(): if thumb and thumb.startswith('http'): art_data = cover_cache.get(thumb) - file_lock = _get_file_lock(resolved_path) + file_lock = get_file_lock(resolved_path) with file_lock: write_result = write_tags_to_file( resolved_path, db_data, @@ -13523,7 +13178,7 @@ def analyze_track_replaygain(track_id): track_gain_db = _RG_REFERENCE_LUFS - lufs - file_lock = _get_file_lock(file_path) + file_lock = get_file_lock(file_path) with file_lock: ok = _rg_write_tags(file_path, track_gain_db, peak_dbfs) @@ -13625,7 +13280,7 @@ def analyze_album_replaygain(album_id): continue file_path, track_gain_db, peak_dbfs = entry try: - file_lock = _get_file_lock(file_path) + file_lock = get_file_lock(file_path) with file_lock: _rg_write_tags(file_path, track_gain_db, peak_dbfs, album_gain_db, album_peak_dbfs) @@ -13711,7 +13366,7 @@ def analyze_tracks_replaygain_batch(): try: lufs, peak_dbfs = _rg_analyze_track(file_path) track_gain_db = _RG_REFERENCE_LUFS - lufs - file_lock = _get_file_lock(file_path) + file_lock = get_file_lock(file_path) with file_lock: _rg_write_tags(file_path, track_gain_db, peak_dbfs) with _rg_batch_lock: @@ -16562,7 +16217,7 @@ def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_ar _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() with _mb_release_cache_lock: - _mb_release_cache[(_normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid + _mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid _mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid with _mb_release_detail_cache_lock: _mb_release_detail_cache[_pf_mbid] = _pf_release @@ -16703,7 +16358,7 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album): _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() with _mb_release_cache_lock: - _mb_release_cache[(_normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid + _mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid _mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid with _mb_release_detail_cache_lock: _mb_release_detail_cache[_pf_mbid] = _pf_release @@ -16776,9 +16431,6 @@ def _start_album_download_tasks(album_result, spotify_artist, spotify_album): return started_count - - - @app.route('/api/download/matched', methods=['POST']) def start_matched_download(): """ @@ -16903,82 +16555,12 @@ def start_matched_download(): traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 - - - - - def _parse_filename_metadata(filename: str) -> dict: """ A direct port of the metadata parsing logic from the GUI's soulseek_client.py. This is the crucial missing step that cleans filenames BEFORE Spotify matching. """ - import re - import os - - metadata = { - 'artist': None, - 'title': None, - 'album': None, - 'track_number': None - } - - # Get just the filename without extension and path - base_name = os.path.splitext(os.path.basename(filename))[0] - - # --- Logic from soulseek_client.py --- - patterns = [ - # Pattern: 01 - Artist - Title (three-part with track number) - r'^(?P\d{1,2})\s*[-\.]\s*(?P.+?)\s*[-–]\s*(?P.+)$', - # Pattern: 01 - Title (track number + title — must come before Artist - Title - # to prevent "08 - Kilburn Market Dub" matching as artist="08") - r'^(?P<track_number>\d{1,2})\s*[-\.]\s*(?P<title>.+)$', - # Pattern: Artist - Title - r'^(?P<artist>.+?)\s*[-–]\s*(?P<title>.+)$', - ] - - for pattern in patterns: - match = re.match(pattern, base_name) - if match: - match_dict = match.groupdict() - metadata['track_number'] = int(match_dict['track_number']) if match_dict.get('track_number') else None - metadata['artist'] = match_dict.get('artist', '').strip() or None - metadata['title'] = match_dict.get('title', '').strip() or None - break # Stop after first successful match - - # If title is still missing, use the whole base_name - if not metadata['title']: - metadata['title'] = base_name.strip() - - # Fallback for underscore formats like 'Artist_Album_01_Title' - if not metadata['artist'] and '_' in base_name: - parts = base_name.split('_') - if len(parts) >= 3: - # A common pattern is Artist_Album_TrackNum_Title - if parts[-2].isdigit(): - metadata['artist'] = parts[0].strip() - metadata['title'] = parts[-1].strip() - metadata['track_number'] = int(parts[-2]) - metadata['album'] = parts[1].strip() - - # Final cleanup on title if it contains the artist - if metadata['artist'] and metadata['title'] and metadata['artist'].lower() in metadata['title'].lower(): - metadata['title'] = metadata['title'].replace(metadata['artist'], '').lstrip(' -–_').strip() - - - # Try to extract album from the full directory path - if '/' in filename or '\\' in filename: - path_parts = filename.replace('\\', '/').split('/') - if len(path_parts) >= 2: - # The parent directory is often the album - potential_album = path_parts[-2] - # Clean common prefixes like '2024 - ' - cleaned_album = re.sub(r'^\d{4}\s*-\s*', '', potential_album).strip() - metadata['album'] = cleaned_album - - logger.info(f"Parsed Filename '{base_name}': Artist='{metadata['artist']}', Title='{metadata['title']}', Album='{metadata['album']}', Track#='{metadata['track_number']}'") - return metadata - + return parse_filename_metadata(filename) def _read_staging_file_metadata(full_path: str, filename: str) -> dict: """Read metadata from a staging file — tags first, filename parsing as fallback. @@ -16986,57 +16568,7 @@ def _read_staging_file_metadata(full_path: str, filename: str) -> dict: Returns dict with: title, artist, albumartist, album, track_number, disc_number. Only falls back to filename parsing when BOTH title AND artist tags are empty. """ - meta = { - 'title': None, 'artist': None, 'albumartist': None, - 'album': None, 'track_number': None, 'disc_number': None, - } - - # Phase 1: Read embedded tags (most reliable) - try: - from mutagen import File as MutagenFile - tags = MutagenFile(full_path, easy=True) - if tags: - def _first(tag_list): - if isinstance(tag_list, list) and tag_list: - val = str(tag_list[0]).strip() - return val if val else None - return None - - meta['title'] = _first(tags.get('title')) - meta['artist'] = _first(tags.get('artist')) - meta['albumartist'] = _first(tags.get('albumartist')) - meta['album'] = _first(tags.get('album')) - - tn = _first(tags.get('tracknumber')) - if tn: - try: - meta['track_number'] = int(tn.split('/')[0]) - except (ValueError, IndexError): - pass - - dn = _first(tags.get('discnumber')) - if dn: - try: - meta['disc_number'] = int(dn.split('/')[0]) - except (ValueError, IndexError): - pass - except Exception: - pass - - # Phase 2: Only fall back to filename parsing when tags are genuinely empty - if not meta['title'] and not meta['artist']: - parsed = _parse_filename_metadata(filename) - meta['title'] = parsed.get('title') or os.path.splitext(os.path.basename(filename))[0] - meta['artist'] = parsed.get('artist') - if not meta['track_number']: - meta['track_number'] = parsed.get('track_number') - if not meta['album']: - meta['album'] = parsed.get('album') - elif not meta['title']: - # Has artist tag but no title — use filename for title only - meta['title'] = os.path.splitext(os.path.basename(filename))[0] - - return meta + return read_staging_file_metadata(full_path, filename) # =================================================================== @@ -17160,223 +16692,6 @@ def _search_track_in_album_context(original_search: dict, artist: dict) -> dict: return None - - -def _detect_album_info_web(context: dict, artist: dict) -> dict: - """ - Enhanced album detection with GUI parity - multi-priority logic. - (Updated to match GUI downloads.py logic exactly) - """ - try: - # Log available data for debugging (GUI PARITY) - original_search = context.get("original_search_result", {}) - logger.info( - "[Album Detection] start: track=%r clean_spotify_title=%r clean_spotify_album=%r " - "filename_album=%r artist=%r clean_data=%s album_download=%s", - original_search.get('title', 'Unknown'), - original_search.get('spotify_clean_title', 'None'), - original_search.get('spotify_clean_album', 'None'), - original_search.get('album', 'None'), - artist.get('name', 'Unknown'), - context.get('has_clean_spotify_data', False), - context.get('is_album_download', False), - ) - spotify_album_context = context.get("spotify_album") - is_album_download = context.get("is_album_download", False) - artist_name = artist['name'] - - logger.info( - "[Album Detection] track=%r artist=%r has_album_attr=%s album=%r", - original_search.get('title', 'Unknown'), - artist_name, - bool(original_search.get('album')), - original_search.get('album'), - ) - - # --- THIS IS THE CRITICAL FIX --- - # If this is part of a matched album download, we TRUST the context data completely. - # This is the exact logic from downloads.py. - if is_album_download and spotify_album_context: - # We exclusively use the track number and title that were matched - # *before* the download started. We do not try to re-parse the filename. - track_number = original_search.get('track_number', 1) - clean_track_name = original_search.get('title', 'Unknown Track') - - logger.info( - "[Album Detection] using matched context: track_number=%s title=%r album=%r", - track_number, - clean_track_name, - spotify_album_context['name'], - ) - - return { - 'is_album': True, - 'album_name': spotify_album_context['name'], - 'track_number': track_number, - 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album_context.get('image_url') - } - - # PRIORITY 1: Try album-aware search using clean Spotify album name (GUI PARITY) - # Prioritize clean Spotify album name over filename-parsed album - clean_album_name = original_search.get('spotify_clean_album') - fallback_album_name = original_search.get('album') - - album_name_to_use = None - album_source = None - - if clean_album_name and clean_album_name.strip() and clean_album_name != "Unknown Album": - album_name_to_use = clean_album_name - album_source = "CLEAN_SPOTIFY" - elif fallback_album_name and fallback_album_name.strip() and fallback_album_name != "Unknown Album": - album_name_to_use = fallback_album_name - album_source = "FILENAME_PARSED" - - if album_name_to_use: - track_title = original_search.get('spotify_clean_title') or original_search.get('title', 'Unknown') - logger.info(f"ALBUM-AWARE SEARCH ({album_source}): Looking for '{track_title}' in album '{album_name_to_use}'") - - # Temporarily set the album for the search - original_album = original_search.get('album') - original_search['album'] = album_name_to_use - - try: - album_result = _search_track_in_album_context_web(context, artist) - if album_result: - logger.info(f"PRIORITY 1 SUCCESS: Found track using {album_source} album name - FORCING album classification") - return album_result - else: - logger.error(f"PRIORITY 1 FAILED: Track not found using {album_source} album name") - finally: - # Restore original album value - if original_album is not None: - original_search['album'] = original_album - else: - original_search.pop('album', None) - - # PRIORITY 2: Fallback to individual track search for clean metadata - logger.info("Searching Spotify for individual track info (PRIORITY 2)...") - - # Clean the track title before searching - remove artist prefix - # Prioritize clean Spotify title over filename-parsed title - track_title_to_use = original_search.get('spotify_clean_title') or original_search.get('title', '') - clean_title = _clean_track_title_web(track_title_to_use, artist_name) - logger.info(f"Cleaned title: '{track_title_to_use}' -> '{clean_title}'") - - # Search for the track by artist and cleaned title - query = f"artist:{artist_name} track:{clean_title}" - tracks = spotify_client.search_tracks(query, limit=5) - - # Find the best matching track (prefer album versions over singles) - best_match = None - best_confidence = 0 - - if tracks: - from core.matching_engine import MusicMatchingEngine - matching_engine = MusicMatchingEngine() - for track in tracks: - # Calculate confidence based on artist and title similarity - artist_confidence = matching_engine.similarity_score( - matching_engine.normalize_string(artist_name), - matching_engine.normalize_string(track.artists[0] if track.artists else '') - ) - title_confidence = matching_engine.similarity_score( - matching_engine.normalize_string(clean_title), - matching_engine.normalize_string(track.name) - ) - - combined_confidence = (artist_confidence * 0.6 + title_confidence * 0.4) - - # Small bonus for album tracks so they win ties over singles/EPs - album_type = getattr(track, 'album_type', None) or '' - if album_type == 'album': - combined_confidence += 0.02 - elif album_type == 'ep': - combined_confidence += 0.01 - - if combined_confidence > best_confidence and combined_confidence > 0.75: # Higher threshold to avoid bad matches - best_match = track - best_confidence = combined_confidence - - # If we found a good Spotify match, use it for clean metadata - if best_match and best_confidence > 0.75: - logger.info(f"Found matching Spotify track: '{best_match.name}' - Album: '{best_match.album}' (confidence: {best_confidence:.2f})") - - # Get detailed track information using Spotify's track API - detailed_track = None - if hasattr(best_match, 'id') and best_match.id: - logger.info(f"Getting detailed track info from Spotify API for track ID: {best_match.id}") - detailed_track = spotify_client.get_track_details(best_match.id) - - # Use detailed track data if available - if detailed_track: - logger.info("Got detailed track data from Spotify API") - album_name = _clean_album_title_web(detailed_track['album']['name'], artist_name) - clean_track_name = detailed_track['name'] # Use Spotify's clean track name - album_type = detailed_track['album'].get('album_type', 'album') - total_tracks = detailed_track['album'].get('total_tracks', 1) - spotify_track_number = detailed_track.get('track_number', 1) - - logger.info(f"Spotify album info: '{album_name}' (type: {album_type}, total_tracks: {total_tracks}, track#: {spotify_track_number})") - logger.info(f"Clean track name from Spotify: '{clean_track_name}'") - - # Enhanced album detection using detailed API data (GUI PARITY) - is_album = ( - # Album type is 'album' (not 'single') - album_type == 'album' and - # Album has multiple tracks - total_tracks > 1 and - # Album name different from track name - matching_engine.normalize_string(album_name) != matching_engine.normalize_string(clean_track_name) and - # Album name is not just the artist name - matching_engine.normalize_string(album_name) != matching_engine.normalize_string(artist_name) - ) - - album_image_url = None - if detailed_track['album'].get('images'): - album_image_url = detailed_track['album']['images'][0].get('url') - - logger.info(f"Album classification: {is_album} (type={album_type}, tracks={total_tracks})") - - return { - 'is_album': is_album, - 'album_name': album_name, - 'track_number': spotify_track_number, - 'clean_track_name': clean_track_name, - 'album_image_url': album_image_url, - 'confidence': best_confidence, - 'source': 'spotify_api_detailed' - } - - # Fallback: Use original data with basic cleaning - logger.warning("No good Spotify match found, using original data") - fallback_title = _clean_track_title_web(original_search.get('title', 'Unknown Track'), artist_name) - - # Preserve track_number from context if available (playlist sync tracks have it) - _ctx_track_number = (original_search.get('track_number') - or context.get('track_info', {}).get('track_number') - or 1) - - return { - 'is_album': False, - 'clean_track_name': fallback_title, - 'album_name': fallback_title, - 'track_number': _ctx_track_number, - 'confidence': 0.0, - 'source': 'fallback_original' - } - - except Exception as e: - logger.error(f"Error in _detect_album_info_web: {e}") - clean_title = _clean_track_title_web(context.get("original_search_result", {}).get('title', 'Unknown'), artist.get('name', '')) - _err_tn = (context.get("original_search_result", {}).get('track_number') - or context.get('track_info', {}).get('track_number') - or 1) - return {'is_album': False, 'clean_track_name': clean_title, 'album_name': clean_title, 'track_number': _err_tn} - - - - def _cleanup_empty_directories(download_path, moved_file_path): """Cleans up empty directories after a file move, ignoring hidden files.""" import os @@ -18520,333 +17835,6 @@ def _get_audio_quality_string(file_path): logger.debug(f"Could not determine audio quality for {file_path}: {e}") return '' -def _downsample_hires_flac(final_path, context): - """Downsample a 24-bit hi-res FLAC to 16-bit/44.1kHz CD quality. - - Only runs when downsample_hires is enabled and the file is a 24-bit FLAC. - Replaces the original file in-place (write to temp, verify, swap). - - Returns the (possibly renamed) final_path, or None if no conversion needed. - """ - if not config_manager.get('lossy_copy.downsample_hires', False): - return None - - ext = os.path.splitext(final_path)[1].lower() - if ext != '.flac': - return None - - # Check current bit depth — only downsample if hi-res (>16 bit) - try: - from mutagen.flac import FLAC - audio = FLAC(final_path) - original_bits = audio.info.bits_per_sample - original_rate = audio.info.sample_rate - except Exception as e: - logger.error(f"[Downsample] Could not read FLAC info: {e}") - return None - - if original_bits <= 16 and original_rate <= 44100: - return None # Already CD quality or below - - logger.info(f"[Downsample] Converting {original_bits}-bit/{original_rate}Hz → 16-bit/44100Hz: {os.path.basename(final_path)}") - - ffmpeg_bin = shutil.which('ffmpeg') - if not ffmpeg_bin: - local = os.path.join(os.path.dirname(__file__), 'tools', 'ffmpeg') - if os.path.isfile(local): - ffmpeg_bin = local - else: - logger.warning("[Downsample] ffmpeg not found — skipping hi-res conversion") - return None - - temp_path = final_path + '.tmp.flac' - try: - result = subprocess.run([ - ffmpeg_bin, '-i', final_path, - '-sample_fmt', 's16', - '-ar', '44100', - '-map_metadata', '0', - '-compression_level', '8', - '-y', temp_path - ], capture_output=True, text=True, timeout=300) - - if result.returncode != 0: - logger.error(f"[Downsample] ffmpeg failed: {result.stderr[:200]}") - if os.path.exists(temp_path): - os.remove(temp_path) - return None - - # Verify the output is a valid 16-bit FLAC - if not os.path.isfile(temp_path) or os.path.getsize(temp_path) == 0: - logger.warning("[Downsample] Output file missing or empty") - if os.path.exists(temp_path): - os.remove(temp_path) - return None - - verify_audio = FLAC(temp_path) - if verify_audio.info.bits_per_sample != 16: - logger.info(f"[Downsample] Output not 16-bit ({verify_audio.info.bits_per_sample}-bit), aborting") - os.remove(temp_path) - return None - - # Atomic swap — replace original with downsampled version - os.replace(temp_path, final_path) - logger.info(f"[Downsample] Converted to 16-bit/44.1kHz: {os.path.basename(final_path)}") - - # Update QUALITY tag in the new file - new_quality = 'FLAC 16bit' - try: - updated_audio = FLAC(final_path) - updated_audio['QUALITY'] = new_quality - updated_audio.save() - except Exception as tag_err: - logger.error(f"[Downsample] Could not update QUALITY tag: {tag_err}") - - # Update context so downstream (lossy copy, metadata) reflects new quality - old_quality = context.get('_audio_quality', '') - context['_audio_quality'] = new_quality - - # If filename contains old quality string (from $quality template), rename - if old_quality and old_quality != new_quality and old_quality in os.path.basename(final_path): - new_basename = os.path.basename(final_path).replace(old_quality, new_quality) - new_path = os.path.join(os.path.dirname(final_path), new_basename) - try: - os.rename(final_path, new_path) - logger.info(f"[Downsample] Renamed: {os.path.basename(final_path)} → {new_basename}") - # Rename matching lyrics sidecar file if it exists (.lrc or .txt) - for lyrics_ext in ('.lrc', '.txt'): - old_lyrics = os.path.splitext(final_path)[0] + lyrics_ext - if os.path.isfile(old_lyrics): - new_lyrics = os.path.splitext(new_path)[0] + lyrics_ext - os.rename(old_lyrics, new_lyrics) - return new_path - except Exception as rename_err: - logger.error(f"[Downsample] Could not rename file: {rename_err}") - - return final_path - - except subprocess.TimeoutExpired: - logger.info(f"[Downsample] Conversion timed out for: {os.path.basename(final_path)}") - if os.path.exists(temp_path): - os.remove(temp_path) - except Exception as e: - logger.error(f"[Downsample] Conversion error: {e}") - if os.path.exists(temp_path): - try: - os.remove(temp_path) - except Exception: - pass - return None - - -def _create_lossy_copy(final_path): - """Convert a FLAC file to a lossy codec at the user's configured bitrate. - - Supported codecs: mp3 (libmp3lame), opus (libopus), aac (aac/libfdk_aac). - Only runs when lossy_copy is enabled and the file is a FLAC. - Places the output alongside the FLAC with the same basename. - - Returns the output path if Blasphemy Mode deleted the original, else None. - """ - if not config_manager.get('lossy_copy.enabled', False): - return None - - ext = os.path.splitext(final_path)[1].lower() - if ext != '.flac': - return None - - codec = config_manager.get('lossy_copy.codec', 'mp3').lower() - bitrate = config_manager.get('lossy_copy.bitrate', '320') - - # Opus max per-channel bitrate is 256kbps — cap to avoid encoding failures - if codec == 'opus' and int(bitrate) > 256: - bitrate = '256' - - # Codec configuration: (ffmpeg_codec, extension, quality_label, extra_args) - # -vn strips video/image streams (embedded cover art) which can cause - # conversion failures when the output muxer can't handle image streams - codec_map = { - 'mp3': ('libmp3lame', '.mp3', f'MP3-{bitrate}', ['-vn', '-id3v2_version', '3']), - 'opus': ('libopus', '.opus', f'OPUS-{bitrate}', ['-vn', '-map', '0:a', '-vbr', 'on']), - 'aac': ('aac', '.m4a', f'AAC-{bitrate}', ['-vn', '-movflags', '+faststart']), - } - - if codec not in codec_map: - logger.info(f"[Lossy Copy] Unknown codec '{codec}' — skipping conversion") - return None - - ffmpeg_codec, out_ext, quality_label, extra_args = codec_map[codec] - out_path = os.path.splitext(final_path)[0] + out_ext - - # If $quality was used in filename, swap FLAC quality for lossy quality - original_quality = _get_audio_quality_string(final_path) - if original_quality: - out_basename = os.path.basename(out_path) - if original_quality in out_basename: - out_basename = out_basename.replace(original_quality, quality_label) - out_path = os.path.join(os.path.dirname(out_path), out_basename) - - ffmpeg_bin = shutil.which('ffmpeg') - if not ffmpeg_bin: - local = os.path.join(os.path.dirname(__file__), 'tools', 'ffmpeg') - if os.path.isfile(local): - ffmpeg_bin = local - else: - logger.warning(f"[Lossy Copy] ffmpeg not found — skipping {codec.upper()} conversion") - return None - - try: - logger.info(f"[Lossy Copy] Converting to {quality_label}: {os.path.basename(final_path)}") - cmd = [ - ffmpeg_bin, '-i', final_path, - '-codec:a', ffmpeg_codec, - '-b:a', f'{bitrate}k', - '-map_metadata', '0', - ] + extra_args + ['-y', out_path] - - result = subprocess.run(cmd, capture_output=True, text=True, timeout=120) - - if result.returncode == 0: - logger.info(f"[Lossy Copy] Created {quality_label} copy: {os.path.basename(out_path)}") - - # Fix QUALITY tag — the FLAC's tag was copied verbatim by ffmpeg - try: - from mutagen import File as MutagenFile - audio = MutagenFile(out_path) - if audio is not None: - if codec == 'mp3': - from mutagen.id3 import TXXX - audio.tags.add(TXXX(encoding=3, desc='QUALITY', text=[quality_label])) - elif codec == 'opus': - audio['QUALITY'] = [quality_label] - elif codec == 'aac': - from mutagen.mp4 import MP4FreeForm - audio['----:com.apple.iTunes:QUALITY'] = [MP4FreeForm(quality_label.encode('utf-8'))] - audio.save() - except Exception as tag_err: - logger.error(f"[Lossy Copy] Could not update QUALITY tag: {tag_err}") - - # Embed cover art from source FLAC into the lossy copy - # Opus/OGG can't inherit FLAC cover art via ffmpeg -map_metadata alone - if codec in ('opus', 'aac'): - try: - from mutagen import File as MutagenFile - from mutagen.flac import FLAC as MutagenFLAC - source_audio = MutagenFLAC(final_path) - pic = None - if source_audio and source_audio.pictures: - pic = source_audio.pictures[0] - - # Fallback: read cover.jpg from the same directory - if not pic: - cover_path = os.path.join(os.path.dirname(final_path), 'cover.jpg') - if os.path.isfile(cover_path): - try: - from mutagen.flac import Picture - with open(cover_path, 'rb') as f: - img_data = f.read() - pic = Picture() - pic.type = 3 # Cover (front) - pic.mime = 'image/jpeg' - pic.desc = 'Cover' - pic.width = 0 - pic.height = 0 - pic.depth = 0 - pic.colors = 0 - pic.data = img_data - logger.warning("[Lossy Copy] Using cover.jpg as art source (FLAC had no embedded art)") - except Exception: - pass - - if pic: - dest_audio = MutagenFile(out_path) - if dest_audio is not None: - if codec == 'opus': - import base64 - from mutagen.oggopus import OggOpus - if isinstance(dest_audio, OggOpus): - # OGG stores pictures as base64-encoded METADATA_BLOCK_PICTURE - import struct - # Build METADATA_BLOCK_PICTURE block - picture_data = ( - struct.pack('>II', pic.type, len(pic.mime.encode('utf-8'))) - + pic.mime.encode('utf-8') - + struct.pack('>I', len(pic.desc.encode('utf-8'))) - + pic.desc.encode('utf-8') - + struct.pack('>IIII', pic.width, pic.height, pic.depth, pic.colors) - + struct.pack('>I', len(pic.data)) - + pic.data - ) - dest_audio['METADATA_BLOCK_PICTURE'] = [base64.b64encode(picture_data).decode('ascii')] - dest_audio.save() - logger.info("[Lossy Copy] Embedded cover art in Opus file") - elif codec == 'aac': - from mutagen.mp4 import MP4Cover - fmt = MP4Cover.FORMAT_JPEG if 'jpeg' in pic.mime else MP4Cover.FORMAT_PNG - dest_audio['covr'] = [MP4Cover(pic.data, imageformat=fmt)] - dest_audio.save() - logger.info("[Lossy Copy] Embedded cover art in M4A file") - except Exception as art_err: - logger.error(f"[Lossy Copy] Could not embed cover art: {art_err}") - - # Blasphemy Mode: delete original FLAC if enabled and output is verified - if config_manager.get('lossy_copy.delete_original', False): - try: - if os.path.isfile(out_path) and os.path.getsize(out_path) > 0: - from mutagen import File as MutagenFile - test_audio = MutagenFile(out_path) - if test_audio is not None: - # Update provenance record to point to the new transcoded file - try: - db = get_database() - db.update_provenance_file_path(final_path, out_path) - except Exception: - pass - os.remove(final_path) - logger.info(f"[Blasphemy Mode] Deleted original: {os.path.basename(final_path)}") - # Rename lyrics sidecar file to match the output filename - for lyrics_ext in ('.lrc', '.txt'): - src_lyrics = os.path.splitext(final_path)[0] + lyrics_ext - if os.path.isfile(src_lyrics): - dst_lyrics = os.path.splitext(out_path)[0] + lyrics_ext - try: - os.rename(src_lyrics, dst_lyrics) - logger.info(f"[Blasphemy Mode] Renamed {lyrics_ext}: {os.path.basename(src_lyrics)} -> {os.path.basename(dst_lyrics)}") - except Exception as lrc_err: - logger.error(f"[Blasphemy Mode] Could not rename {lyrics_ext}: {lrc_err}") - return out_path - else: - logger.error(f"[Blasphemy Mode] Output failed audio validation, keeping original: {os.path.basename(final_path)}") - else: - logger.warning(f"[Blasphemy Mode] Output missing or empty, keeping original: {os.path.basename(final_path)}") - except Exception as del_err: - logger.error(f"[Blasphemy Mode] Error during original deletion, keeping original: {del_err}") - else: - # ffmpeg always prints its version banner to stderr (~300 chars). - # Strip it so the actual error is visible, and show more than 200 chars. - stderr = result.stderr or '' - # Remove the version/config preamble (ends after the first empty line) - stderr_lines = stderr.split('\n') - error_lines = [] - past_banner = False - for line in stderr_lines: - if past_banner: - error_lines.append(line) - elif line.strip() == '': - past_banner = True - error_msg = '\n'.join(error_lines).strip() if error_lines else stderr[-500:] - logger.error(f"[Lossy Copy] ffmpeg failed (exit code {result.returncode}): {error_msg[:500]}") - # Clean up empty/broken output file - if os.path.isfile(out_path) and os.path.getsize(out_path) == 0: - os.remove(out_path) - logger.warning(f"[Lossy Copy] Removed empty output file: {os.path.basename(out_path)}") - except subprocess.TimeoutExpired: - logger.info(f"[Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}") - if os.path.isfile(out_path) and os.path.getsize(out_path) == 0: - os.remove(out_path) - except Exception as e: - logger.error(f"[Lossy Copy] Conversion error: {e}") - return None def _get_album_type_display(raw_type, track_count) -> str: """ @@ -19096,516 +18084,10 @@ from mutagen.apev2 import APEv2, APENoHeaderError import urllib.request def _wipe_source_tags(file_path: str) -> bool: - """Emergency tag wipe — clears ALL tags from a file without writing new ones. - Used when full metadata enhancement is skipped or fails, to prevent original - Soulseek source tags (especially MusicBrainz IDs from the uploader) from - persisting and causing album splits in media servers like Navidrome.""" - try: - _strip_all_non_audio_tags(file_path) - audio = MutagenFile(file_path) - if audio is None: - return False - if hasattr(audio, 'clear_pictures'): - audio.clear_pictures() - if audio.tags is not None: - tag_count = len(audio.tags) - audio.tags.clear() - else: - audio.add_tags() - tag_count = 0 - if isinstance(audio.tags, ID3): - audio.save(v1=0, v2_version=4) - elif isinstance(audio, FLAC): - audio.save(deleteid3=True) - else: - audio.save() - if tag_count > 0: - logger.info(f"[Tag Wipe] Stripped {tag_count} source tags from: {os.path.basename(file_path)}") - return True - except Exception as e: - logger.error(f"[Tag Wipe] Failed (non-fatal): {e}") - return False - - -def _strip_all_non_audio_tags(file_path: str) -> dict: - """ - Strip ALL non-audio tag containers from a file before metadata rewriting. - MP3 files from Soulseek commonly carry APEv2 tags (foobar2000 users) - with stale metadata that Mutagen's ID3 handler cannot see or clear. - Must run BEFORE MutagenFile() opens the file. - """ - summary = {'apev2_stripped': False, 'apev2_tag_count': 0} - ext = os.path.splitext(file_path)[1].lower() - if ext != '.mp3': - return summary - try: - apev2_tags = APEv2(file_path) - tag_count = len(apev2_tags) - tag_keys = list(apev2_tags.keys()) - apev2_tags.delete(file_path) - summary['apev2_stripped'] = True - summary['apev2_tag_count'] = tag_count - logger.info(f"Stripped {tag_count} APEv2 tags: {', '.join(tag_keys[:10])}") - except APENoHeaderError: - pass # No APEv2 tags — common case - except Exception as e: - logger.error(f"Could not strip APEv2 tags (non-fatal): {e}") - return summary - -def _verify_metadata_written(file_path: str) -> bool: - """Re-open file and verify core metadata fields are present.""" - try: - check = MutagenFile(file_path) - if check is None or check.tags is None: - logger.info(f"[VERIFY] Tags are None after save: {file_path}") - return False - title_found = False - artist_found = False - if isinstance(check.tags, ID3): - title_found = bool(check.tags.getall('TIT2')) - artist_found = bool(check.tags.getall('TPE1')) - # Confirm APEv2 is gone - try: - APEv2(file_path) - logger.info("[VERIFY] APEv2 tags still present after processing!") - return False - except APENoHeaderError: - pass - elif isinstance(check, (FLAC, OggVorbis)) or _is_ogg_opus(check): - title_found = bool(check.get('title')) - artist_found = bool(check.get('artist')) - elif isinstance(check, MP4): - title_found = bool(check.get('\xa9nam')) - artist_found = bool(check.get('\xa9ART')) - if not title_found or not artist_found: - logger.warning(f"[VERIFY] Missing metadata - title:{title_found} artist:{artist_found}") - return False - logger.info("[VERIFY] Metadata verified OK") - return True - except Exception as e: - logger.error(f"[VERIFY] Verification error (non-fatal): {e}") - return False - -def _is_ogg_opus(audio_file): - """Check if a Mutagen file object is OggOpus (uses VorbisComment tags like FLAC/OGG).""" - return type(audio_file).__name__ == 'OggOpus' + return metadata_enrichment.wipe_source_tags(file_path) def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: - """ - Core function to enhance audio file metadata using Spotify data. - Thread-safe with per-file locking to prevent concurrent metadata writes. - - Opens the file once in non-easy mode, clears all tags in memory, writes - new tags using format-specific frames/keys, embeds album art and source - IDs, then saves once. This avoids the old clear→save→reopen pattern - which stripped the ID3v2 header from MP3 files, leaving them tagless. - """ - if not config_manager.get('metadata_enhancement.enabled', True): - logger.warning("Metadata enhancement disabled in config.") - return True - - # Normalize None album_info to empty dict to prevent AttributeError on .get() calls - if album_info is None: - album_info = {} - - # Acquire per-file lock to prevent concurrent metadata writes to the same file - file_lock = _get_file_lock(file_path) - with file_lock: - logger.info(f"Enhancing metadata for: {os.path.basename(file_path)}") - try: - # Strip APEv2 tags from MP3 (invisible to ID3 handler) - strip_summary = _strip_all_non_audio_tags(file_path) - - audio_file = MutagenFile(file_path) - if audio_file is None: - logger.error(f"Could not load audio file with Mutagen: {file_path}") - return False - - # ── Wipe ALL existing tags and save immediately ── - # Files from Soulseek carry random metadata (wrong comments, - # encoder info, ReplayGain, old album art, random TXXX frames). - # Save the cleared state FIRST so that if anything below throws, - # the file at least has clean (empty) tags instead of junk that - # causes album fragmentation in media servers. - if hasattr(audio_file, 'clear_pictures'): - audio_file.clear_pictures() - - if audio_file.tags is not None: - if len(audio_file.tags) > 0: - tag_keys = list(audio_file.tags.keys())[:15] - logger.info(f"Clearing {len(audio_file.tags)} existing tags: " - f"{', '.join(str(k) for k in tag_keys)}") - audio_file.tags.clear() - else: - audio_file.add_tags() - - # Persist the wipe — guarantees junk tags are gone even if later steps fail - if isinstance(audio_file.tags, ID3): - audio_file.save(v1=0, v2_version=4) - elif isinstance(audio_file, FLAC): - audio_file.save(deleteid3=True) - else: - audio_file.save() - - metadata = _extract_spotify_metadata(context, artist, album_info) - if not metadata: - logger.error("Could not extract Spotify metadata, saving with cleared tags.") - if isinstance(audio_file.tags, ID3): - audio_file.save(v1=0, v2_version=4) - elif isinstance(audio_file, FLAC): - audio_file.save(deleteid3=True) - else: - audio_file.save() - return True - - # ── Write standard tags using format-specific API ── - track_num_str = f"{metadata.get('track_number', 1)}/{metadata.get('total_tracks', 1)}" - - _write_multi = config_manager.get('metadata_enhancement.tags.write_multi_artist', False) - _artists_list = metadata.get('_artists_list', []) - - if isinstance(audio_file.tags, ID3): - # MP3: write ID3 frames directly - if metadata.get('title'): - audio_file.tags.add(TIT2(encoding=3, text=[metadata['title']])) - if metadata.get('artist'): - audio_file.tags.add(TPE1(encoding=3, text=[metadata['artist']])) - # Multi-value: write each artist as separate TPE1 text value - if _write_multi and len(_artists_list) > 1: - audio_file.tags.add(TPE1(encoding=3, text=_artists_list)) - if metadata.get('album_artist'): - audio_file.tags.add(TPE2(encoding=3, text=[metadata['album_artist']])) - if metadata.get('album'): - audio_file.tags.add(TALB(encoding=3, text=[metadata['album']])) - if metadata.get('date'): - audio_file.tags.add(TDRC(encoding=3, text=[metadata['date']])) - if metadata.get('genre'): - audio_file.tags.add(TCON(encoding=3, text=[metadata['genre']])) - audio_file.tags.add(TRCK(encoding=3, text=[track_num_str])) - if metadata.get('disc_number'): - audio_file.tags.add(TPOS(encoding=3, text=[str(metadata['disc_number'])])) - - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - # FLAC / OGG Vorbis / OGG Opus: dict-style VorbisComment tags - if metadata.get('title'): - audio_file['title'] = [metadata['title']] - if metadata.get('artist'): - audio_file['artist'] = [metadata['artist']] - # Multi-value: write ARTISTS tag with individual values - if _write_multi and len(_artists_list) > 1: - audio_file['artists'] = _artists_list - if metadata.get('album_artist'): - audio_file['albumartist'] = [metadata['album_artist']] - if metadata.get('album'): - audio_file['album'] = [metadata['album']] - if metadata.get('date'): - audio_file['date'] = [metadata['date']] - if metadata.get('genre'): - audio_file['genre'] = [metadata['genre']] - audio_file['tracknumber'] = [track_num_str] - if metadata.get('disc_number'): - audio_file['discnumber'] = [str(metadata['disc_number'])] - - elif isinstance(audio_file, MP4): - # MP4 / M4A: Apple-style tag keys - if metadata.get('title'): - audio_file['\xa9nam'] = [metadata['title']] - if metadata.get('artist'): - # Multi-value: write each artist as separate list entry - if _write_multi and len(_artists_list) > 1: - audio_file['\xa9ART'] = _artists_list - else: - audio_file['\xa9ART'] = [metadata['artist']] - if metadata.get('album_artist'): - audio_file['aART'] = [metadata['album_artist']] - if metadata.get('album'): - audio_file['\xa9alb'] = [metadata['album']] - if metadata.get('date'): - audio_file['\xa9day'] = [metadata['date']] - if metadata.get('genre'): - audio_file['\xa9gen'] = [metadata['genre']] - track_num = metadata.get('track_number', 1) - total_tracks = metadata.get('total_tracks', 1) - audio_file['trkn'] = [(track_num, total_tracks)] - if metadata.get('disc_number'): - audio_file['disk'] = [(metadata['disc_number'], 0)] - - # ── Embed source IDs (Spotify, MusicBrainz, etc.) on the same object ── - # Runs before album art so MusicBrainz release ID is available for - # Cover Art Archive high-resolution lookup. - _embed_source_ids(audio_file, metadata, context) - - # Propagate MusicBrainz release ID to album_info so _download_cover_art - # can use it for Cover Art Archive high-res cover.jpg - if album_info is not None and metadata.get('musicbrainz_release_id'): - album_info['musicbrainz_release_id'] = metadata['musicbrainz_release_id'] - - # ── Embed album art on the same object ── - if config_manager.get('metadata_enhancement.embed_album_art', True): - _embed_album_art_metadata(audio_file, metadata) - - # ── Embed audio quality tag ── - quality = context.get('_audio_quality', '') - if quality and config_manager.get('metadata_enhancement.tags.quality_tag', True) is not False: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='QUALITY', text=[quality])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['quality'] = [quality] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:QUALITY'] = [MP4FreeForm(quality.encode('utf-8'))] - - # ── Single save for everything ── - if isinstance(audio_file.tags, ID3): - audio_file.save(v1=0, v2_version=4) - elif isinstance(audio_file, FLAC): - audio_file.save(deleteid3=True) - else: - audio_file.save() - - # Verify metadata was written - verified = _verify_metadata_written(file_path) - if verified: - logger.info("Metadata enhanced successfully.") - else: - logger.info("Metadata saved but verification found issues (see above).") - return True - except Exception as e: - import traceback - logger.error(f"Error enhancing metadata for {file_path}: {e}") - logger.error(f"[Metadata Debug] Exception type: {type(e).__name__}") - logger.info(f"[Metadata Debug] File exists: {os.path.exists(file_path)}") - logger.warning(f"[Metadata Debug] Artist: {artist.get('name', 'MISSING') if artist else 'None'}") - logger.warning(f"[Metadata Debug] Album info: {album_info.get('album_name', 'MISSING') if album_info else 'None'}") - logger.error(f"[Metadata Debug] Traceback:\n{traceback.format_exc()}") - return False - -def _generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool: - """ - Generate LRC lyrics file using LRClib API. - Elegant addition to post-processing - extracts metadata from existing context. - """ - if not config_manager.get('metadata_enhancement.lrclib_enabled', True): - return False - try: - # Extract track information from existing context (same as metadata enhancement) - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album") - - # Get track metadata - track_name = (original_search.get('spotify_clean_title') or - original_search.get('title', 'Unknown Track')) - - # Handle artist parameter (can be dict or object) - if isinstance(artist, dict): - artist_name = artist.get('name', 'Unknown Artist') - elif hasattr(artist, 'name'): - artist_name = artist.name - else: - artist_name = str(artist) if artist else 'Unknown Artist' - album_name = None - duration_seconds = None - - # Get album name if available - if album_info.get('is_album'): - album_name = (original_search.get('spotify_clean_album') or - album_info.get('album_name') or - (spotify_album.get('name') if spotify_album else None)) - - # Get duration from original search context - if original_search.get('duration_ms'): - duration_seconds = int(original_search['duration_ms'] / 1000) - - # Generate LRC file using lyrics client - success = lyrics_client.create_lrc_file( - audio_file_path=file_path, - track_name=track_name, - artist_name=artist_name, - album_name=album_name, - duration_seconds=duration_seconds - ) - - if success: - logger.info(f"LRC file generated for: {track_name}") - else: - logger.warning(f"No lyrics found for: {track_name}") - - return success - - except Exception as e: - logger.error(f"Error generating LRC file for {file_path}: {e}") - return False - -def _extract_spotify_metadata(context: dict, artist: dict, album_info: dict) -> dict: - """Extracts a comprehensive metadata dictionary from the provided context.""" - metadata = {} - if album_info is None: - album_info = {} - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album") - - # Priority 1: Spotify clean title from context - if original_search.get('spotify_clean_title'): - metadata['title'] = original_search['spotify_clean_title'] - logger.info(f"Metadata: Using Spotify clean title: '{metadata['title']}'") - # Priority 2: Album info clean name - elif album_info.get('clean_track_name'): - metadata['title'] = album_info['clean_track_name'] - logger.info(f"Metadata: Using album info clean name: '{metadata['title']}'") - # Priority 3: Original title as fallback - else: - metadata['title'] = original_search.get('title', '') - logger.warning(f"Metadata: Using original title as fallback: '{metadata['title']}'") - # Handle multiple artists from Spotify data - original_search = context.get("original_search_result", {}) - if 'artists' in original_search and isinstance(original_search['artists'], list) and len(original_search['artists']) > 0: - all_artists = [] - for a in original_search['artists']: - if isinstance(a, dict) and 'name' in a: - all_artists.append(a['name']) - elif isinstance(a, str): - all_artists.append(a) - else: - all_artists.append(str(a)) - metadata['artist'] = ', '.join(all_artists) - logger.info(f"Metadata: Using all artists: '{metadata['artist']}'") - else: - # Fallback to single artist - metadata['artist'] = artist.get('name', '') - logger.info(f"Metadata: Using primary artist: '{metadata['artist']}'") - - # Resolve album_artist for consistent tagging across all tracks in an album. - # Priority: 1) explicit batch artist context (same artist for whole album) - # 2) album-level artists from spotify_album - # 3) context-level artist (spotify_artist parameter) - # 4) collab mode first-artist resolution (per-track, last resort) - # Using album-level artist prevents media server album splits when an artist - # changed names (Kanye West → Ye) and Spotify returns different per-track artists. - _raw_album_artist = artist.get('name', '') - _track_info_ctx = context.get('track_info', {}) or {} - _explicit_aa = _track_info_ctx.get('_explicit_artist_context') if isinstance(_track_info_ctx, dict) else None - - # Build album-level artists list for collab mode resolution. - # Using album-level artists (instead of per-track) ensures collab mode produces - # the SAME album_artist tag for every track, preventing media server album splits. - _album_artists_for_collab = None - if isinstance(_explicit_aa, dict) and _explicit_aa.get('name'): - _raw_album_artist = _explicit_aa['name'] - _album_artists_for_collab = [_explicit_aa] - elif isinstance(_explicit_aa, str) and _explicit_aa: - _raw_album_artist = _explicit_aa - _album_artists_for_collab = [{'name': _explicit_aa}] - elif spotify_album and isinstance(spotify_album, dict): - _sa_aa = spotify_album.get('artists', []) - if _sa_aa: - _first_aa = _sa_aa[0] - if isinstance(_first_aa, dict) and _first_aa.get('name'): - _raw_album_artist = _first_aa['name'] - elif isinstance(_first_aa, str) and _first_aa: - _raw_album_artist = _first_aa - _album_artists_for_collab = _sa_aa - - collab_mode = config_manager.get('file_organization.collab_artist_mode', 'first') - if collab_mode == 'first' and _raw_album_artist: - original_search = context.get("original_search_result", {}) - # Prefer album-level artists for collab resolution (consistent per album) - _ctx_artists = _album_artists_for_collab or original_search.get('artists') or _track_info_ctx.get('artists') or [] - if len(_ctx_artists) > 1: - # Multiple artist objects (Spotify) — use first - first = _ctx_artists[0] - _raw_album_artist = first.get('name', first) if isinstance(first, dict) else str(first) - elif len(_ctx_artists) == 1 and (',' in _raw_album_artist or ' & ' in _raw_album_artist): - # Single combined string (iTunes) — resolve via artist ID - _aid = str(artist.get('id', '')) - _src = original_search.get('_source') or _track_info_ctx.get('_source', '') - if _aid.isdigit() and _src != 'deezer': - try: - resolved = _get_itunes_client().resolve_primary_artist(_aid) - if resolved and resolved != _raw_album_artist: - _raw_album_artist = resolved - except Exception: - pass - metadata['album_artist'] = _raw_album_artist # Crucial for library organization - - if album_info.get('is_album'): - metadata['album'] = album_info.get('album_name', 'Unknown Album') - track_num = album_info.get('track_number', 1) - metadata['track_number'] = track_num - metadata['total_tracks'] = spotify_album.get('total_tracks', 1) if spotify_album else 1 - logger.info(f"[METADATA] Album track - track_number: {track_num}, album: {metadata['album']}") - else: - # SAFEGUARD: If we have spotify_album context, never use track title as album name - # This prevents album tracks from being tagged as singles due to classification errors - if spotify_album and spotify_album.get('name'): - logger.info("[SAFEGUARD] Using spotify_album name instead of track title for album metadata") - metadata['album'] = spotify_album['name'] - # Use corrected track_number from album_info (which should be updated by post-processing) - corrected_track_number = album_info.get('track_number', 1) if album_info else 1 - metadata['track_number'] = corrected_track_number - metadata['total_tracks'] = spotify_album.get('total_tracks', 1) - logger.info(f"[SAFEGUARD] Using track_number: {corrected_track_number}") - else: - metadata['album'] = metadata['title'] # For true singles, album is the title - metadata['track_number'] = 1 - metadata['total_tracks'] = 1 - - # Always write disc_number to overwrite any stale tags from the soulseek source. - # Without this, original disc tags persist and can cause media servers (Plex) to - # split a single album into standard/deluxe based on differing disc numbers. - # Priority: original_search context (from API) > album_info > default to 1 - disc_num = original_search.get('disc_number') - if disc_num is None and album_info: - disc_num = album_info.get('disc_number') - if disc_num is None: - disc_num = 1 - metadata['disc_number'] = disc_num - - if spotify_album and spotify_album.get('release_date'): - metadata['date'] = spotify_album['release_date'][:4] - - if artist.get('genres'): - from core.genre_filter import filter_genres - _genre_list = filter_genres(list(artist['genres'][:2]), config_manager) - if _genre_list: - metadata['genre'] = ', '.join(_genre_list) - - metadata['album_art_url'] = album_info.get('album_image_url') if album_info else None - - # Playlist mode fallback: album_info is None, try to get art from spotify_album context - if not metadata['album_art_url']: - _spa = context.get('spotify_album', {}) - if _spa: - _spa_img = _spa.get('image_url') - if not _spa_img and _spa.get('images'): - _spa_img = _spa['images'][0].get('url') if isinstance(_spa['images'][0], dict) else None - metadata['album_art_url'] = _spa_img - - # Extract source IDs (Spotify or iTunes) for tag embedding - track_info = context.get("track_info", {}) - if track_info and track_info.get('id'): - # Spotify track IDs are alphanumeric strings; iTunes IDs are numeric - # Beatport IDs (beatport_*) are neither — skip them for external ID tagging - track_id = str(track_info['id']) - if track_id.isdigit(): - metadata['itunes_track_id'] = track_id - elif not track_id.startswith('beatport_'): - metadata['spotify_track_id'] = track_id - if artist.get('id'): - artist_id = str(artist['id']) - if artist_id.isdigit(): - metadata['itunes_artist_id'] = artist_id - elif not artist_id.startswith('beatport_'): - metadata['spotify_artist_id'] = artist_id - if spotify_album and spotify_album.get('id'): - album_id = str(spotify_album['id']) - if album_id.isdigit(): - metadata['itunes_album_id'] = album_id - elif not album_id.startswith('beatport_'): - metadata['spotify_album_id'] = album_id - - # Summary log for debugging metadata issues (e.g. wrong album_artist / track_number) - logger.info(f"[Metadata Summary] title='{metadata.get('title')}' | artist='{metadata.get('artist')}' | album_artist='{metadata.get('album_artist')}' | album='{metadata.get('album')}' | track={metadata.get('track_number')}/{metadata.get('total_tracks')} | disc={metadata.get('disc_number')}") - - return metadata + return metadata_enrichment.enhance_file_metadata(file_path, context, artist, album_info) def _get_image_dimensions(data: bytes): """Extract width/height from JPEG or PNG image data without PIL.""" @@ -19633,870 +18115,12 @@ def _get_image_dimensions(data: bytes): return None, None -def _embed_album_art_metadata(audio_file, metadata: dict): - """Downloads and embeds album art — tries Cover Art Archive (full resolution) - first if MusicBrainz release ID is available, falls back to Spotify/iTunes URL.""" - try: - image_data = None - mime_type = None - - # Try Cover Art Archive first (often 1200x1200+, original quality) — opt-in - release_mbid = metadata.get('musicbrainz_release_id') - if release_mbid and config_manager.get('metadata_enhancement.prefer_caa_art', False): - try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" - req = urllib.request.Request(caa_url, headers={'Accept': 'image/*'}) - with urllib.request.urlopen(req, timeout=10) as response: - image_data = response.read() - mime_type = response.info().get_content_type() or 'image/jpeg' - if image_data and len(image_data) > 1000: - logger.info(f"Cover art from Cover Art Archive ({len(image_data) // 1024}KB)") - else: - image_data = None # Too small, likely an error page - except Exception: - image_data = None # Fall through to Spotify/iTunes URL - - # Fallback to Spotify/iTunes/Deezer URL (typically 640x640) - if not image_data: - art_url = metadata.get('album_art_url') - if not art_url: - logger.warning("No album art URL available for embedding.") - return - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - mime_type = response.info().get_content_type() - - if not image_data: - logger.error("Failed to download album art data.") - return - - # MP3 (ID3) - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(APIC(encoding=3, mime=mime_type, type=3, desc='Cover', data=image_data)) - # FLAC - elif isinstance(audio_file, FLAC): - picture = Picture() - picture.data = image_data - picture.type = 3 - picture.mime = mime_type - # Detect actual dimensions from image data - _img_w, _img_h = _get_image_dimensions(image_data) - picture.width = _img_w or 640 - picture.height = _img_h or 640 - picture.depth = 24 - audio_file.add_picture(picture) - # MP4/M4A - elif isinstance(audio_file, MP4): - fmt = MP4Cover.FORMAT_JPEG if 'jpeg' in mime_type else MP4Cover.FORMAT_PNG - audio_file['covr'] = [MP4Cover(image_data, imageformat=fmt)] - - logger.info("Album art successfully embedded.") - except Exception as e: - logger.error(f"Error embedding album art: {e}") - -def _embed_source_ids(audio_file, metadata: dict, context: dict = None): - """ - Lookup MusicBrainz, Deezer, AudioDB, Tidal, Qobuz, Last.fm, and Genius - metadata, then embed them along with Spotify/iTunes source IDs as custom - tags into the audio file. - Tags written: source IDs, BPM (Deezer), mood/style (AudioDB), ISRC - (MB→Deezer→Tidal→Qobuz fallback), copyright (Tidal→Qobuz), - label (Qobuz), URLs (Last.fm/Genius), - and merged genres (Spotify+MB+AudioDB+Last.fm). - One file write, one shot. Concurrent calls are safe — each service has - its own global rate limiter. - Operates on a non-easy-mode MutagenFile object (caller must save). - """ - try: - # ── Per-tag config: maps internal tag name → config path ── - # Each tag can be individually toggled via {service}.tags.{tag_name} - _TAG_CONFIG = { - # Spotify (from metadata, no API call) - 'SPOTIFY_TRACK_ID': 'spotify.tags.track_id', - 'SPOTIFY_ARTIST_ID': 'spotify.tags.artist_id', - 'SPOTIFY_ALBUM_ID': 'spotify.tags.album_id', - # iTunes (from metadata, no API call) - 'ITUNES_TRACK_ID': 'itunes.tags.track_id', - 'ITUNES_ARTIST_ID': 'itunes.tags.artist_id', - 'ITUNES_ALBUM_ID': 'itunes.tags.album_id', - # MusicBrainz IDs - 'MUSICBRAINZ_RECORDING_ID': 'musicbrainz.tags.recording_id', - 'MUSICBRAINZ_ARTIST_ID': 'musicbrainz.tags.artist_id', - 'MUSICBRAINZ_RELEASE_ID': 'musicbrainz.tags.release_id', - 'MUSICBRAINZ_RELEASEGROUPID': 'musicbrainz.tags.release_group_id', - 'MUSICBRAINZ_ALBUMARTISTID': 'musicbrainz.tags.album_artist_id', - 'MUSICBRAINZ_RELEASETRACKID': 'musicbrainz.tags.release_track_id', - # MusicBrainz Release Info - 'RELEASETYPE': 'musicbrainz.tags.release_type', - 'ORIGINALDATE': 'musicbrainz.tags.original_date', - 'RELEASESTATUS': 'musicbrainz.tags.release_status', - 'RELEASECOUNTRY': 'musicbrainz.tags.release_country', - 'BARCODE': 'musicbrainz.tags.barcode', - 'MEDIA': 'musicbrainz.tags.media', - 'TOTALDISCS': 'musicbrainz.tags.total_discs', - 'CATALOGNUMBER': 'musicbrainz.tags.catalog_number', - 'SCRIPT': 'musicbrainz.tags.script', - 'ASIN': 'musicbrainz.tags.asin', - # Deezer - 'DEEZER_TRACK_ID': 'deezer.tags.track_id', - 'DEEZER_ARTIST_ID': 'deezer.tags.artist_id', - # AudioDB - 'AUDIODB_TRACK_ID': 'audiodb.tags.track_id', - # Tidal - 'TIDAL_TRACK_ID': 'tidal.tags.track_id', - 'TIDAL_ARTIST_ID': 'tidal.tags.artist_id', - # Qobuz - 'QOBUZ_TRACK_ID': 'qobuz.tags.track_id', - 'QOBUZ_ARTIST_ID': 'qobuz.tags.artist_id', - # Genius - 'GENIUS_TRACK_ID': 'genius.tags.track_id', - } - - def _tag_enabled(config_path): - """Check if an individual tag is enabled (defaults to True).""" - return config_manager.get(config_path, True) is not False - - # ── Helper: normalize + compare names (same logic as enrichment workers) ── - from difflib import SequenceMatcher - def _names_match(a: str, b: str, threshold: float = 0.75) -> bool: - if not a or not b: - return False - norm = lambda s: re.sub(r'[^a-z0-9 ]', '', re.sub(r'\(.*?\)', '', s).lower()).strip() - return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold - - # ── 1. Collect Spotify / iTunes IDs already in metadata ── - id_tags = {} - if config_manager.get('spotify.embed_tags', True) is not False: - if metadata.get('spotify_track_id'): - id_tags['SPOTIFY_TRACK_ID'] = metadata['spotify_track_id'] - if metadata.get('spotify_artist_id'): - id_tags['SPOTIFY_ARTIST_ID'] = metadata['spotify_artist_id'] - if metadata.get('spotify_album_id'): - id_tags['SPOTIFY_ALBUM_ID'] = metadata['spotify_album_id'] - if config_manager.get('itunes.embed_tags', True) is not False: - if metadata.get('itunes_track_id'): - id_tags['ITUNES_TRACK_ID'] = metadata['itunes_track_id'] - if metadata.get('itunes_artist_id'): - id_tags['ITUNES_ARTIST_ID'] = metadata['itunes_artist_id'] - if metadata.get('itunes_album_id'): - id_tags['ITUNES_ALBUM_ID'] = metadata['itunes_album_id'] - - # Shared post-processing context for modular lookups - track_title = metadata.get('title', '') - artist_name = metadata.get('album_artist', '') or metadata.get('artist', '') - - # Extract batch-level artist name for stable MB release cache keys. - # When downloading an album batch, all tracks should use the same artist key - # to guarantee they hit the same preflight-cached release MBID. - _track_info_for_pp = (context or {}).get('track_info', {}) or {} - _explicit_artist_for_pp = _track_info_for_pp.get('_explicit_artist_context') if isinstance(_track_info_for_pp, dict) else None - _batch_artist_name = None - if isinstance(_explicit_artist_for_pp, dict) and _explicit_artist_for_pp.get('name'): - _batch_artist_name = _explicit_artist_for_pp['name'] - elif isinstance(_explicit_artist_for_pp, str) and _explicit_artist_for_pp: - _batch_artist_name = _explicit_artist_for_pp - - pp = { - 'id_tags': id_tags, - 'track_title': track_title, - 'artist_name': artist_name, - 'batch_artist_name': _batch_artist_name, - 'metadata': metadata, - 'recording_mbid': None, - 'artist_mbid': None, - 'release_mbid': '', - 'mb_genres': [], - 'isrc': None, - 'deezer_bpm': None, 'deezer_isrc': None, - 'audiodb_mood': None, 'audiodb_style': None, 'audiodb_genre': None, - 'tidal_isrc': None, 'tidal_copyright': None, - 'qobuz_isrc': None, 'qobuz_copyright': None, 'qobuz_label': None, - 'lastfm_tags': [], 'lastfm_url': None, - 'genius_url': None, - 'release_year': None, # First source to find a year wins - } - - # Run each metadata source lookup in configured order - _pp_source_order = config_manager.get('metadata_enhancement.post_process_order', None) - if not _pp_source_order or not isinstance(_pp_source_order, list): - _pp_source_order = ['musicbrainz', 'deezer', 'audiodb', 'tidal', 'qobuz', 'lastfm', 'genius'] - - _pp_lookup_map = { - 'musicbrainz': _pp_lookup_musicbrainz, - 'deezer': _pp_lookup_deezer, - 'audiodb': _pp_lookup_audiodb, - 'tidal': _pp_lookup_tidal, - 'qobuz': _pp_lookup_qobuz, - 'lastfm': _pp_lookup_lastfm, - 'genius': _pp_lookup_genius, - } - - for source_name in _pp_source_order: - fn = _pp_lookup_map.get(source_name) - if fn: - fn(pp, _names_match) - - # Extract results from shared context after all lookups - recording_mbid = pp['recording_mbid'] - artist_mbid = pp['artist_mbid'] - _rc_mbid = pp['release_mbid'] - mb_genres = pp['mb_genres'] - isrc = pp['isrc'] - deezer_bpm = pp['deezer_bpm'] - deezer_isrc = pp['deezer_isrc'] - audiodb_mood = pp['audiodb_mood'] - audiodb_style = pp['audiodb_style'] - audiodb_genre = pp['audiodb_genre'] - tidal_isrc = pp['tidal_isrc'] - tidal_copyright = pp['tidal_copyright'] - qobuz_isrc = pp['qobuz_isrc'] - qobuz_copyright = pp['qobuz_copyright'] - qobuz_label = pp['qobuz_label'] - lastfm_tags = pp['lastfm_tags'] - lastfm_url = pp['lastfm_url'] - genius_url = pp['genius_url'] - id_tags = pp['id_tags'] - release_year = pp['release_year'] - - # If metadata already has a date from Spotify context, use that as fallback - if not release_year and metadata.get('date'): - yr = str(metadata['date'])[:4] - if yr.isdigit(): - release_year = yr - - # Store release MBID in metadata for downstream use (e.g. Cover Art Archive) - if _rc_mbid: - metadata['musicbrainz_release_id'] = _rc_mbid - - # Write release year to file tags if not already present - if release_year and 'ORIGINALDATE' not in id_tags: - id_tags['ORIGINALDATE'] = release_year - # If the file was written without a date tag, flag it for writing below - _needs_date_tag = release_year and not metadata.get('date') - if _needs_date_tag: - metadata['date'] = release_year - - # Update DB album year if currently missing - if release_year: - try: - _pp_album_name = metadata.get('album', '') - _pp_artist_name = metadata.get('album_artist', '') or metadata.get('artist', '') - if _pp_album_name and _pp_artist_name: - conn = get_database()._get_connection() - try: - cursor = conn.cursor() - cursor.execute(""" - UPDATE albums SET year = ? - WHERE (year IS NULL OR year = 0) - AND id IN ( - SELECT al.id FROM albums al - JOIN artists ar ON ar.id = al.artist_id - WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?) - ) - """, (int(release_year), _pp_album_name, _pp_artist_name)) - if cursor.rowcount > 0: - conn.commit() - logger.info(f"Updated album year to {release_year} in database") - else: - conn.rollback() - finally: - conn.close() - except Exception as e: - logger.error(f"Could not update album year in DB: {e}") - - # (All source lookups now handled by _pp_lookup_* functions called via configurable order above) - if False: # Dead code — old inline blocks preserved for reference during transition - try: - mb_service_for_detail = mb_worker.mb_service if mb_worker else None - if mb_service_for_detail: - with _mb_release_detail_cache_lock: - release_detail = _mb_release_detail_cache.get(_rc_mbid) - if release_detail is None: - release_detail = mb_service_for_detail.mb_client.get_release( - _rc_mbid, includes=['release-groups', 'labels', 'media', 'artist-credits', 'recordings'] - ) or {} - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[_rc_mbid] = release_detail - if release_detail: - rg = release_detail.get('release-group', {}) - if rg.get('id'): - id_tags['MUSICBRAINZ_RELEASEGROUPID'] = rg['id'] - ac = release_detail.get('artist-credit', []) - if ac and isinstance(ac[0], dict): - aa_artist = ac[0].get('artist', {}) - if aa_artist.get('id'): - id_tags['MUSICBRAINZ_ALBUMARTISTID'] = aa_artist['id'] - primary_type = rg.get('primary-type', '') - if primary_type: - id_tags['RELEASETYPE'] = primary_type - orig_date = rg.get('first-release-date', '') - if orig_date: - id_tags['ORIGINALDATE'] = orig_date - status = release_detail.get('status', '') - if status: - id_tags['RELEASESTATUS'] = status - country = release_detail.get('country', '') - if country: - id_tags['RELEASECOUNTRY'] = country - barcode = release_detail.get('barcode', '') - if barcode: - id_tags['BARCODE'] = barcode - media_list = release_detail.get('media', []) - if media_list: - media_format = media_list[0].get('format', '') - if media_format: - id_tags['MEDIA'] = media_format - id_tags['TOTALDISCS'] = str(len(media_list)) - label_info = release_detail.get('label-info', []) - if label_info and isinstance(label_info[0], dict): - cat_num = label_info[0].get('catalog-number', '') - if cat_num: - id_tags['CATALOGNUMBER'] = cat_num - text_rep = release_detail.get('text-representation', {}) - if isinstance(text_rep, dict) and text_rep.get('script'): - id_tags['SCRIPT'] = text_rep['script'] - asin = release_detail.get('asin', '') - if asin: - id_tags['ASIN'] = asin - # Release Track ID — match by disc + track position - _trk_num = metadata.get('track_number') - _disc_num = metadata.get('disc_number') or 1 - if _trk_num and media_list: - try: - _trk_num_int = int(_trk_num) - _disc_num_int = int(_disc_num) - for medium in media_list: - if medium.get('position', 1) == _disc_num_int: - for mtrack in (medium.get('tracks') or medium.get('track-list', [])): - if mtrack.get('position') == _trk_num_int and mtrack.get('id'): - id_tags['MUSICBRAINZ_RELEASETRACKID'] = mtrack['id'] - break - break - except (ValueError, TypeError): - pass - logger.info(f"MusicBrainz release details: type={primary_type or '?'}, " - f"country={country or '?'}, media={id_tags.get('MEDIA', '?')}") - except Exception as e: - logger.error(f"MusicBrainz release detail lookup failed (non-fatal): {e}") - - # ── 2b. Deezer lookup for BPM, ISRC fallback, and source IDs ── - deezer_bpm = None - deezer_isrc = None - if not config_manager.get('deezer.embed_tags', True): - pass - elif track_title and artist_name: - try: - dz_client = deezer_worker.client if deezer_worker else None - if dz_client: - dz_result = dz_client.search_track(artist_name, track_title) - if dz_result and _names_match(dz_result.get('title', ''), track_title) and \ - _names_match(dz_result.get('artist', {}).get('name', ''), artist_name): - dz_track_id = dz_result['id'] - id_tags['DEEZER_TRACK_ID'] = str(dz_track_id) - dz_artist_id = dz_result.get('artist', {}).get('id') - if dz_artist_id: - id_tags['DEEZER_ARTIST_ID'] = str(dz_artist_id) - logger.info(f"Deezer track matched: {dz_track_id}") - - # Get full track details for BPM and ISRC - dz_details = dz_client.get_track_details(dz_track_id) - if dz_details: - bpm_val = dz_details.get('bpm') - if bpm_val and bpm_val > 0: - deezer_bpm = bpm_val - dz_isrc = dz_details.get('isrc') - if dz_isrc: - deezer_isrc = dz_isrc - else: - logger.info("Deezer worker not available, skipping Deezer lookup") - except Exception as e: - logger.error(f"Deezer lookup failed (non-fatal): {e}") - - # ── 2c. AudioDB lookup for mood, style, genre, and source ID ── - audiodb_mood = None - audiodb_style = None - audiodb_genre = None - if not config_manager.get('audiodb.embed_tags', True): - pass - elif track_title and artist_name: - try: - adb_client = audiodb_worker.client if audiodb_worker else None - if adb_client: - adb_result = adb_client.search_track(artist_name, track_title) - if adb_result and _names_match(adb_result.get('strTrack', ''), track_title) and \ - _names_match(adb_result.get('strArtist', ''), artist_name): - adb_track_id = adb_result.get('idTrack') - if adb_track_id: - id_tags['AUDIODB_TRACK_ID'] = str(adb_track_id) - logger.info(f"AudioDB track matched: {adb_track_id}") - # Use AudioDB's MusicBrainz IDs as fallbacks for any missing from MB lookup - adb_mb_track = adb_result.get('strMusicBrainzID') - if adb_mb_track and 'MUSICBRAINZ_RECORDING_ID' not in id_tags: - id_tags['MUSICBRAINZ_RECORDING_ID'] = adb_mb_track - recording_mbid = adb_mb_track - logger.warning(f"MusicBrainz recording ID from AudioDB fallback: {adb_mb_track}") - # NOTE: AudioDB's strMusicBrainzAlbumID is intentionally - # NOT used as a fallback for MUSICBRAINZ_RELEASE_ID. - # AudioDB links each track to its original album in MB, - # which differs per track on compilations and splits - # albums in players like Navidrome. Album MBID must come - # from match_release (cached) to stay consistent. - adb_mb_artist = adb_result.get('strMusicBrainzArtistID') - if adb_mb_artist and 'MUSICBRAINZ_ARTIST_ID' not in id_tags: - id_tags['MUSICBRAINZ_ARTIST_ID'] = adb_mb_artist - artist_mbid = adb_mb_artist - logger.warning(f"MusicBrainz artist ID from AudioDB fallback: {adb_mb_artist}") - audiodb_mood = adb_result.get('strMood') or None - audiodb_style = adb_result.get('strStyle') or None - audiodb_genre = adb_result.get('strGenre') or None - else: - logger.info("AudioDB worker not available, skipping AudioDB lookup") - except Exception as e: - logger.error(f"AudioDB lookup failed (non-fatal): {e}") - - # ── 2d. Tidal lookup for ISRC fallback, copyright, and source IDs ── - tidal_isrc = None - tidal_copyright = None - if not config_manager.get('tidal.embed_tags', True): - pass - elif track_title and artist_name: - try: - if tidal_client and tidal_client.is_authenticated(): - td_result = tidal_client.search_track(artist_name, track_title) - if td_result and _names_match(td_result.get('title', ''), track_title): - td_track_id = td_result.get('id') - if td_track_id: - id_tags['TIDAL_TRACK_ID'] = str(td_track_id) - logger.info(f"Tidal track matched: {td_track_id}") - td_artist = td_result.get('artist', {}) - if isinstance(td_artist, dict) and td_artist.get('id'): - id_tags['TIDAL_ARTIST_ID'] = str(td_artist['id']) - # Get full details for ISRC and copyright - if td_track_id: - td_details = tidal_client.get_track(str(td_track_id)) - if td_details: - td_isrc = td_details.get('isrc') - if td_isrc: - tidal_isrc = td_isrc - td_copyright = td_details.get('copyright') - if isinstance(td_copyright, dict): - td_copyright = td_copyright.get('text', td_copyright.get('name', '')) - if td_copyright: - tidal_copyright = td_copyright - except Exception as e: - logger.error(f"Tidal lookup failed (non-fatal): {e}") - - # ── 2e. Qobuz lookup for ISRC fallback, copyright, label, and source IDs ── - qobuz_isrc = None - qobuz_copyright = None - qobuz_label = None - if not config_manager.get('qobuz.embed_tags', True): - pass - elif track_title and artist_name: - try: - qz_client = qobuz_enrichment_worker.client if qobuz_enrichment_worker else None - if qz_client and qz_client.is_authenticated(): - qz_result = qz_client.search_track(artist_name, track_title) - if qz_result: - qz_performer = (qz_result.get('performer') or {}) - if not isinstance(qz_performer, dict): - qz_performer = {} - qz_artist_name = qz_performer.get('name', '') - if _names_match(qz_result.get('title', ''), track_title) and \ - _names_match(qz_artist_name, artist_name): - qz_track_id = qz_result.get('id') - if qz_track_id: - id_tags['QOBUZ_TRACK_ID'] = str(qz_track_id) - logger.info(f"Qobuz track matched: {qz_track_id}") - if isinstance(qz_performer, dict) and qz_performer.get('id'): - id_tags['QOBUZ_ARTIST_ID'] = str(qz_performer['id']) - qz_isrc = qz_result.get('isrc') - if isinstance(qz_isrc, dict): - qz_isrc = qz_isrc.get('value', qz_isrc.get('id', '')) - if qz_isrc: - qobuz_isrc = qz_isrc - qz_copyright = qz_result.get('copyright') - if isinstance(qz_copyright, dict): - qz_copyright = qz_copyright.get('text', qz_copyright.get('name', '')) - if qz_copyright and isinstance(qz_copyright, str): - qobuz_copyright = qz_copyright - qz_album = qz_result.get('album', {}) - if isinstance(qz_album, dict): - qz_label_info = qz_album.get('label', {}) - if isinstance(qz_label_info, dict) and qz_label_info.get('name'): - qobuz_label = qz_label_info['name'] - except Exception as e: - logger.error(f"Qobuz lookup failed (non-fatal): {e}") - - # ── 2f. Last.fm lookup for tags (genre merge) and URL ── - lastfm_tags = [] - lastfm_url = None - if not config_manager.get('lastfm.embed_tags', True): - pass - elif track_title and artist_name: - try: - lf_client = lastfm_worker.client if lastfm_worker else None - if lf_client: - lf_result = lf_client.get_track_info(artist_name, track_title) - if lf_result: - lf_url = lf_result.get('url') - if lf_url: - lastfm_url = lf_url - lf_toptags = lf_result.get('toptags', {}) - if isinstance(lf_toptags, dict): - tag_list = lf_toptags.get('tag', []) - if isinstance(tag_list, list): - lastfm_tags = [t.get('name', '') for t in tag_list if isinstance(t, dict) and t.get('name')] - elif isinstance(tag_list, dict) and tag_list.get('name'): - lastfm_tags = [tag_list['name']] - logger.info(f"Last.fm track info found: {len(lastfm_tags)} tags") - except Exception as e: - logger.error(f"Last.fm lookup failed (non-fatal): {e}") - - # ── 2g. Genius lookup for source ID and URL ── - # Genius has an aggressive global rate limiter (30→60→120s backoff) that - # blocks ALL callers including post-processing. We check the backoff - # state directly and skip immediately if Genius is rate-limited, rather - # than entering search_song which would sleep for up to 120s. - genius_url = None - if not config_manager.get('genius.embed_tags', True): - pass - elif track_title and artist_name: - try: - import core.genius_client as _genius_module - if time.time() < _genius_module._rate_limit_until: - logger.info("Genius rate-limited, skipping (non-blocking)") - else: - g_client = genius_worker.client if genius_worker else None - if g_client: - g_result = g_client.search_song(artist_name, track_title) - if g_result: - g_id = g_result.get('id') - if g_id: - id_tags['GENIUS_TRACK_ID'] = str(g_id) - logger.info(f"Genius song matched: {g_id}") - g_url = g_result.get('url') - if g_url: - genius_url = g_url - except Exception as e: - logger.error(f"Genius lookup failed (non-fatal): {e}") - - if not id_tags and not deezer_bpm and not deezer_isrc and not audiodb_mood and not audiodb_style: - return - - # ── 3. Filter tags by per-tag config, then write ── - filtered_tags = {} - for tag_name, value in id_tags.items(): - config_path = _TAG_CONFIG.get(tag_name) - if config_path and not _tag_enabled(config_path): - continue - filtered_tags[tag_name] = value - - # ── 3a. Write ID tags (MusicBrainz, source IDs, release info) ── - written = [] - - # Format-specific tag name mappings for Picard-compatible output - _ID3_TAG_MAP = { - 'MUSICBRAINZ_RECORDING_ID': ('UFID', 'http://musicbrainz.org'), - 'MUSICBRAINZ_ARTIST_ID': ('TXXX', 'MusicBrainz Artist Id'), - 'MUSICBRAINZ_RELEASE_ID': ('TXXX', 'MusicBrainz Album Id'), - 'MUSICBRAINZ_RELEASEGROUPID': ('TXXX', 'MusicBrainz Release Group Id'), - 'MUSICBRAINZ_ALBUMARTISTID': ('TXXX', 'MusicBrainz Album Artist Id'), - 'MUSICBRAINZ_RELEASETRACKID': ('TXXX', 'MusicBrainz Release Track Id'), - 'RELEASETYPE': ('TXXX', 'MusicBrainz Album Type'), - 'RELEASESTATUS': ('TXXX', 'MusicBrainz Album Status'), - 'RELEASECOUNTRY': ('TXXX', 'MusicBrainz Album Release Country'), - 'ORIGINALDATE': ('TDOR', None), - 'MEDIA': ('TMED', None), - } - _VORBIS_TAG_MAP = { - 'MUSICBRAINZ_RECORDING_ID': 'MUSICBRAINZ_TRACKID', - 'MUSICBRAINZ_ARTIST_ID': 'MUSICBRAINZ_ARTISTID', - 'MUSICBRAINZ_RELEASE_ID': 'MUSICBRAINZ_ALBUMID', - 'MUSICBRAINZ_RELEASEGROUPID': 'MUSICBRAINZ_RELEASEGROUPID', - 'MUSICBRAINZ_ALBUMARTISTID': 'MUSICBRAINZ_ALBUMARTISTID', - 'MUSICBRAINZ_RELEASETRACKID': 'MUSICBRAINZ_RELEASETRACKID', - } - _MP4_TAG_MAP = { - 'MUSICBRAINZ_RECORDING_ID': 'MusicBrainz Track Id', - 'MUSICBRAINZ_ARTIST_ID': 'MusicBrainz Artist Id', - 'MUSICBRAINZ_RELEASE_ID': 'MusicBrainz Album Id', - 'MUSICBRAINZ_RELEASEGROUPID': 'MusicBrainz Release Group Id', - 'MUSICBRAINZ_ALBUMARTISTID': 'MusicBrainz Album Artist Id', - 'MUSICBRAINZ_RELEASETRACKID': 'MusicBrainz Release Track Id', - 'RELEASETYPE': 'MusicBrainz Album Type', - 'RELEASESTATUS': 'MusicBrainz Album Status', - 'RELEASECOUNTRY': 'MusicBrainz Album Release Country', - } - - # MP3 (ID3) - if isinstance(audio_file.tags, ID3): - for tag_name, value in filtered_tags.items(): - id3_spec = _ID3_TAG_MAP.get(tag_name) - if id3_spec: - frame_type, desc = id3_spec - if frame_type == 'UFID': - audio_file.tags.add(UFID(owner=desc, data=value.encode('ascii'))) - written.append(f'UFID:{desc}') - elif frame_type == 'TDOR': - audio_file.tags.add(TDOR(encoding=3, text=[value])) - written.append('TDOR') - elif frame_type == 'TMED': - audio_file.tags.add(TMED(encoding=3, text=[value])) - written.append('TMED') - else: # TXXX - audio_file.tags.add(TXXX(encoding=3, desc=desc, text=[value])) - written.append(f'TXXX:{desc}') - else: - audio_file.tags.add(TXXX(encoding=3, desc=tag_name, text=[str(value)])) - written.append(f'TXXX:{tag_name}') - - # FLAC / OGG Vorbis - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - for tag_name, value in filtered_tags.items(): - vorbis_key = _VORBIS_TAG_MAP.get(tag_name, tag_name) - audio_file[vorbis_key] = [str(value)] - written.append(vorbis_key) - - # MP4 (M4A/AAC) - elif isinstance(audio_file, MP4): - for tag_name, value in filtered_tags.items(): - mp4_desc = _MP4_TAG_MAP.get(tag_name, tag_name) - key = f'----:com.apple.iTunes:{mp4_desc}' - audio_file[key] = [MP4FreeForm(str(value).encode('utf-8'))] - written.append(key) - - if written: - logger.info(f"Embedded IDs: {', '.join(written)}") - - # ── 3a½. Write date tag if discovered during lookups (initial write had no date) ── - if _needs_date_tag and release_year: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TDRC(encoding=3, text=[release_year])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['date'] = [release_year] - elif isinstance(audio_file, MP4): - audio_file['\xa9day'] = [release_year] - logger.info(f"Date tag: {release_year}") - - # ── 3b. Write BPM tag (from Deezer) ── - if _tag_enabled('deezer.tags.bpm') and deezer_bpm and deezer_bpm > 0: - bpm_int = int(deezer_bpm) - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TBPM(encoding=3, text=[str(bpm_int)])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['BPM'] = [str(bpm_int)] - elif isinstance(audio_file, MP4): - audio_file['tmpo'] = [bpm_int] - logger.info(f"BPM: {bpm_int}") - - # ── 3c. Write mood tag (from AudioDB) ── - if _tag_enabled('audiodb.tags.mood') and audiodb_mood: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='MOOD', text=[audiodb_mood])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['MOOD'] = [audiodb_mood] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:MOOD'] = [MP4FreeForm(audiodb_mood.encode('utf-8'))] - logger.info(f"Mood: {audiodb_mood}") - - # ── 3d. Write style tag (from AudioDB) ── - if _tag_enabled('audiodb.tags.style') and audiodb_style: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='STYLE', text=[audiodb_style])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['STYLE'] = [audiodb_style] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:STYLE'] = [MP4FreeForm(audiodb_style.encode('utf-8'))] - logger.info(f"Style: {audiodb_style}") - - # ── 4. Merge genres (Spotify + MusicBrainz + AudioDB + Last.fm) and overwrite tag ── - if _tag_enabled('metadata_enhancement.tags.genre_merge'): - enrichment_genres = (mb_genres if _tag_enabled('musicbrainz.tags.genres') else []) + \ - ([audiodb_genre] if audiodb_genre and _tag_enabled('audiodb.tags.genre') else []) + \ - (lastfm_tags if _tag_enabled('lastfm.tags.genres') else []) - if enrichment_genres: - from core.genre_filter import filter_genres as _gf - enrichment_genres = _gf(enrichment_genres, config_manager) - spotify_genres = [g.strip() for g in metadata.get('genre', '').split(',') if g.strip()] - seen = set() - merged = [] - for g in spotify_genres + enrichment_genres: - key = g.strip().lower() - if key and key not in seen: - seen.add(key) - merged.append(g.strip().title()) - if len(merged) >= 5: - break - - if merged: - genre_string = ', '.join(merged) - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TCON(encoding=3, text=[genre_string])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['GENRE'] = [genre_string] - elif isinstance(audio_file, MP4): - audio_file['\xa9gen'] = [genre_string] - logger.info(f"Genres merged: {genre_string}") - - # ── 5. Write ISRC if available (per-source fallback chain) ── - _isrc_candidates = [] - if isrc and _tag_enabled('musicbrainz.tags.isrc'): - _isrc_candidates.append(('MusicBrainz', isrc)) - if deezer_isrc and _tag_enabled('deezer.tags.isrc'): - _isrc_candidates.append(('Deezer', deezer_isrc)) - if tidal_isrc and _tag_enabled('tidal.tags.isrc'): - _isrc_candidates.append(('Tidal', tidal_isrc)) - if qobuz_isrc and _tag_enabled('qobuz.tags.isrc'): - _isrc_candidates.append(('Qobuz', qobuz_isrc)) - if _isrc_candidates: - source, final_isrc = _isrc_candidates[0] - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TSRC(encoding=3, text=[final_isrc])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['ISRC'] = [final_isrc] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:ISRC'] = [MP4FreeForm(final_isrc.encode('utf-8'))] - logger.info(f"ISRC ({source}): {final_isrc}") - - # ── 6. Write copyright tag (Tidal → Qobuz fallback) ── - _copyright_candidates = [] - if tidal_copyright and _tag_enabled('tidal.tags.copyright'): - _copyright_candidates.append(('Tidal', tidal_copyright)) - if qobuz_copyright and _tag_enabled('qobuz.tags.copyright'): - _copyright_candidates.append(('Qobuz', qobuz_copyright)) - if _copyright_candidates: - source, final_copyright = _copyright_candidates[0] - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TCOP(encoding=3, text=[final_copyright])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['COPYRIGHT'] = [final_copyright] - elif isinstance(audio_file, MP4): - audio_file['cprt'] = [final_copyright] - logger.info(f"©️ Copyright ({source}): {final_copyright[:60]}") - - # ── 7. Write label/publisher tag (from Qobuz) ── - if _tag_enabled('qobuz.tags.label') and qobuz_label: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TPUB(encoding=3, text=[qobuz_label])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['LABEL'] = [qobuz_label] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:LABEL'] = [MP4FreeForm(qobuz_label.encode('utf-8'))] - logger.info(f"Label (Qobuz): {qobuz_label}") - - # ── 8. Write Last.fm and Genius URLs as custom tags ── - if _tag_enabled('lastfm.tags.url') and lastfm_url: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='LASTFM_URL', text=[lastfm_url])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['LASTFM_URL'] = [lastfm_url] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:LASTFM_URL'] = [MP4FreeForm(lastfm_url.encode('utf-8'))] - - if _tag_enabled('genius.tags.url') and genius_url: - if isinstance(audio_file.tags, ID3): - audio_file.tags.add(TXXX(encoding=3, desc='GENIUS_URL', text=[genius_url])) - elif isinstance(audio_file, (FLAC, OggVorbis)) or _is_ogg_opus(audio_file): - audio_file['GENIUS_URL'] = [genius_url] - elif isinstance(audio_file, MP4): - audio_file['----:com.apple.iTunes:GENIUS_URL'] = [MP4FreeForm(genius_url.encode('utf-8'))] - - except Exception as e: - logger.error(f"Error embedding source IDs (non-fatal): {e}") - def _download_cover_art(album_info: dict, target_dir: str, context: dict = None): - """Downloads cover.jpg into the specified directory. - Tries Cover Art Archive first (high-res) if MusicBrainz release ID is available - (set by _enhance_file_metadata during post-processing), falls back to source URL. - Accepts optional context to extract image URL from spotify_album when album_info lacks it.""" - if not config_manager.get('metadata_enhancement.cover_art_download', True): - return - try: - cover_path = os.path.join(target_dir, "cover.jpg") - release_mbid = album_info.get('musicbrainz_release_id') if album_info else None - prefer_caa = config_manager.get('metadata_enhancement.prefer_caa_art', False) - - # If cover.jpg exists but we now have a CAA MBID, check if we should upgrade - if os.path.exists(cover_path): - if release_mbid and prefer_caa: - try: - existing_size = os.path.getsize(cover_path) - # Typical Spotify/iTunes cover is ~50-150KB at 640x640 - # CAA covers are usually 300KB+ at 1200x1200 - if existing_size > 200_000: - return # Already high-res, skip - # Low-res cover exists — try to upgrade from CAA - is_upgrade = True - logger.info(f"Existing cover.jpg is {existing_size // 1024}KB — attempting CAA upgrade...") - except Exception: - return - else: - return - else: - is_upgrade = False - - image_data = None - - # Try Cover Art Archive first (often 1200x1200+, original quality) — opt-in - # The MBID is stored in album_info by _enhance_file_metadata before this is called - if release_mbid and prefer_caa: - try: - caa_url = f"https://coverartarchive.org/release/{release_mbid}/front" - req = urllib.request.Request(caa_url, headers={'Accept': 'image/*'}) - with urllib.request.urlopen(req, timeout=10) as response: - image_data = response.read() - if image_data and len(image_data) > 1000: - logger.info(f"Cover art from Cover Art Archive ({len(image_data) // 1024}KB)") - else: - image_data = None - except Exception: - image_data = None - - # If upgrading and CAA failed, keep existing cover — don't overwrite with same low-res - if is_upgrade and not image_data: - logger.error("CAA upgrade failed — keeping existing cover.jpg") - return - - # Fallback to Spotify/iTunes/Deezer URL - if not image_data: - art_url = album_info.get('album_image_url') - # If album_info lacks the URL, try the context's spotify_album - if not art_url and context: - spotify_album = context.get('spotify_album') or {} - art_url = spotify_album.get('image_url') - # Also try images array (raw Spotify API format) - if not art_url: - images = spotify_album.get('images', []) - if images and isinstance(images, list) and len(images) > 0: - art_url = images[0].get('url') if isinstance(images[0], dict) else None - if art_url: - logger.info("Using cover art URL from spotify_album context") - # Upgrade to highest available resolution before fetching - if art_url and 'i.scdn.co' in art_url: - from core.spotify_client import _upgrade_spotify_image_url - art_url = _upgrade_spotify_image_url(art_url) - elif art_url and 'mzstatic.com' in art_url: - import re as _re - art_url = _re.sub(r'\d+x\d+bb', '3000x3000bb', art_url) - if not art_url: - logger.warning("No cover art URL available for download.") - return - with urllib.request.urlopen(art_url, timeout=10) as response: - image_data = response.read() - - if not image_data: - return - - with open(cover_path, 'wb') as f: - f.write(image_data) - - logger.info(f"Cover art downloaded to: {cover_path}") - except Exception as e: - logger.error(f"Error downloading cover.jpg: {e}") - - - + return metadata_enrichment.download_cover_art( + album_info, + target_dir, + context, + ) def _get_spotify_album_tracks(spotify_album: dict) -> list: """Fetches all tracks for a given Spotify album ID.""" @@ -20568,695 +18192,29 @@ def _match_track_to_spotify_title(slsk_track_meta: dict, spotify_tracks: list) - return slsk_track_meta # Fallback to original - -# --- Post-Processing Logic --- -# ── Modular post-processing metadata lookup functions ── -# Each function receives a shared `pp` context dict and writes its results to it. -# The orchestrator in _post_process_matched_download calls them in configurable order. - -def _pp_lookup_musicbrainz(pp, _names_match): - """MusicBrainz: recording MBID, artist MBID, release MBID, ISRC, genres, release details.""" - if not config_manager.get('musicbrainz.embed_tags', True): - return - track_title = pp['track_title'] - artist_name = pp['artist_name'] - metadata = pp['metadata'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - mb_service = mb_worker.mb_service if mb_worker else None - if not mb_service: - logger.info("MusicBrainz worker not available, skipping MBID lookup") - return - result = mb_service.match_recording(track_title, artist_name) - if result and result.get('mbid'): - pp['recording_mbid'] = result['mbid'] - id_tags['MUSICBRAINZ_RECORDING_ID'] = pp['recording_mbid'] - logger.info(f"MusicBrainz recording matched: {pp['recording_mbid']}") - details = mb_service.mb_client.get_recording(pp['recording_mbid'], includes=['isrcs', 'genres']) - if details: - isrcs = details.get('isrcs', []) - if isrcs: - pp['isrc'] = isrcs[0] - pp['mb_genres'] = [g['name'] for g in sorted(details.get('genres', []), key=lambda x: x.get('count', 0), reverse=True)] - - track_artist_name = metadata.get('artist', '') or artist_name - if ', ' in track_artist_name: - track_artist_name = track_artist_name.split(', ')[0] - artist_result = mb_service.match_artist(track_artist_name) - if artist_result and artist_result.get('mbid'): - pp['artist_mbid'] = artist_result['mbid'] - id_tags['MUSICBRAINZ_ARTIST_ID'] = pp['artist_mbid'] - - album_name_for_mb = metadata.get('album', '') - if album_name_for_mb: - # Use normalized key (strips edition suffixes) so "Album (Deluxe Edition)" - # and "Album (Deluxe)" and "Album" all share the same cached MBID. - # This prevents Navidrome from splitting one album into multiple entries. - # Prefer batch-level artist name (from album download context) for the cache key - # so all tracks in the batch hit the same preflight-cached entry, even if - # per-track metadata spells the artist slightly differently. - _artist_key = (pp.get('batch_artist_name') or artist_name).lower().strip() - _rc_key_norm = (_normalize_album_cache_key(album_name_for_mb), _artist_key) - _rc_key_exact = (album_name_for_mb.lower().strip(), _artist_key) - with _mb_release_cache_lock: - # Check normalized key first (catches edition variants) - cached = _mb_release_cache.get(_rc_key_norm) - if cached is None: - cached = _mb_release_cache.get(_rc_key_exact) - if cached is not None: - pp['release_mbid'] = cached - else: - try: - _rc_result = mb_service.match_release(album_name_for_mb, artist_name) - pp['release_mbid'] = _rc_result.get('mbid', '') if _rc_result else '' - except Exception: - pp['release_mbid'] = '' - # Cache under both normalized and exact keys - _mb_release_cache[_rc_key_norm] = pp['release_mbid'] - _mb_release_cache[_rc_key_exact] = pp['release_mbid'] - if pp['release_mbid']: - id_tags['MUSICBRAINZ_RELEASE_ID'] = pp['release_mbid'] - - # Release details (group, barcode, media, etc.) - if pp['release_mbid']: - with _mb_release_detail_cache_lock: - release_detail = _mb_release_detail_cache.get(pp['release_mbid']) - if release_detail is None: - release_detail = mb_service.mb_client.get_release( - pp['release_mbid'], includes=['release-groups', 'labels', 'media', 'artist-credits', 'recordings'] - ) or {} - with _mb_release_detail_cache_lock: - _mb_release_detail_cache[pp['release_mbid']] = release_detail - if release_detail: - rg = release_detail.get('release-group', {}) - if rg.get('id'): id_tags['MUSICBRAINZ_RELEASEGROUPID'] = rg['id'] - ac = release_detail.get('artist-credit', []) - if ac and isinstance(ac[0], dict): - aa = ac[0].get('artist', {}) - if aa.get('id'): id_tags['MUSICBRAINZ_ALBUMARTISTID'] = aa['id'] - if rg.get('primary-type'): id_tags['RELEASETYPE'] = rg['primary-type'] - if rg.get('first-release-date'): - id_tags['ORIGINALDATE'] = rg['first-release-date'] - if not pp['release_year'] and len(rg['first-release-date']) >= 4: - yr = rg['first-release-date'][:4] - if yr.isdigit(): - pp['release_year'] = yr - if release_detail.get('status'): id_tags['RELEASESTATUS'] = release_detail['status'] - if release_detail.get('country'): id_tags['RELEASECOUNTRY'] = release_detail['country'] - if release_detail.get('barcode'): id_tags['BARCODE'] = release_detail['barcode'] - media_list = release_detail.get('media', []) - if media_list: - fmt = media_list[0].get('format', '') - if fmt: id_tags['MEDIA'] = fmt - id_tags['TOTALDISCS'] = str(len(media_list)) - label_info = release_detail.get('label-info', []) - if label_info and isinstance(label_info[0], dict): - cat = label_info[0].get('catalog-number', '') - if cat: id_tags['CATALOGNUMBER'] = cat - text_rep = release_detail.get('text-representation', {}) - if isinstance(text_rep, dict) and text_rep.get('script'): - id_tags['SCRIPT'] = text_rep['script'] - if release_detail.get('asin'): id_tags['ASIN'] = release_detail['asin'] - # Picard-style: pull recording MBID from the release tracklist - # instead of using the independent match_recording() result. - # This guarantees the recording ID is consistent with the release. - _trk_num = metadata.get('track_number') - _disc_num = metadata.get('disc_number') or 1 - if _trk_num and media_list: - try: - _trk_num_int, _disc_num_int = int(_trk_num), int(_disc_num) - for medium in media_list: - if medium.get('position', 1) == _disc_num_int: - for mtrack in (medium.get('tracks') or medium.get('track-list', [])): - if mtrack.get('position') == _trk_num_int: - if mtrack.get('id'): - id_tags['MUSICBRAINZ_RELEASETRACKID'] = mtrack['id'] - # Override recording MBID with the one from this release's tracklist - _release_recording = mtrack.get('recording', {}) - if _release_recording.get('id'): - pp['recording_mbid'] = _release_recording['id'] - id_tags['MUSICBRAINZ_RECORDING_ID'] = _release_recording['id'] - logger.info(f"MusicBrainz recording from release tracklist: {_release_recording['id']}") - break - break - except (ValueError, TypeError): - pass - except Exception as e: - logger.error(f"MusicBrainz lookup failed (non-fatal): {e}") - - -def _pp_lookup_deezer(pp, _names_match): - """Deezer: BPM, ISRC fallback, track/artist IDs.""" - if not config_manager.get('deezer.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - dz_client = deezer_worker.client if deezer_worker else None - if not dz_client: - logger.info("Deezer worker not available, skipping Deezer lookup") - return - dz_result = dz_client.search_track(artist_name, track_title) - if dz_result and _names_match(dz_result.get('title', ''), track_title) and \ - _names_match(dz_result.get('artist', {}).get('name', ''), artist_name): - dz_track_id = dz_result['id'] - id_tags['DEEZER_TRACK_ID'] = str(dz_track_id) - dz_artist_id = dz_result.get('artist', {}).get('id') - if dz_artist_id: - id_tags['DEEZER_ARTIST_ID'] = str(dz_artist_id) - logger.info(f"Deezer track matched: {dz_track_id}") - dz_details = dz_client.get_track_details(dz_track_id) - if dz_details: - bpm_val = dz_details.get('bpm') - if bpm_val and bpm_val > 0: - pp['deezer_bpm'] = bpm_val - dz_isrc = dz_details.get('isrc') - if dz_isrc: - pp['deezer_isrc'] = dz_isrc - # Release year from Deezer album - if not pp['release_year']: - dz_album = dz_result.get('album', {}) - dz_release = (dz_album.get('release_date', '') if isinstance(dz_album, dict) else '') or '' - if len(dz_release) >= 4 and dz_release[:4].isdigit(): - pp['release_year'] = dz_release[:4] - except Exception as e: - logger.error(f"Deezer lookup failed (non-fatal): {e}") - - -def _pp_lookup_audiodb(pp, _names_match): - """AudioDB: mood, style, genre, track ID, MusicBrainz ID fallbacks.""" - if not config_manager.get('audiodb.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - adb_client = audiodb_worker.client if audiodb_worker else None - if not adb_client: - logger.info("AudioDB worker not available, skipping AudioDB lookup") - return - adb_result = adb_client.search_track(artist_name, track_title) - if adb_result and _names_match(adb_result.get('strTrack', ''), track_title) and \ - _names_match(adb_result.get('strArtist', ''), artist_name): - adb_track_id = adb_result.get('idTrack') - if adb_track_id: - id_tags['AUDIODB_TRACK_ID'] = str(adb_track_id) - logger.info(f"AudioDB track matched: {adb_track_id}") - adb_mb_track = adb_result.get('strMusicBrainzID') - if adb_mb_track and 'MUSICBRAINZ_RECORDING_ID' not in id_tags: - id_tags['MUSICBRAINZ_RECORDING_ID'] = adb_mb_track - pp['recording_mbid'] = adb_mb_track - logger.warning(f"MusicBrainz recording ID from AudioDB fallback: {adb_mb_track}") - adb_mb_artist = adb_result.get('strMusicBrainzArtistID') - if adb_mb_artist and 'MUSICBRAINZ_ARTIST_ID' not in id_tags: - id_tags['MUSICBRAINZ_ARTIST_ID'] = adb_mb_artist - pp['artist_mbid'] = adb_mb_artist - logger.warning(f"MusicBrainz artist ID from AudioDB fallback: {adb_mb_artist}") - pp['audiodb_mood'] = adb_result.get('strMood') or None - pp['audiodb_style'] = adb_result.get('strStyle') or None - pp['audiodb_genre'] = adb_result.get('strGenre') or None - except Exception as e: - logger.error(f"AudioDB lookup failed (non-fatal): {e}") - - -def _pp_lookup_tidal(pp, _names_match): - """Tidal: ISRC fallback, copyright, track/artist IDs.""" - if not config_manager.get('tidal.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - if not (tidal_client and tidal_client.is_authenticated()): - return - td_result = tidal_client.search_track(artist_name, track_title) - if td_result and _names_match(td_result.get('title', ''), track_title): - td_track_id = td_result.get('id') - if td_track_id: - id_tags['TIDAL_TRACK_ID'] = str(td_track_id) - logger.info(f"Tidal track matched: {td_track_id}") - td_artist = td_result.get('artist', {}) - if isinstance(td_artist, dict) and td_artist.get('id'): - id_tags['TIDAL_ARTIST_ID'] = str(td_artist['id']) - if td_track_id: - td_details = tidal_client.get_track(str(td_track_id)) - if td_details: - td_isrc = td_details.get('isrc') - if td_isrc: - pp['tidal_isrc'] = td_isrc - td_copyright = td_details.get('copyright') - if isinstance(td_copyright, dict): - td_copyright = td_copyright.get('text', td_copyright.get('name', '')) - if td_copyright: - pp['tidal_copyright'] = td_copyright - # Release year from Tidal album - if not pp['release_year']: - td_album = td_result.get('album', {}) - td_release = '' - if isinstance(td_album, dict): - td_release = str(td_album.get('release_date', '') or td_album.get('releaseDate', '') or '') - if len(td_release) >= 4 and td_release[:4].isdigit(): - pp['release_year'] = td_release[:4] - except Exception as e: - logger.error(f"Tidal lookup failed (non-fatal): {e}") - - -def _pp_lookup_qobuz(pp, _names_match): - """Qobuz: ISRC fallback, copyright, label, track/artist IDs.""" - if not config_manager.get('qobuz.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - qz_client = qobuz_enrichment_worker.client if qobuz_enrichment_worker else None - if not (qz_client and qz_client.is_authenticated()): - return - qz_result = qz_client.search_track(artist_name, track_title) - if qz_result: - qz_performer = (qz_result.get('performer') or {}) - if not isinstance(qz_performer, dict): - qz_performer = {} - qz_artist_name = qz_performer.get('name', '') - if _names_match(qz_result.get('title', ''), track_title) and \ - _names_match(qz_artist_name, artist_name): - qz_track_id = qz_result.get('id') - if qz_track_id: - id_tags['QOBUZ_TRACK_ID'] = str(qz_track_id) - logger.info(f"Qobuz track matched: {qz_track_id}") - if isinstance(qz_performer, dict) and qz_performer.get('id'): - id_tags['QOBUZ_ARTIST_ID'] = str(qz_performer['id']) - qz_isrc = qz_result.get('isrc') - if isinstance(qz_isrc, dict): - qz_isrc = qz_isrc.get('value', qz_isrc.get('id', '')) - if qz_isrc: - pp['qobuz_isrc'] = qz_isrc - qz_copyright = qz_result.get('copyright') - if isinstance(qz_copyright, dict): - qz_copyright = qz_copyright.get('text', qz_copyright.get('name', '')) - if qz_copyright and isinstance(qz_copyright, str): - pp['qobuz_copyright'] = qz_copyright - qz_album = qz_result.get('album', {}) - if isinstance(qz_album, dict): - qz_label_info = qz_album.get('label', {}) - if isinstance(qz_label_info, dict) and qz_label_info.get('name'): - pp['qobuz_label'] = qz_label_info['name'] - # Release year from Qobuz album (prefer date string over Unix timestamp) - if not pp['release_year']: - qz_release = str(qz_album.get('release_date_original', '') or '') - if not qz_release: - # released_at is a Unix timestamp — convert to year - qz_ts = qz_album.get('released_at') - if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0: - import datetime as _dt - qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year) - if len(qz_release) >= 4 and qz_release[:4].isdigit(): - pp['release_year'] = qz_release[:4] - except Exception as e: - logger.error(f"Qobuz lookup failed (non-fatal): {e}") - - -def _pp_lookup_lastfm(pp, _names_match): - """Last.fm: genre tags, track URL.""" - if not config_manager.get('lastfm.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - if not track_title or not artist_name: - return - try: - lf_client = lastfm_worker.client if lastfm_worker else None - if not lf_client: - return - lf_result = lf_client.get_track_info(artist_name, track_title) - if lf_result: - lf_url = lf_result.get('url') - if lf_url: - pp['lastfm_url'] = lf_url - lf_toptags = lf_result.get('toptags', {}) - if isinstance(lf_toptags, dict): - tag_list = lf_toptags.get('tag', []) - if isinstance(tag_list, list): - pp['lastfm_tags'] = [t.get('name', '') for t in tag_list if isinstance(t, dict) and t.get('name')] - elif isinstance(tag_list, dict) and tag_list.get('name'): - pp['lastfm_tags'] = [tag_list['name']] - logger.info(f"Last.fm track info found: {len(pp['lastfm_tags'])} tags") - except Exception as e: - logger.error(f"Last.fm lookup failed (non-fatal): {e}") - - -def _pp_lookup_genius(pp, _names_match): - """Genius: track ID, URL.""" - if not config_manager.get('genius.embed_tags', True): - return - track_title, artist_name = pp['track_title'], pp['artist_name'] - id_tags = pp['id_tags'] - if not track_title or not artist_name: - return - try: - import core.genius_client as _genius_module - if time.time() < _genius_module._rate_limit_until: - logger.info("Genius rate-limited, skipping (non-blocking)") - return - g_client = genius_worker.client if genius_worker else None - if not g_client: - return - g_result = g_client.search_song(artist_name, track_title) - if g_result: - g_id = g_result.get('id') - if g_id: - id_tags['GENIUS_TRACK_ID'] = str(g_id) - logger.info(f"Genius song matched: {g_id}") - g_url = g_result.get('url') - if g_url: - pp['genius_url'] = g_url - except Exception as e: - logger.error(f"Genius lookup failed (non-fatal): {e}") - - def _post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id): """ NEW VERIFICATION WORKFLOW: Enhanced post-processing with file verification. Only sets task status to 'completed' after successful file verification and move operation. """ - _pp = pp_logger - try: - # Call the existing post-processing logic (but skip its completion callback) - # We'll handle the completion callback ourselves after verification - original_task_id = context.pop('task_id', None) # Temporarily remove to prevent double callback - original_batch_id = context.pop('batch_id', None) - _post_process_matched_download(context_key, context, file_path) - # Restore the IDs for our own callback - if original_task_id: - context['task_id'] = original_task_id - if original_batch_id: - context['batch_id'] = original_batch_id + from core.import_pipeline import post_process_matched_download_with_verification + return post_process_matched_download_with_verification( + context_key, + context, + file_path, + task_id, + batch_id, + _build_import_pipeline_runtime(), + ) - # Check if race guard detected the source file was already gone - if context.get('_race_guard_failed'): - _pp.info(f"Race guard: source file gone for task {task_id} — marking as failed") - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = 'Source file was already processed or removed by another task' - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - _on_download_completed(batch_id, task_id, success=False) - return - - # Check if AcoustID quarantined the file — no further processing needed - if context.get('_acoustid_quarantined'): - failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed') - _pp.info(f"File was quarantined by AcoustID verification (task={task_id}): {failure_msg}") - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {failure_msg}" - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - _on_download_completed(batch_id, task_id, success=False) - return - - # Check if simple download handler already completed everything - if context.get('_simple_download_completed'): - expected_final_path = context.get('_final_path') - - if expected_final_path and os.path.exists(expected_final_path): - with tasks_lock: - if task_id in download_tasks: - _mark_task_completed(task_id, context.get('track_info')) - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=True) - return - else: - _pp.info(f"FAILED simple download file not found at: {expected_final_path} (task={task_id}, context={context_key})") - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f'Downloaded file not found at expected location: {os.path.basename(expected_final_path)}' - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=False) - return - - # VERIFICATION: Use the actual path that _post_process_matched_download computed and - # moved the file to, instead of recomputing independently. Independent recomputation - # can produce path mismatches (e.g., track_number extraction/validation differences) - # causing false "file verification failed" errors on successfully processed files. - expected_final_path = context.get('_final_processed_path') - if not expected_final_path: - _pp.info(f"No _final_processed_path in context for task {task_id} — cannot verify, assuming success") - with tasks_lock: - if task_id in download_tasks: - _mark_task_completed(task_id, context.get('track_info')) - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - _on_download_completed(batch_id, task_id, success=True) - return - - # VERIFICATION: Check if file exists at the path processing actually used - if os.path.exists(expected_final_path): - # Mark task as completed only after successful verification - redownload_ctx = None - with tasks_lock: - if task_id in download_tasks: - _mark_task_completed(task_id, context.get('track_info')) - download_tasks[task_id]['metadata_enhanced'] = True - redownload_ctx = download_tasks[task_id].get('_redownload_context') - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - # Redownload hook: delete old file and update DB path (outside locks) - if redownload_ctx: - try: - old_path = redownload_ctx.get('old_file_path') - lib_track_id = redownload_ctx.get('library_track_id') - if redownload_ctx.get('delete_old_file') and old_path and os.path.exists(old_path): - if os.path.normpath(old_path) != os.path.normpath(expected_final_path): - os.remove(old_path) - logger.info(f"[Redownload] Deleted old file: {old_path}") - if lib_track_id and expected_final_path: - _rd_db = get_database() - _rd_conn = _rd_db._get_connection() - _rd_cursor = _rd_conn.cursor() - _rd_cursor.execute(""" - UPDATE tracks SET file_path = ?, updated_at = CURRENT_TIMESTAMP - WHERE id = ? - """, (expected_final_path, lib_track_id)) - _rd_conn.commit() - _rd_conn.close() - logger.info(f"[Redownload] Updated DB path for track {lib_track_id}") - except Exception as e: - logger.error(f"[Redownload] Post-processing hook error: {e}") - - _on_download_completed(batch_id, task_id, success=True) - else: - # Log failure details for diagnosis - track_name = context.get('original_search_result', {}).get('spotify_clean_title', context_key) - _pp.info(f"FAILED verification for '{track_name}' (task={task_id})") - _pp.info(f" expected_final_path: {expected_final_path}") - _pp.info(f" file_path (source): {file_path}, exists={os.path.exists(file_path)}") - _pp.info(f" is_album={context.get('is_album_download', False)}, has_clean_data={context.get('has_clean_spotify_data', False)}") - expected_dir = os.path.dirname(expected_final_path) - if os.path.exists(expected_dir): - dir_contents = os.listdir(expected_dir) - _pp.info(f" directory contains {len(dir_contents)} files: {dir_contents[:20]}") - else: - _pp.info(f" directory does not exist: {expected_dir}") - - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f'File verification failed: expected file at {os.path.basename(expected_final_path)} but it was not found after processing' - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=False) - - except Exception as e: - import traceback - _pp.info(f"EXCEPTION in post-processing for '{context_key}' (task={task_id}): {e}") - _pp.info(traceback.format_exc()) - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"Post-processing verification failed: {str(e)}" - - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - _on_download_completed(batch_id, task_id, success=False) - - -def _check_flac_bit_depth(file_path, context, context_key): - """ - Check if a FLAC file matches the user's preferred bit depth. - Returns True if the file was rejected (caller should return), False if OK. - With fallback enabled (default), accepts any FLAC bit depth rather than quarantining. - """ - if not context.get('_audio_quality', '').startswith('FLAC'): - return False - - from database.music_database import MusicDatabase - _qp_db = MusicDatabase() - _quality_profile = _qp_db.get_quality_profile() - _flac_config = _quality_profile.get('qualities', {}).get('flac', {}) - _flac_pref = _flac_config.get('bit_depth', 'any') - - if _flac_pref == 'any': - return False - - # Parse actual bit depth from quality string like "FLAC 16bit" - _actual_bits = context['_audio_quality'].replace('FLAC ', '').replace('bit', '') - if _actual_bits == _flac_pref: - return False - - # Bit depth doesn't match preference — check if fallback or downsample is enabled - _flac_fallback = _flac_config.get('bit_depth_fallback', True) - _downsample_enabled = config_manager.get('lossy_copy.downsample_hires', False) - - if _flac_fallback or _downsample_enabled: - # Accept the file — it will be downsampled or a FLAC at any bit depth is better than a failed download - track_info = context.get('track_info', {}) - track_name = track_info.get('name', os.path.basename(file_path)) - if _downsample_enabled: - logger.info(f"[FLAC Downsample] Accepted {_actual_bits}-bit FLAC (will be downsampled to {_flac_pref}-bit): {track_name}") - else: - logger.warning(f"[FLAC Fallback] Accepted {_actual_bits}-bit FLAC (preferred {_flac_pref}-bit): {track_name}") - return False - - # Strict mode — reject and quarantine - rejection_msg = f"FLAC bit depth mismatch: file is {_actual_bits}-bit, preference is {_flac_pref}-bit" - try: - quarantine_path = _move_to_quarantine(file_path, context, rejection_msg) - logger.info(f"File quarantined due to bit depth filter: {quarantine_path}") - except Exception as quarantine_error: - logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}") - try: - os.remove(file_path) - except Exception: - pass - - context['_bitdepth_rejected'] = True - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id: - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_msg}" - if task_id and batch_id: - _on_download_completed(batch_id, task_id, success=False) - return True - - -def _move_to_quarantine(file_path: str, context: dict, reason: str) -> str: - """ - Move a file to quarantine folder when AcoustID verification fails. - Creates a JSON sidecar file with metadata about why the file was quarantined. - - Args: - file_path: Original file path - context: Download context with track info - reason: Reason for quarantine - - Returns: - Path to quarantined file - """ - import json - from pathlib import Path - from datetime import datetime - - # Get quarantine directory (inside download folder — always writable, even in Docker) - download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - quarantine_dir = Path(download_dir) / "ss_quarantine" - quarantine_dir.mkdir(parents=True, exist_ok=True) - - # Create quarantine entry with timestamp - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - original_name = Path(file_path).stem - file_ext = Path(file_path).suffix - - # Build quarantine filename: TIMESTAMP_originalname.ext.quarantined - # The .quarantined extension prevents audio file searches and media servers - # from picking up known-wrong files sitting in the downloads folder. - quarantine_filename = f"{timestamp}_{original_name}{file_ext}.quarantined" - quarantine_path = quarantine_dir / quarantine_filename - - # Move file to quarantine - _safe_move_file(file_path, str(quarantine_path)) - - # Write metadata sidecar file - metadata_path = quarantine_dir / f"{timestamp}_{original_name}.json" - - # Extract track info from context - track_info = context.get('track_info', {}) - original_search = context.get('original_search_result', {}) - spotify_artist = context.get('spotify_artist', {}) - - metadata = { - 'original_filename': Path(file_path).name, - 'quarantine_reason': reason, - 'timestamp': datetime.now().isoformat(), - 'expected_track': ( - original_search.get('spotify_clean_title') or - track_info.get('name') or - original_search.get('title', 'Unknown') - ), - 'expected_artist': spotify_artist.get('name', 'Unknown'), - 'context_key': context.get('context_key', 'unknown') - } - - try: - with open(metadata_path, 'w', encoding='utf-8') as f: - json.dump(metadata, f, indent=2, ensure_ascii=False) - except Exception as e: - logger.warning(f"Failed to write quarantine metadata: {e}") - - logger.warning(f"File quarantined: {quarantine_path} - Reason: {reason}") - - try: - if automation_engine: - ti = context.get('track_info', {}) - artists = ti.get('artists', []) - artist_name = '' - if artists: - a = artists[0] - artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) - automation_engine.emit('download_quarantined', { - 'artist': artist_name, - 'title': ti.get('name', ''), - 'reason': reason or 'Unknown', - }) - except Exception: - pass - - return str(quarantine_path) +def _build_import_pipeline_runtime(): + """Collect live controller dependencies for the shared import pipeline.""" + return types.SimpleNamespace( + automation_engine=automation_engine, + on_download_completed=_on_download_completed, + web_scan_manager=web_scan_manager, + repair_worker=repair_worker, + ) def _safe_move_file(src, dst): @@ -21362,872 +18320,8 @@ def _post_process_matched_download(context_key, context, file_path): Also handles simple downloads (from search page "Download" button) which just move files to /Transfer without metadata enhancement. """ - # --- PER-FILE LOCK --- - # Acquire a per-context-key lock so only one thread processes a given file at a time. - # The Stream Processor and Verification Worker both call this function for the same file. - # Without serialization, they race to move the source file and the loser gets FileNotFoundError. - # With the lock, the second thread waits, then the existing protection checks detect - # "source gone + destination exists" and return early. - with _post_process_locks_lock: - if context_key not in _post_process_locks: - _post_process_locks[context_key] = threading.Lock() - file_lock = _post_process_locks[context_key] - - file_lock.acquire() - try: - import os - import shutil - import time - from pathlib import Path - - # --- RACE CONDITION GUARD --- - # If source file is already gone, another thread (stream processor or - # verification worker) already processed it. Check immediately after - # acquiring the lock to avoid wasting time on stability checks / acoustid. - if not os.path.exists(file_path): - existing_final = context.get('_final_processed_path') - if existing_final and os.path.exists(existing_final): - logger.info(f"[Race Guard] Source gone but destination exists — already processed by another thread: {os.path.basename(existing_final)}") - return - logger.error(f"[Race Guard] Source file gone and no known destination — marking as failed: {os.path.basename(file_path)}") - context['_race_guard_failed'] = True - return - # --- END RACE CONDITION GUARD --- - - # --- FILE STABILITY CHECK --- - # Wait for the file to stop growing before processing. slskd may still be - # flushing write buffers when it reports "Completed". We poll the file size - # a few times; once it stabilises we know the write is finished. - _basename = os.path.basename(file_path) - _prev_size = -1 - for _stability_check in range(5): - try: - _cur_size = os.path.getsize(file_path) - except OSError: - _cur_size = -1 - if _cur_size == _prev_size and _cur_size > 0: - # Size unchanged — file is stable - break - _prev_size = _cur_size - if _stability_check == 0: - logger.info(f"Waiting for file to stabilise: {_basename} ({_cur_size} bytes)") - time.sleep(1.5) - else: - logger.info(f"File may still be writing after stability checks: {_basename} ({_prev_size} bytes)") - # --- END FILE STABILITY CHECK --- - - # --- ACOUSTID VERIFICATION --- - # Optional verification that downloaded audio matches expected track. - # Only runs if enabled and configured. Fails gracefully (skips on any error). - # Runs for ALL download sources — streaming APIs can return wrong versions - # (live, remix, cover) despite downloading by track ID. - _skip_acoustid = False - - try: - from core.acoustid_verification import AcoustIDVerification, VerificationResult - - verifier = AcoustIDVerification() - available, available_reason = verifier.quick_check_available() - - if available and not _skip_acoustid: - # Extract expected track info from context - track_info = context.get('track_info', {}) - original_search = context.get('original_search_result', {}) - spotify_artist = context.get('spotify_artist', {}) - - expected_track = ( - original_search.get('spotify_clean_title') or - track_info.get('name') or - original_search.get('title', '') - ) - - # Use track-level artist for verification, NOT album artist. - # For compilations, spotify_artist is "Various Artists" which - # will never match AcoustID's actual track artist. - expected_artist = '' - track_artists = track_info.get('artists', []) - if track_artists: - first = track_artists[0] - if isinstance(first, dict): - expected_artist = first.get('name', '') - elif isinstance(first, str): - expected_artist = first - # Fallback to album artist if no track artists available - if not expected_artist: - expected_artist = spotify_artist.get('name', '') - - if expected_track and expected_artist: - logger.info(f"Running AcoustID verification for: '{expected_track}' by '{expected_artist}'") - verification_result, verification_msg = verifier.verify_audio_file( - file_path, - expected_track, - expected_artist, - context - ) - logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}") - context['_acoustid_result'] = verification_result.value - - if verification_result == VerificationResult.FAIL: - # Move to quarantine instead of Transfer - try: - quarantine_path = _move_to_quarantine(file_path, context, verification_msg) - logger.error(f"File quarantined due to verification failure: {quarantine_path}") - except Exception as quarantine_error: - # Quarantine failed — delete the known-wrong file instead - # NEVER save a file we've confirmed is wrong - logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}") - logger.error(f"Quarantine failed, deleting wrong file: {file_path}") - try: - os.remove(file_path) - except Exception as del_error: - logger.error(f"Could not delete wrong file either: {del_error}") - - # These always execute for FAIL — whether quarantine succeeded or not - context['_acoustid_quarantined'] = True - context['_acoustid_failure_msg'] = verification_msg - - # Clean up context - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - # Mark as failed in download tasks if we have task info - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id: - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['status'] = 'failed' - download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {verification_msg}" - - # Call completion callback with failure - if task_id and batch_id: - _on_download_completed(batch_id, task_id, success=False) - - return # NEVER continue processing a known-wrong file - else: - logger.warning("AcoustID verification skipped: missing track/artist info") - context['_acoustid_result'] = 'skip' - else: - logger.info(f"ℹ️ AcoustID verification not available: {available_reason}") - context['_acoustid_result'] = 'disabled' - except Exception as verify_error: - # Any verification error should NOT block the download - fail open - logger.error(f"AcoustID verification error (continuing normally): {verify_error}") - context['_acoustid_result'] = 'error' - # --- END ACOUSTID VERIFICATION --- - - # --- SIMPLE DOWNLOAD HANDLING --- - # Check if this is a simple download (search page "Download ⬇" button only) - search_result = context.get('search_result', {}) - is_simple_download = search_result.get('is_simple_download', False) - - if is_simple_download: - # Simple transfer: move to Transfer folder, no metadata enhancement - logger.info(f"Processing simple download (no metadata enhancement): {file_path}") - - transfer_path = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) - - # Check if this download has album info (from path or search result) - album_name = None - original_filename = search_result.get('filename', '') - - if '/' in original_filename or '\\' in original_filename: - # Get parent directory as album name - path_parts = original_filename.replace('\\', '/').split('/') - if len(path_parts) >= 2: - album_name = path_parts[-2] # Parent directory - - # If no album from path, check search result - if not album_name: - album_name = search_result.get('album') - - # Determine destination - filename = Path(file_path).name - - if album_name and album_name.lower() not in ['unknown', 'unknown album', '']: - # Has album info - create album folder - import re - album_name = re.sub(r'[<>:"/\\|?*]', '_', album_name).strip() - album_folder = Path(transfer_path) / album_name - album_folder.mkdir(parents=True, exist_ok=True) - destination = album_folder / filename - logger.info(f"Moving to album folder: {album_name}") - else: - # No album info - move directly to Transfer root (singles) - Path(transfer_path).mkdir(parents=True, exist_ok=True) - destination = Path(transfer_path) / filename - logger.info("Moving to Transfer root (single track)") - - _safe_move_file(file_path, destination) - logger.info(f"Moved simple download to: {destination}") - - # Clean up context - with matched_context_lock: - if context_key in matched_downloads_context: - del matched_downloads_context[context_key] - - # Trigger library scan (using correct method name) - if web_scan_manager: - threading.Thread( - target=lambda: web_scan_manager.request_scan("Simple download completed"), - daemon=True - ).start() - - add_activity_item("", "Download Complete", f"{album_name}/{filename}", "Now") - logger.info(f"Simple download post-processing complete: {album_name}/{filename}") - - # Set flag in context so verification function knows this was fully handled - context['_simple_download_completed'] = True - context['_final_path'] = str(destination) - _emit_track_downloaded(context) - _record_library_history_download(context) - _record_download_provenance(context) - return - # --- END SIMPLE DOWNLOAD HANDLING --- - - logger.info(f"Starting robust post-processing for: {context_key}") - - spotify_artist = context.get("spotify_artist") - if not spotify_artist: - logger.error("Post-processing failed: Missing spotify_artist context.") - return - - # ── UNKNOWN ARTIST GUARD ── - # If artist name is junk, attempt to resolve from track metadata before proceeding. - # This prevents files from landing in "Unknown Artist/" folders. - _junk_artist_names = {'', 'unknown', 'unknown artist', 'various artists', 'none', 'null'} - _artist_name = (spotify_artist.get('name', '') if isinstance(spotify_artist, dict) else '').strip() - if _artist_name.lower() in _junk_artist_names: - logger.info(f"[Unknown Artist Guard] Artist name is '{_artist_name}' — attempting to resolve") - _resolved = False - track_info_guard = context.get("track_info", {}) or {} - original_search_guard = context.get("original_search_result", {}) or {} - - # Try 1: Pull artist from track_info.artists - _ti_artists = track_info_guard.get('artists', []) - if isinstance(_ti_artists, list) and _ti_artists: - _first = _ti_artists[0] - _name = _first.get('name', '') if isinstance(_first, dict) else str(_first) - if _name and _name.strip().lower() not in _junk_artist_names: - spotify_artist['name'] = _name.strip() - logger.info(f"[Unknown Artist Guard] Resolved from track_info.artists: '{_name}'") - _resolved = True - - # Try 2: Pull from original_search_result - if not _resolved: - _os_artist = original_search_guard.get('artist') or original_search_guard.get('artist_name') or '' - if isinstance(_os_artist, str) and _os_artist.strip().lower() not in _junk_artist_names: - spotify_artist['name'] = _os_artist.strip() - logger.info(f"[Unknown Artist Guard] Resolved from original_search_result: '{_os_artist}'") - _resolved = True - - # Try 3: Re-fetch from metadata source using track ID - if not _resolved: - _track_id = track_info_guard.get('id') or track_info_guard.get('track_id') or '' - if _track_id: - try: - _fb_client = _get_metadata_fallback_client() - if hasattr(_fb_client, 'get_track_details'): - _details = _fb_client.get_track_details(str(_track_id)) - if _details and isinstance(_details, dict): - _d_artists = _details.get('artists', []) - if isinstance(_d_artists, list) and _d_artists: - _d_first = _d_artists[0] - _d_name = _d_first.get('name', '') if isinstance(_d_first, dict) else str(_d_first) - if _d_name and _d_name.strip().lower() not in _junk_artist_names: - spotify_artist['name'] = _d_name.strip() - logger.info(f"[Unknown Artist Guard] Resolved from metadata API: '{_d_name}'") - _resolved = True - except Exception as _guard_err: - logger.error(f"[Unknown Artist Guard] Metadata re-fetch failed: {_guard_err}") - - if not _resolved: - logger.error(f"[Unknown Artist Guard] Could not resolve artist — proceeding with '{_artist_name}'") - - context['spotify_artist'] = spotify_artist - # ── END UNKNOWN ARTIST GUARD ── - - # Check if playlist folder mode is enabled (sync page playlists only) - track_info = context.get("track_info", {}) - playlist_folder_mode = track_info.get("_playlist_folder_mode", False) - - logger.debug(f"[Debug] Post-processing - track_info type: {type(track_info)}, is None: {track_info is None}, is empty: {not track_info}") - logger.debug(f"[Debug] Post-processing - playlist_folder_mode: {playlist_folder_mode}") - if track_info: - logger.debug(f"[Debug] Post-processing - track_info keys: {list(track_info.keys())}") - - if playlist_folder_mode: - # Use shared path builder for playlist mode - playlist_name = track_info.get("_playlist_name", "Unknown Playlist") - logger.info(f"[Playlist Folder Mode] Organizing in playlist folder: {playlist_name}") - - file_ext = os.path.splitext(file_path)[1] - - # Build final path FIRST so we can check for already-processed files - final_path, _ = _build_final_path_for_track(context, spotify_artist, None, file_ext) - logger.info(f"Playlist mode final path: '{final_path}'") - - # RACE CONDITION GUARD: If source file is gone but destination exists, - # another thread (stream processor or verification worker) already moved it. - # Return early to avoid deleting the successfully processed file. - if not os.path.exists(file_path): - if os.path.exists(final_path): - logger.info(f"[Playlist Folder Mode] Source gone but destination exists — already processed by another thread: {os.path.basename(final_path)}") - context['_final_processed_path'] = final_path - return - else: - pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: Source file not found and destination does not exist: {file_path}") - raise FileNotFoundError(f"Source file not found and destination does not exist: {file_path}") - - context['_audio_quality'] = _get_audio_quality_string(file_path) - if context['_audio_quality']: - logger.info(f"Audio quality detected: {context['_audio_quality']}") - - # FLAC bit depth filter - if _check_flac_bit_depth(file_path, context, context_key): - return - - # Enhance metadata before moving - try: - logger.warning(f"[Metadata Input] Playlist mode - artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})") - _enhance_file_metadata(file_path, context, spotify_artist, None) - except Exception as meta_err: - import traceback - pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") - _wipe_source_tags(file_path) - - # Move file to playlist folder - logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") - _safe_move_file(file_path, final_path) - - # Store final path for verification wrapper (before conversions may override) - context['_final_processed_path'] = final_path - - # ReplayGain analysis — write track-level gain tags if enabled - if config_manager.get('post_processing.replaygain_enabled', False): - try: - from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF - if _rg_ffmpeg_ok(): - lufs, peak_dbfs = _rg_analyze(final_path) - gain_db = _RG_REF - lufs - _rg_write(final_path, gain_db, peak_dbfs) - pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB — {os.path.basename(final_path)}") - except Exception as rg_err: - pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") - - # Downsample hi-res FLAC to CD quality if enabled (must run before lossy copy) - downsampled_path = _downsample_hires_flac(final_path, context) - if downsampled_path: - final_path = downsampled_path - context['_final_processed_path'] = final_path - - # Lossy copy: create MP3 version if enabled - blasphemy_path = _create_lossy_copy(final_path) - if blasphemy_path: - context['_final_processed_path'] = blasphemy_path - - # Clean up empty directories in downloads folder - downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - _cleanup_empty_directories(downloads_path, file_path) - - logger.info(f"[Playlist Folder Mode] Post-processing complete: {final_path}") - - # WISHLIST REMOVAL: Check if this track should be removed from wishlist - try: - _check_and_remove_from_wishlist(context) - except Exception as wishlist_error: - logger.error(f"[Playlist Folder] Error checking wishlist removal: {wishlist_error}") - - _emit_track_downloaded(context) - _record_library_history_download(context) - _record_download_provenance(context) - - # Mark as stream processed so the verification worker doesn't search - # for the file by its original Soulseek name (which no longer exists after rename) - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id and batch_id: - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['stream_processed'] = True - download_tasks[task_id]['status'] = 'completed' - logger.info(f"[Playlist Folder Mode] Marked task {task_id} as completed") - _on_download_completed(batch_id, task_id, success=True) - - return # Skip normal album/artist folder structure processing - - is_album_download = context.get("is_album_download", False) - has_clean_spotify_data = context.get("has_clean_spotify_data", False) - - if is_album_download and has_clean_spotify_data: - # Build album_info directly from clean Spotify metadata (GUI PARITY) - logger.info("Album context with clean Spotify data found - using direct album info") - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album", {}) - - # Use clean Spotify metadata (matches GUI's SpotifyBasedSearchResult approach) - clean_track_name = original_search.get('spotify_clean_title', 'Unknown Track') - clean_album_name = original_search.get('spotify_clean_album', 'Unknown Album') - - logger.debug( - "Path 1 - Clean Spotify data path: keys=%s has_track_number=%s track_number=%s", - list(original_search.keys()), - 'track_number' in original_search, - original_search.get('track_number', 'NOT_FOUND'), - ) - - album_info = { - 'is_album': True, - 'album_name': clean_album_name, # Use clean Spotify album name - 'track_number': original_search.get('track_number', 1), - 'disc_number': original_search.get('disc_number', 1), - 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album.get('image_url'), - 'confidence': 1.0, # High confidence since we have clean Spotify data - 'source': 'clean_spotify_metadata' - } - - logger.info(f"Using clean Spotify album: '{clean_album_name}' for track: '{clean_track_name}'") - elif is_album_download: - # CRITICAL FIX: Album context without clean Spotify data - still force album treatment - logger.warning("Album context found but no clean Spotify data - using enhanced fallback") - original_search = context.get("original_search_result", {}) - spotify_album = context.get("spotify_album", {}) - clean_track_name = original_search.get('spotify_clean_title') or original_search.get('title', 'Unknown Track') - - logger.debug( - "Path 2 - Enhanced fallback album context path: keys=%s has_track_number=%s track_number=%s spotify_album=%s", - list(original_search.keys()), - 'track_number' in original_search, - original_search.get('track_number', 'NOT_FOUND'), - spotify_album.get('name', 'NOT_FOUND'), - ) - - # ENHANCEMENT: Use spotify_clean_album if available for consistency - album_name = (original_search.get('spotify_clean_album') or - spotify_album.get('name') or - 'Unknown Album') - - album_info = { - 'is_album': True, # FORCE TRUE - user explicitly selected album for download - 'album_name': album_name, - 'track_number': original_search.get('track_number', 1), - 'disc_number': original_search.get('disc_number', 1), - 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album.get('image_url'), - 'confidence': 0.9, # Higher confidence - user explicitly chose album - 'source': 'enhanced_fallback_album_context' - } - logger.info(f"[FORCED ALBUM] Using album: '{album_name}' for track: '{clean_track_name}'") - else: - # For singles, we still need to detect if they belong to an album. - logger.info("Single track download - attempting album detection") - album_info = _detect_album_info_web(context, spotify_artist) - - # --- Album grouping resolution --- - # Only run smart grouping for singles/auto-detected albums. - # Explicit album downloads already have the correct Spotify album name — - # re-grouping would mangle names like "(Reworked and Remastered)" into "(Deluxe Edition)". - if album_info and album_info['is_album'] and not is_album_download: - logger.info( - "SMART ALBUM GROUPING for track=%r original_album=%r", - album_info.get('clean_track_name', 'Unknown'), - album_info.get('album_name', 'None'), - ) - - # Get original album name from context if available - original_album = None - if context.get("original_search_result", {}).get("album"): - original_album = context["original_search_result"]["album"] - - # Use the GUI's smart album grouping algorithm - consistent_album_name = _resolve_album_group(spotify_artist, album_info, original_album) - album_info['album_name'] = consistent_album_name - - logger.info("Album grouping complete: final_album=%r", consistent_album_name) - elif album_info and album_info['is_album'] and is_album_download: - logger.info( - "EXPLICIT ALBUM DOWNLOAD - preserving Spotify album name=%r; skipping smart grouping", - album_info.get('album_name', 'None'), - ) - - # 1. Get transfer path (directory creation handled by _build_final_path_for_track) - file_ext = os.path.splitext(file_path)[1] - context['_audio_quality'] = _get_audio_quality_string(file_path) - if context['_audio_quality']: - logger.info(f"Audio quality detected: {context['_audio_quality']}") - - # FLAC bit depth filter - if _check_flac_bit_depth(file_path, context, context_key): - return - - # 2. Build the final path using GUI-style track naming with multiple fallback sources - if album_info and album_info['is_album']: - album_name_sanitized = _sanitize_filename(album_info['album_name']) - - # --- GUI PARITY: Use multiple sources for clean track name --- - original_search = context.get("original_search_result", {}) - clean_track_name = album_info['clean_track_name'] - - # Priority 1: Spotify clean title from context - if original_search.get('spotify_clean_title'): - clean_track_name = original_search['spotify_clean_title'] - logger.info(f"Using Spotify clean title: '{clean_track_name}'") - # Priority 2: Album info clean name - elif album_info.get('clean_track_name'): - clean_track_name = album_info['clean_track_name'] - logger.info(f"Using album info clean name: '{clean_track_name}'") - # Priority 3: Original title as fallback - else: - clean_track_name = original_search.get('title', 'Unknown Track') - logger.warning(f"Using original title as fallback: '{clean_track_name}'") - - final_track_name_sanitized = _sanitize_filename(clean_track_name) - track_number = album_info['track_number'] - - logger.debug( - "Final track_number processing: source=%s album_info_track_number=%s track_number=%s", - album_info.get('source', 'unknown'), - album_info.get('track_number', 'NOT_FOUND'), - track_number, - ) - - # Fix: Handle None track_number - if track_number is None: - track_number = _extract_track_number_from_filename(file_path) - logger.info( - "Track number was None; extracted from filename=%r -> %s", - os.path.basename(file_path), - track_number, - ) - - # Ensure track_number is valid - if not isinstance(track_number, int) or track_number < 1: - logger.error(f"Invalid track number ({track_number}), defaulting to 1") - track_number = 1 - - logger.debug(f"FINAL track_number used for filename: {track_number}") - - # CRITICAL FIX: Update album_info with corrected track_number for metadata enhancement - album_info['track_number'] = track_number - album_info['clean_track_name'] = clean_track_name # Ensure clean name is in album_info - logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata") - - # Use shared path builder for album mode - final_path, _ = _build_final_path_for_track(context, spotify_artist, album_info, file_ext) - logger.info(f"Album path: '{final_path}'") - else: - # Single track structure: Transfer/ARTIST/ARTIST - SINGLE/SINGLE.ext - # --- GUI PARITY: Use multiple sources for clean track name --- - original_search = context.get("original_search_result", {}) - clean_track_name = album_info['clean_track_name'] - - # Priority 1: Spotify clean title from context - if original_search.get('spotify_clean_title'): - clean_track_name = original_search['spotify_clean_title'] - logger.info(f"Using Spotify clean title: '{clean_track_name}'") - # Priority 2: Album info clean name - elif album_info and album_info.get('clean_track_name'): - clean_track_name = album_info['clean_track_name'] - logger.info(f"Using album info clean name: '{clean_track_name}'") - # Priority 3: Original title as fallback - else: - clean_track_name = original_search.get('title', 'Unknown Track') - logger.warning(f"Using original title as fallback: '{clean_track_name}'") - - # Ensure clean name is in album_info for path builder - if album_info: - album_info['clean_track_name'] = clean_track_name - - # Use shared path builder for single mode - final_path, _ = _build_final_path_for_track(context, spotify_artist, album_info, file_ext) - logger.info(f"Single path: '{final_path}'") - - # Store the actual computed path so verification uses this exact path - # instead of recomputing independently (which can produce mismatches) - context['_final_processed_path'] = final_path - - # 3. Enhance metadata, move file, download art, and cleanup - try: - logger.warning(f"[Metadata Input] artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})") - if album_info: - logger.warning(f"[Metadata Input] album: '{album_info.get('album_name', 'MISSING')}', track#: {album_info.get('track_number', 'MISSING')}, disc#: {album_info.get('disc_number', 'MISSING')}, source: {album_info.get('source', 'unknown')}") - else: - logger.info("[Metadata Input] album_info: None (single track)") - _enhance_file_metadata(file_path, context, spotify_artist, album_info) - except Exception as meta_err: - import traceback - pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}") - # Wipe source tags even though enhancement failed — prevents Soulseek - # uploader's MusicBrainz IDs from causing album splits in Navidrome - _wipe_source_tags(file_path) - - # Detect enhance mode from track context - _enhance_source_info = context.get('track_info', {}).get('source_info') or {} - if isinstance(_enhance_source_info, str): - try: - _enhance_source_info = json.loads(_enhance_source_info) - except (json.JSONDecodeError, TypeError): - _enhance_source_info = {} - is_enhance_download = _enhance_source_info.get('enhance', False) - - logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'") - if os.path.exists(final_path): - # PROTECTION: If destination already exists, check before overwriting - # If the source file is gone, another thread already handled this - don't delete the destination - if not os.path.exists(file_path): - logger.info(f"[Protection] Destination exists and source already gone - file already transferred: {os.path.basename(final_path)}") - return - try: - from mutagen import File as MutagenFile - existing_file = MutagenFile(final_path) - has_metadata = existing_file is not None and len(existing_file.tags or {}) > 2 # More than basic tags - - if has_metadata and not is_enhance_download: - # Check if incoming file is higher quality and setting is enabled - _replace_lower = config_manager.get('import.replace_lower_quality', False) - if _replace_lower: - _existing_tier = _get_quality_tier_from_extension(final_path) - _incoming_tier = _get_quality_tier_from_extension(file_path) - if _incoming_tier[1] < _existing_tier[1]: - # Incoming is higher quality (lower tier number) — replace - logger.info(f"[Quality Replace] Replacing {_existing_tier[0]} with {_incoming_tier[0]}: {os.path.basename(final_path)}") - try: - os.remove(final_path) - except Exception as e: - logger.error(f"[Quality Replace] Could not remove existing file: {e}") - else: - logger.info(f"[Protection] Existing file is same or better quality ({_existing_tier[0]} vs {_incoming_tier[0]}) - skipping: {os.path.basename(final_path)}") - try: - os.remove(file_path) - except FileNotFoundError: - pass - except Exception as e: - logger.error(f"[Protection] Error removing redundant file: {e}") - return - else: - logger.info(f"[Protection] Existing file already has metadata enhancement - skipping overwrite: {os.path.basename(final_path)}") - logger.info(f"[Protection] Removing redundant download file: {os.path.basename(file_path)}") - try: - os.remove(file_path) - except FileNotFoundError: - logger.error(f"[Protection] Could not remove redundant file (already gone): {file_path}") - except Exception as e: - logger.error(f"[Protection] Error removing redundant file: {e}") - return # Don't overwrite the good file - elif is_enhance_download: - # ENHANCE BYPASS: Allow overwrite — backup original, then remove to allow move - logger.info(f"[Enhance] Quality enhance mode — replacing existing file: {os.path.basename(final_path)}") - try: - os.remove(final_path) - except Exception as e: - logger.error(f"[Enhance] Could not remove existing file for replacement: {e}") - else: - logger.info(f"[Protection] Existing file lacks metadata - safe to overwrite: {os.path.basename(final_path)}") - try: - os.remove(final_path) - except FileNotFoundError: - pass # It was just there, but now gone? - except Exception as check_error: - logger.error(f"[Protection] Error checking existing file metadata, proceeding with overwrite: {check_error}") - try: - if os.path.exists(final_path): - os.remove(final_path) - except Exception as e: - logger.error(f"[Protection] Failed to remove existing file for overwrite: {e}") - - # --- PRE-MOVE SOURCE CHECK --- - # Right before moving, verify the source file still exists. - # Another thread (Stream Processor or Verification Worker) may have - # already moved this file during the sleep + metadata enhancement window. - if not os.path.exists(file_path): - if os.path.exists(final_path): - logger.info(f"[Pre-Move] Source already gone and destination exists - another thread completed transfer: {os.path.basename(final_path)}") - # Still do cover art + lyrics since the other thread might not have finished those - _download_cover_art(album_info, os.path.dirname(final_path), context) - _generate_lrc_file(final_path, context, spotify_artist, album_info) - return - else: - # Source gone, exact destination not found — check if stream processor already - # moved it with a quality tag (e.g. "track [FLAC 24bit].flac" vs "track.flac") - expected_dir = os.path.dirname(final_path) - expected_stem = os.path.splitext(os.path.basename(final_path))[0] - expected_ext = os.path.splitext(final_path)[1] - found_variant = None - # Also check for lossy output if Blasphemy Mode may have deleted the .flac - check_exts = {expected_ext} - if expected_ext == '.flac' and config_manager.get('lossy_copy.enabled', False) and config_manager.get('lossy_copy.delete_original', False): - _lossy_ext_map = {'mp3': '.mp3', 'opus': '.opus', 'aac': '.m4a'} - _lossy_codec = config_manager.get('lossy_copy.codec', 'mp3') - check_exts.add(_lossy_ext_map.get(_lossy_codec, '.mp3')) - if os.path.exists(expected_dir): - for f in os.listdir(expected_dir): - # Match files that start with the expected stem and have a matching extension - # This catches "01 - track [FLAC 24bit].flac" when expecting "01 - track.flac" - # and "01 - track [MP3-320].mp3" when Blasphemy Mode deleted the FLAC - f_ext = os.path.splitext(f)[1].lower() - if f_ext in check_exts and os.path.splitext(f)[0].startswith(expected_stem): - found_variant = os.path.join(expected_dir, f) - break - if found_variant: - logger.debug(f"[Pre-Move] Source gone but found variant in destination (stream processor handled it): {os.path.basename(found_variant)}") - context['_final_processed_path'] = found_variant - _download_cover_art(album_info, expected_dir, context) - _generate_lrc_file(found_variant, context, spotify_artist, album_info) - return - else: - logger.warning(f"[Pre-Move] Source file gone and no matching file in destination: {os.path.basename(file_path)}") - raise FileNotFoundError(f"Source file vanished before move and destination does not exist: {file_path}") - - _safe_move_file(file_path, final_path) - - # ENHANCE CLEANUP: If format changed (e.g. MP3→FLAC), remove old-format file - if is_enhance_download and _enhance_source_info.get('original_file_path'): - original_enhance_path = _enhance_source_info['original_file_path'] - if os.path.normpath(original_enhance_path) != os.path.normpath(final_path) and os.path.exists(original_enhance_path): - try: - os.remove(original_enhance_path) - old_fmt = os.path.splitext(original_enhance_path)[1] - new_fmt = os.path.splitext(final_path)[1] - logger.info(f"[Enhance] Upgraded {old_fmt} → {new_fmt}: {os.path.basename(final_path)}") - except Exception as e: - logger.error(f"[Enhance] Could not remove old-format file: {e}") - elif is_enhance_download: - old_fmt = _enhance_source_info.get('original_format', 'unknown') - new_fmt = os.path.splitext(final_path)[1] - logger.info(f"[Enhance] Replaced in-place ({old_fmt} → {new_fmt}): {os.path.basename(final_path)}") - - _download_cover_art(album_info, os.path.dirname(final_path), context) - - # 4. Generate LRC lyrics file at final location (elegant addition) - _generate_lrc_file(final_path, context, spotify_artist, album_info) - - # 5. ReplayGain analysis — write track-level gain tags if enabled - if config_manager.get('post_processing.replaygain_enabled', False): - try: - from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF - if _rg_ffmpeg_ok(): - lufs, peak_dbfs = _rg_analyze(final_path) - gain_db = _RG_REF - lufs - _rg_write(final_path, gain_db, peak_dbfs) - pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB, peak {peak_dbfs:.2f} dBFS — {os.path.basename(final_path)}") - except Exception as rg_err: - pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}") - - # Downsample hi-res FLAC to CD quality if enabled (must run before lossy copy) - downsampled_path = _downsample_hires_flac(final_path, context) - if downsampled_path: - final_path = downsampled_path - context['_final_processed_path'] = final_path - - # Lossy copy: create MP3 version if enabled - blasphemy_path = _create_lossy_copy(final_path) - if blasphemy_path: - context['_final_processed_path'] = blasphemy_path - - downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) - _cleanup_empty_directories(downloads_path, file_path) - - logger.info(f"Post-processing complete for: {context.get('_final_processed_path', final_path)}") - - _emit_track_downloaded(context) - _record_library_history_download(context) - _record_download_provenance(context) - _record_soulsync_library_entry(context, spotify_artist, album_info) - - # RETAG DATA CAPTURE: Record completed album/single downloads for retag tool - try: - if not playlist_folder_mode: - completed_path = context.get('_final_processed_path', final_path) - _record_retag_download(context, spotify_artist, album_info, completed_path) - except Exception as retag_err: - logger.error(f"[Post-Process] Retag data capture failed (non-fatal): {retag_err}") - - # REPAIR: Register album folder for repair scanning when batch completes - try: - completed_path = context.get('_final_processed_path', final_path) - batch_id_for_repair = context.get('batch_id') - if completed_path and batch_id_for_repair and repair_worker: - album_folder = os.path.dirname(str(completed_path)) - if album_folder: - repair_worker.register_folder(batch_id_for_repair, album_folder) - except Exception as repair_err: - logger.error(f"[Post-Process] Repair folder registration failed: {repair_err}") - - # ALBUM CONSISTENCY: Register completed file for post-batch MB tag reconciliation - try: - completed_path = context.get('_final_processed_path', final_path) - batch_id_for_consistency = context.get('batch_id') - if completed_path and batch_id_for_consistency and album_info and album_info.get('is_album'): - _original_search = context.get('original_search_result', {}) - _file_info = { - 'path': str(completed_path), - 'track_number': album_info.get('track_number', 1), - 'disc_number': album_info.get('disc_number', 1), - 'title': _original_search.get('spotify_clean_title', '') or album_info.get('clean_track_name', ''), - } - with tasks_lock: - if batch_id_for_consistency in download_batches: - download_batches[batch_id_for_consistency].setdefault('_consistency_files', []).append(_file_info) - except Exception as cons_err: - logger.error(f"[Post-Process] Album consistency registration failed: {cons_err}") - - # WISHLIST REMOVAL: Check if this track should be removed from wishlist after successful download - try: - _check_and_remove_from_wishlist(context) - except Exception as wishlist_error: - logger.error(f"[Post-Process] Error checking wishlist removal: {wishlist_error}") - - # Call completion callback for missing downloads tasks to start next batch - task_id = context.get('task_id') - batch_id = context.get('batch_id') - if task_id and batch_id: - logger.info(f"[Post-Process] Calling completion callback for task {task_id} in batch {batch_id}") - - # Mark task as stream processed and set terminal status so - # _validate_worker_counts won't count this task as active - # (prevents active_count inflation race). - # NOTE: Only set status here — don't call _mark_task_completed() because - # _run_post_processing_worker will call it later with the session counter - # increment. Calling it here too would double-count the download. - with tasks_lock: - if task_id in download_tasks: - download_tasks[task_id]['stream_processed'] = True - download_tasks[task_id]['status'] = 'completed' - logger.info(f"[Post-Process] Marked task {task_id} as completed") - - _on_download_completed(batch_id, task_id, success=True) - - except Exception as e: - import traceback - pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: {e}") - pp_logger.info(traceback.format_exc()) - logger.error(f"\nCRITICAL ERROR in post-processing for {context_key}: {e}") - traceback.print_exc() - - # Only retry if the source file still exists - otherwise retrying is pointless - # and creates an infinite loop of failures - import os as _os - source_exists = _os.path.exists(file_path) if file_path else False - if source_exists: - # Remove from processed set so it can be retried - if context_key in _processed_download_ids: - _processed_download_ids.remove(context_key) - logger.warning(f"Removed {context_key} from processed set - will retry on next check") - - # Re-add to matched context for retry - with matched_context_lock: - if context_key not in matched_downloads_context: - matched_downloads_context[context_key] = context - logger.warning(f"Re-added {context_key} to context for retry") - else: - logger.warning(f"Source file gone, not retrying: {context_key}") - finally: - file_lock.release() - # Clean up the lock entry to prevent unbounded memory growth - with _post_process_locks_lock: - _post_process_locks.pop(context_key, None) + from core.import_pipeline import post_process_matched_download + return post_process_matched_download(context_key, context, file_path, _build_import_pipeline_runtime()) # Keep track of processed downloads to avoid re-processing _processed_download_ids = set() @@ -22236,12 +18330,6 @@ _processed_download_ids = set() # so we only log the warning once per key instead of spamming every poll cycle _stale_transfer_keys = set() -# Per-context-key locks to prevent two threads from processing the same file simultaneously. -# Without this, the Stream Processor and Verification Worker race to move the same source file, -# and the loser gets a FileNotFoundError because the winner already moved it. -_post_process_locks = {} # {context_key: threading.Lock()} -_post_process_locks_lock = threading.Lock() # protects the dict itself - # --- File Discovery Retry System --- # Prevents race condition where slskd reports completion before file is written to disk # Tracks retry attempts per download to give files time to finish writing @@ -22249,83 +18337,6 @@ _download_retry_attempts = {} # {context_key: {'count': N, 'first_attempt': tim _download_retry_max = 10 # Max retries before giving up (10 seconds with 1s poll interval) _download_retry_lock = threading.Lock() -def _record_retag_download(context, spotify_artist, album_info, final_path): - """Record a completed download in the retag tables for later re-tagging.""" - from database.music_database import get_database - db = get_database() - - # Extract artist name - if isinstance(spotify_artist, dict): - artist_name = spotify_artist.get('name', 'Unknown Artist') - else: - artist_name = getattr(spotify_artist, 'name', 'Unknown Artist') - - spotify_album = context.get('spotify_album', {}) - original_search = context.get('original_search_result', {}) - track_info = context.get('track_info', {}) - - is_album = album_info and album_info.get('is_album', False) - group_type = 'album' if is_album else 'single' - album_name = album_info.get('album_name', '') if album_info else ( - original_search.get('spotify_clean_title', 'Unknown')) - - # Determine album IDs (Spotify vs iTunes) — skip beatport_ prefixed IDs - spotify_album_id = None - itunes_album_id = None - if spotify_album: - album_id_raw = str(spotify_album.get('id', '')) - if album_id_raw and album_id_raw.isdigit(): - itunes_album_id = album_id_raw - elif album_id_raw and not album_id_raw.startswith('beatport_'): - spotify_album_id = album_id_raw - - image_url = album_info.get('album_image_url') if album_info else None - total_tracks = spotify_album.get('total_tracks', 1) if spotify_album else 1 - release_date = spotify_album.get('release_date', '') if spotify_album else '' - - # Find or create group (avoid duplicating for multi-track albums) - group_id = db.find_retag_group(artist_name, album_name) - if group_id is None: - group_id = db.add_retag_group( - group_type=group_type, artist_name=artist_name, album_name=album_name, - image_url=image_url, spotify_album_id=spotify_album_id, - itunes_album_id=itunes_album_id, total_tracks=total_tracks, - release_date=release_date - ) - if group_id is None: - return - - # Track details - track_number = album_info.get('track_number', 1) if album_info else 1 - disc_number = original_search.get('disc_number') or ( - album_info.get('disc_number', 1) if album_info else 1) - title = original_search.get('spotify_clean_title') or ( - album_info.get('clean_track_name', 'Unknown Track') if album_info else 'Unknown Track') - file_format = os.path.splitext(str(final_path))[1].lstrip('.').lower() - - # Track IDs (Spotify vs iTunes) — skip beatport_ prefixed IDs - spotify_track_id = None - itunes_track_id = None - if track_info and track_info.get('id'): - tid = str(track_info['id']) - if tid.isdigit(): - itunes_track_id = tid - elif not tid.startswith('beatport_'): - spotify_track_id = tid - - # Avoid duplicate track entries - if not db.retag_track_exists(group_id, str(final_path)): - db.add_retag_track( - group_id=group_id, track_number=track_number, disc_number=disc_number, - title=title, file_path=str(final_path), file_format=file_format, - spotify_track_id=spotify_track_id, itunes_track_id=itunes_track_id - ) - logger.info(f"[Retag] Recorded track for retag: '{title}' in '{album_name}'") - - # Cap retag groups at 100, remove oldest - db.trim_retag_groups(100) - - def _execute_retag(group_id, album_id): """Execute a retag operation: re-tag files in a group with metadata from a new album match.""" global retag_state @@ -27449,7 +23460,7 @@ def _on_download_completed(batch_id, task_id, success=True): artist_name=_cons_artist_name, mb_service=_cons_mb_svc, total_discs=_cons_album.get('total_discs', 1), - file_lock_fn=_get_file_lock, + file_lock_fn=get_file_lock, ) if _cons_result.get('success'): logger.info(f"[Album Consistency] {_cons_result['tags_written']}/{_cons_result['total_files']} files " @@ -27533,7 +23544,7 @@ def _run_full_missing_tracks_process(batch_id, playlist_id, tracks_json): if release and release.get('id'): release_mbid = release['id'] _artist_key = artist_name_pf.lower().strip() - _rc_key_norm = (_normalize_album_cache_key(album_name_pf), _artist_key) + _rc_key_norm = (normalize_album_cache_key(album_name_pf), _artist_key) _rc_key_exact = (album_name_pf.lower().strip(), _artist_key) with _mb_release_cache_lock: _mb_release_cache[_rc_key_norm] = release_mbid @@ -28123,19 +24134,19 @@ def _run_post_processing_worker(task_id, batch_id): original_search = context.get("original_search_result", {}) logger.info(f"[Post-Processing] original_search keys: {list(original_search.keys())}") - spotify_clean_title = original_search.get('spotify_clean_title') + clean_title = get_import_clean_title(context, default=original_search.get('title', '')) track_number = original_search.get('track_number') - logger.info(f"[Post-Processing] spotify_clean_title: '{spotify_clean_title}', track_number: {track_number}") + logger.info(f"[Post-Processing] clean_title: '{clean_title}', track_number: {track_number}") - if spotify_clean_title and track_number: + if clean_title and track_number: # Generate expected final filename that stream processor would create # Pattern: f"{track_number:02d} - {clean_title}.flac" - sanitized_title = spotify_clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') + sanitized_title = clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') expected_final_filename = f"{track_number:02d} - {sanitized_title}.flac" logger.info(f"[Post-Processing] Generated expected final filename: {expected_final_filename}") else: - logger.warning(f"[Post-Processing] Missing required data - spotify_clean_title: {bool(spotify_clean_title)}, track_number: {bool(track_number)}") + logger.warning(f"[Post-Processing] Missing required data - clean_title: {bool(clean_title)}, track_number: {bool(track_number)}") except Exception as e: logger.error(f"[Post-Processing] Error generating expected filename: {e}") import traceback @@ -28160,19 +24171,19 @@ def _run_post_processing_worker(task_id, batch_id): original_search = context.get("original_search_result", {}) logger.info(f"[Post-Processing] fuzzy context original_search keys: {list(original_search.keys())}") - spotify_clean_title = original_search.get('spotify_clean_title') + clean_title = get_import_clean_title(context, default=original_search.get('title', '')) track_number = original_search.get('track_number') - logger.info(f"[Post-Processing] fuzzy context spotify_clean_title: '{spotify_clean_title}', track_number: {track_number}") + logger.info(f"[Post-Processing] fuzzy context clean_title: '{clean_title}', track_number: {track_number}") - if spotify_clean_title and track_number: + if clean_title and track_number: # Generate expected final filename that stream processor would create # Pattern: f"{track_number:02d} - {clean_title}.flac" - sanitized_title = spotify_clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') + sanitized_title = clean_title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('*', '_').replace('?', '_').replace('"', '_').replace('<', '_').replace('>', '_').replace('|', '_') expected_final_filename = f"{track_number:02d} - {sanitized_title}.flac" logger.info(f"[Post-Processing] Generated expected final filename from fuzzy match: {expected_final_filename}") else: - logger.warning(f"[Post-Processing] Missing required data from fuzzy match - spotify_clean_title: {bool(spotify_clean_title)}, track_number: {bool(track_number)}") + logger.warning(f"[Post-Processing] Missing required data from fuzzy match - clean_title: {bool(clean_title)}, track_number: {bool(track_number)}") except Exception as e: logger.error(f"[Post-Processing] Error generating expected filename from fuzzy match: {e}") import traceback @@ -28312,22 +24323,22 @@ def _run_post_processing_worker(task_id, batch_id): # Attempt to complete metadata enhancement using context if context and expected_final_filename: try: + context = normalize_import_context(context) # Extract required data from context - original_search = context.get("original_search_result", {}) - spotify_artist = context.get("spotify_artist") - spotify_album = context.get("spotify_album") + original_search = get_import_original_search(context) + artist_context = get_import_context_artist(context) + album_context = get_import_context_album(context) - if spotify_artist and spotify_album: + if artist_context and album_context: # CRITICAL FIX: Create album_info dict with proper structure for metadata enhancement # This must match the format used in main stream processor to ensure consistency # Extract track number from context (should be available from fuzzy match) - original_search = context.get("original_search_result", {}) track_number = original_search.get('track_number', 1) # If no track number in context, extract from filename if track_number == 1 and found_file: - track_number = _extract_track_number_from_filename(found_file) + track_number = extract_track_number_from_filename(found_file) logger.warning( "[Verification] missing track_number; extracted from filename=%r -> %s", os.path.basename(found_file), @@ -28339,17 +24350,22 @@ def _run_post_processing_worker(task_id, batch_id): logger.error(f"[Verification] Invalid track number ({track_number}), defaulting to 1") track_number = 1 - # Get clean track name - clean_track_name = (original_search.get('spotify_clean_title') or - original_search.get('title', 'Unknown Track')) + # Get clean track name + clean_track_name = get_import_clean_title(context, default=original_search.get('title', 'Unknown Track')) + album_name = get_import_clean_album(context, default=album_context.get('name', 'Unknown Album')) + album_image_url = album_context.get('image_url') + if not album_image_url and album_context.get('images'): + album_images = album_context.get('images', []) + if album_images and isinstance(album_images[0], dict): + album_image_url = album_images[0].get('url') album_info = { 'is_album': True, # CRITICAL: Mark as album track - 'album_name': spotify_album.get('name', 'Unknown Album'), # CORRECT KEY + 'album_name': album_name, 'track_number': track_number, # CORRECTED TRACK NUMBER 'disc_number': original_search.get('disc_number', 1), 'clean_track_name': clean_track_name, - 'album_image_url': spotify_album.get('images', [{}])[0].get('url') if spotify_album.get('images') else None, + 'album_image_url': album_image_url, 'confidence': 0.9, 'source': 'verification_worker_corrected' } @@ -28366,19 +24382,19 @@ def _run_post_processing_worker(task_id, batch_id): original_album_ctx = raw_album_ctx['name'] else: original_album_ctx = None - consistent_album_name = _resolve_album_group(spotify_artist, album_info, original_album_ctx) + consistent_album_name = _resolve_album_group(artist_context, album_info, original_album_ctx) album_info['album_name'] = consistent_album_name except Exception as group_err: logger.error(f"[Verification] Album grouping failed, using raw name: {group_err}") else: - logger.info(f"[Verification] Explicit album download - preserving Spotify album name: '{album_info['album_name']}'") + logger.info(f"[Verification] Explicit album download - preserving album name: '{album_info['album_name']}'") logger.info(f"[Verification] Created proper album_info - track_number: {track_number}, album: {album_info['album_name']}") logger.info(f"[Post-Processing] Attempting metadata enhancement for: {found_file}") - logger.warning(f"[Metadata Input] Verification worker - artist: '{spotify_artist.get('name', 'MISSING')}' (id: {spotify_artist.get('id', 'MISSING')})") + logger.warning(f"[Metadata Input] Verification worker - artist: '{artist_context.get('name', 'MISSING')}' (id: {artist_context.get('id', 'MISSING')})") logger.warning(f"[Metadata Input] Verification worker - album: '{album_info.get('album_name', 'MISSING')}', track#: {album_info.get('track_number', 'MISSING')}, source: {album_info.get('source', 'unknown')}") - enhancement_success = _enhance_file_metadata(found_file, context, spotify_artist, album_info) + enhancement_success = _enhance_file_metadata(found_file, context, artist_context, album_info) if enhancement_success: with tasks_lock: @@ -28388,8 +24404,8 @@ def _run_post_processing_worker(task_id, batch_id): else: logger.info(f"[Post-Processing] Metadata enhancement returned False for: {os.path.basename(found_file)}") else: - logger.warning("[Post-Processing] Missing spotify_artist or spotify_album in context") - logger.info(f"[Post-Processing] spotify_artist: {spotify_artist is not None}, spotify_album: {spotify_album is not None}") + logger.warning("[Post-Processing] Missing artist or album in context") + logger.info(f"[Post-Processing] artist_context: {artist_context is not None}, album_context: {album_context is not None}") # Wipe source tags even without full enhancement — prevents # Soulseek uploader's MusicBrainz IDs from causing album splits if found_file and os.path.exists(found_file): @@ -28466,7 +24482,6 @@ def _run_post_processing_worker(task_id, batch_id): download_tasks[task_id]['error_message'] = f"Critical post-processing error: {str(e)}" _on_download_completed(batch_id, task_id, success=False) - def _download_track_worker(task_id, batch_id=None): """ Enhanced download worker that matches the GUI's exact retry logic. @@ -29126,7 +25141,7 @@ def _get_staging_file_cache(batch_id): if batch_id in _staging_cache: return _staging_cache[batch_id] - staging_path = _get_staging_path() + staging_path = get_staging_path() if not os.path.isdir(staging_path): with _staging_cache_lock: _staging_cache[batch_id] = [] @@ -30654,7 +26669,7 @@ def _check_batch_completion_v2(batch_id): artist_name=_cons_artist_name, mb_service=_cons_mb_svc, total_discs=_cons_album.get('total_discs', 1), - file_lock_fn=_get_file_lock, + file_lock_fn=get_file_lock, ) if _cons_result.get('success'): logger.info(f"[Album Consistency V2] {_cons_result['tags_written']}/{_cons_result['total_files']} files " @@ -45808,142 +41823,6 @@ def cancel_listenbrainz_sync(playlist_mbid): logger.error(f"Error cancelling ListenBrainz sync: {e}") return jsonify({"error": str(e)}), 500 - -# OLD ENDPOINT - REMOVE ALL THE CODE BELOW FOR THE OLD IMPLEMENTATION -def _old_get_listenbrainz_playlist_tracks_DEPRECATED(playlist_mbid): - """DEPRECATED - Old implementation that fetches from API""" - try: - from core.listenbrainz_client import ListenBrainzClient - - client = ListenBrainzClient() - - playlist = client.get_playlist_details(playlist_mbid, fetch_metadata=True) - - if not playlist: - return jsonify({ - "success": False, - "error": "Playlist not found or not accessible" - }), 404 - - # Extract tracks from JSPF format - jspf_tracks = playlist.get('track', []) - - # Convert to our standard format - prepare tracks first without cover art - tracks = [] - logger.info(f"Processing {len(jspf_tracks)} tracks from playlist") - - # First pass: extract all track data without cover art - track_data_list = [] - for idx, track in enumerate(jspf_tracks): - # Get recording MBID from identifier - recording_mbid = None - identifiers = track.get('identifier', []) - for identifier in identifiers: - if 'musicbrainz.org/recording/' in identifier: - recording_mbid = identifier.split('/')[-1] - break - - # Get extension data (has MusicBrainz metadata) - extension = track.get('extension', {}) - mb_data = extension.get('https://musicbrainz.org/doc/jspf#track', {}) - - if idx == 0: - logger.debug(f"Sample track extension data: {extension}") - logger.debug(f"Sample mb_data keys: {mb_data.keys() if mb_data else 'None'}") - - # Extract release MBID for cover art - release_mbid = None - if mb_data: - # Check in additional_metadata first - additional_metadata = mb_data.get('additional_metadata', {}) - if 'caa_release_mbid' in additional_metadata: - release_mbid = additional_metadata['caa_release_mbid'] - # Fallback to top-level release_mbid - elif 'release_mbid' in mb_data: - release_mbid = mb_data['release_mbid'] - - if idx == 0: - logger.debug(f"🆔 First track release_mbid: {release_mbid}") - - track_data = { - 'track_name': track.get('title', 'Unknown Track'), - 'artist_name': track.get('creator', 'Unknown Artist'), - 'album_name': track.get('album', 'Unknown Album'), - 'duration_ms': track.get('duration', 0), - 'mbid': recording_mbid, - 'release_mbid': release_mbid, - 'album_cover_url': None, # Will be fetched in parallel - 'additional_metadata': mb_data - } - - track_data_list.append(track_data) - - # Second pass: fetch cover art in parallel using threading (much faster) - from concurrent.futures import ThreadPoolExecutor, as_completed - import time - - def fetch_cover_art(track_data): - """Fetch cover art for a single track""" - release_mbid = track_data.get('release_mbid') - if not release_mbid: - return None - - try: - cover_art_url = f"https://coverartarchive.org/release/{release_mbid}" - cover_response = requests.get(cover_art_url, timeout=3) - - if cover_response.status_code == 200: - cover_data = cover_response.json() - images = cover_data.get('images', []) - - # Get front cover - for img in images: - if img.get('front'): - return img.get('thumbnails', {}).get('small') or img.get('image') - - # Fallback to first image - if images: - return images[0].get('thumbnails', {}).get('small') or images[0].get('image') - except: - pass - - return None - - logger.info(f"Fetching cover art for {len(track_data_list)} tracks in parallel...") - start_time = time.time() - - # Fetch up to 10 covers at a time - with ThreadPoolExecutor(max_workers=10) as executor: - future_to_track = {executor.submit(fetch_cover_art, track): idx - for idx, track in enumerate(track_data_list)} - - for future in as_completed(future_to_track): - idx = future_to_track[future] - try: - cover_url = future.result() - if cover_url: - track_data_list[idx]['album_cover_url'] = cover_url - except Exception as e: - pass - - elapsed = time.time() - start_time - covers_found = sum(1 for t in track_data_list if t.get('album_cover_url')) - logger.info(f"Fetched {covers_found}/{len(track_data_list)} covers in {elapsed:.2f}s") - - tracks = track_data_list - - return jsonify({ - "success": True, - "tracks": tracks, - "track_count": len(tracks) - }) - - except Exception as e: - logger.error(f"Error getting ListenBrainz playlist tracks: {e}") - import traceback - traceback.print_exc() - return jsonify({"success": False, "error": str(e)}), 500 - @app.route('/api/metadata/start', methods=['POST']) def start_metadata_update(): """Start the metadata update process - EXACT copy of dashboard.py logic""" @@ -52006,17 +47885,11 @@ def repair_job_progress(): AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif', '.ape'} -def _get_staging_path(): - """Get the resolved staging folder path.""" - raw = config_manager.get('import.staging_path', './Staging') - return docker_resolve_path(raw) - - @app.route('/api/import/staging/files', methods=['GET']) def import_staging_files(): """Scan the staging folder and return audio files with tag metadata.""" try: - staging_path = _get_staging_path() + staging_path = get_staging_path() os.makedirs(staging_path, exist_ok=True) files = [] @@ -52028,7 +47901,7 @@ def import_staging_files(): full_path = os.path.join(root, fname) rel_path = os.path.relpath(full_path, staging_path) - meta = _read_staging_file_metadata(full_path, rel_path) + meta = read_staging_file_metadata(full_path, rel_path) files.append({ 'filename': fname, @@ -52058,7 +47931,7 @@ def import_staging_groups(): the same album+artist combo. Returns groups sorted by file count descending. """ try: - staging_path = _get_staging_path() + staging_path = get_staging_path() if not os.path.isdir(staging_path): return jsonify({'success': True, 'groups': []}) @@ -52072,7 +47945,7 @@ def import_staging_groups(): full_path = os.path.join(root, fname) rel_path = os.path.relpath(full_path, staging_path) - meta = _read_staging_file_metadata(full_path, rel_path) + meta = read_staging_file_metadata(full_path, rel_path) album = meta['album'] artist = meta['albumartist'] or meta['artist'] if not album or not artist: @@ -52118,7 +47991,7 @@ def import_staging_groups(): def import_staging_hints(): """Extract album search hints from staging folder (tags + folder names). Fast — no Spotify calls.""" try: - staging_path = _get_staging_path() + staging_path = get_staging_path() if not os.path.isdir(staging_path): return jsonify({'success': True, 'hints': []}) @@ -52182,34 +48055,20 @@ def import_staging_hints(): @app.route('/api/import/search/albums', methods=['GET']) def import_search_albums(): - """Search for albums via Spotify for import matching.""" + """Search for albums using the active metadata provider.""" try: query = request.args.get('q', '').strip() if not query: return jsonify({'success': False, 'error': 'Missing query parameter'}), 400 limit = min(int(request.args.get('limit', 12)), 50) + from core.metadata_service import get_primary_source - if _is_hydrabase_active(): - albums = hydrabase_client.search_albums(query, limit=limit) - else: - if hydrabase_worker and dev_mode_enabled: - hydrabase_worker.enqueue(query, 'albums') - albums = spotify_client.search_albums(query, limit=limit) + if get_primary_source() == 'hydrabase' and hydrabase_worker and dev_mode_enabled: + hydrabase_worker.enqueue(query, 'albums') - results = [] - for a in albums: - results.append({ - 'id': a.id, - 'name': a.name, - 'artist': ', '.join(a.artists) if a.artists else 'Unknown Artist', - 'release_date': a.release_date or '', - 'total_tracks': a.total_tracks, - 'image_url': a.image_url, - 'album_type': a.album_type or 'album' - }) - - return jsonify({'success': True, 'albums': results}) + albums = search_import_albums(query, limit=limit) + return jsonify({'success': True, 'albums': albums}) except Exception as e: logger.error(f"Error searching albums for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @@ -52219,204 +48078,24 @@ def import_search_albums(): def import_album_match(): """Match staging files to an album's tracklist.""" try: - data = request.get_json() + data = request.get_json() or {} album_id = data.get('album_id') album_name = data.get('album_name', '') album_artist = data.get('album_artist', '') + source = str(data.get('source') or '').strip().lower() # Optional: only match specific files (from auto-group selection) filter_file_paths = set(data.get('file_paths', [])) if not album_id: return jsonify({'success': False, 'error': 'Missing album_id'}), 400 - spotify_tracks = None - album_info = None - - # Try Hydrabase first when active — look up by album soul_id - if _is_hydrabase_active(): - try: - hydra_tracks = hydrabase_client.get_album_tracks(album_id, limit=50) - if hydra_tracks: - spotify_tracks = [] - for t in hydra_tracks: - artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else [] - spotify_tracks.append({ - 'name': t.name, - 'track_number': t.track_number or 0, - 'disc_number': t.disc_number or 1, - 'duration_ms': t.duration_ms, - 'id': t.id, - 'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list], - 'uri': '' - }) - album_info = { - 'id': album_id, - 'name': album_name or hydra_tracks[0].album or '', - 'artist': album_artist, - 'artists': [album_artist] if album_artist else [], - 'release_date': '', - 'total_tracks': len(spotify_tracks), - 'image_url': None, - 'genres': [] - } - logger.info(f"Hydrabase album_tracks returned {len(spotify_tracks)} tracks for album_id '{album_id}'") - except Exception as e: - logger.warning(f"Hydrabase album_tracks failed, falling back to Spotify: {e}") - spotify_tracks = None - - # Fall back to Spotify - if spotify_tracks is None: - album_data = spotify_client.get_album(album_id) - if not album_data: - return jsonify({'success': False, 'error': 'Album not found'}), 404 - - tracks_data = spotify_client.get_album_tracks(album_id) - if not tracks_data or 'items' not in tracks_data: - return jsonify({'success': False, 'error': 'Could not get album tracks'}), 500 - - spotify_tracks = tracks_data['items'] - - # Build album summary - album_artists = [a['name'] for a in album_data.get('artists', [])] - album_info = { - 'id': album_id, - 'name': album_data.get('name', 'Unknown Album'), - 'artist': ', '.join(album_artists), - 'artists': album_artists, - 'release_date': album_data.get('release_date', ''), - 'total_tracks': album_data.get('total_tracks', len(spotify_tracks)), - 'image_url': (album_data.get('images', [{}])[0].get('url') if album_data.get('images') else None), - 'genres': album_data.get('genres', []) - } - - # Get artist info for context building later - if album_data.get('artists'): - primary_artist = album_data['artists'][0] - album_info['artist_id'] = primary_artist.get('id', '') - - # Scan staging files - staging_path = _get_staging_path() - staging_files = [] - for root, _dirs, filenames in os.walk(staging_path): - for fname in filenames: - ext = os.path.splitext(fname)[1].lower() - if ext not in AUDIO_EXTENSIONS: - continue - full_path = os.path.join(root, fname) - - meta = _read_staging_file_metadata(full_path, fname) - - staging_files.append({ - 'filename': fname, - 'full_path': full_path, - 'title': meta['title'], - 'artist': meta['albumartist'] or meta['artist'], - 'album': meta['album'], - 'track_number': meta['track_number'], - 'disc_number': meta['disc_number'], - }) - - # Filter to specific files if requested (auto-group selection) - if filter_file_paths: - staging_files = [sf for sf in staging_files if sf['full_path'] in filter_file_paths] - - # Match each Spotify track to the best staging file - matches = [] - used_files = set() - album_name_for_match = album_info.get('name', '') - - for sp_track in spotify_tracks: - sp_name = sp_track.get('name', '') - sp_number = sp_track.get('track_number', 0) - sp_disc = sp_track.get('disc_number', 1) - sp_artists = sp_track.get('artists', []) - sp_artist_name = sp_artists[0]['name'] if sp_artists and isinstance(sp_artists[0], dict) else (sp_artists[0] if sp_artists else '') - - best_match = None - best_score = 0.0 - - for i, sf in enumerate(staging_files): - if i in used_files: - continue - - score = 0.0 - - # Title similarity (weight 0.45) - title_sim = matching_engine.similarity_score( - matching_engine.normalize_string(sp_name), - matching_engine.normalize_string(sf['title'] or '') - ) - score += title_sim * 0.45 - - # Artist similarity (weight 0.15) - sf_artist = sf.get('artist') or '' - if sf_artist and sp_artist_name: - artist_sim = matching_engine.similarity_score( - matching_engine.normalize_string(sp_artist_name), - matching_engine.normalize_string(sf_artist) - ) - score += artist_sim * 0.15 - else: - score += 0.075 # neutral when no artist data - - # Track number match (weight 0.30) - if sf['track_number'] and sp_number: - if sf['track_number'] == sp_number: - score += 0.30 - elif abs(sf['track_number'] - sp_number) <= 1: - score += 0.12 - - # Album tag bonus (weight 0.10) — reward files whose album tag matches - sf_album = sf.get('album') or '' - if sf_album and album_name_for_match: - album_sim = matching_engine.similarity_score( - matching_engine.normalize_string(sf_album), - matching_engine.normalize_string(album_name_for_match) - ) - score += album_sim * 0.10 - - if score > best_score and score >= 0.4: - best_score = score - best_match = i - - if best_match is not None: - used_files.add(best_match) - matches.append({ - 'spotify_track': { - 'name': sp_name, - 'track_number': sp_number, - 'disc_number': sp_disc, - 'duration_ms': sp_track.get('duration_ms', 0), - 'id': sp_track.get('id', ''), - 'artists': [a['name'] for a in sp_track.get('artists', [])], - 'uri': sp_track.get('uri', '') - }, - 'staging_file': staging_files[best_match], - 'confidence': round(best_score, 2) - }) - else: - matches.append({ - 'spotify_track': { - 'name': sp_name, - 'track_number': sp_number, - 'disc_number': sp_disc, - 'duration_ms': sp_track.get('duration_ms', 0), - 'id': sp_track.get('id', ''), - 'artists': [a['name'] for a in sp_track.get('artists', [])], - 'uri': sp_track.get('uri', '') - }, - 'staging_file': None, - 'confidence': 0 - }) - - # Unmatched staging files - unmatched_files = [sf for i, sf in enumerate(staging_files) if i not in used_files] - - return jsonify({ - 'success': True, - 'album': album_info, - 'matches': matches, - 'unmatched_files': unmatched_files - }) + payload = build_album_import_match_payload( + album_id, + album_name=album_name, + album_artist=album_artist, + file_paths=filter_file_paths, + source=source or None, + ) + return jsonify(payload) except Exception as e: logger.error(f"Error matching album for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @@ -52426,7 +48105,7 @@ def import_album_match(): def import_album_process(): """Process matched album files through the post-processing pipeline.""" try: - data = request.get_json() + data = request.get_json() or {} album = data.get('album', {}) matches = data.get('matches', []) @@ -52435,34 +48114,25 @@ def import_album_process(): processed = 0 errors = [] - album_name = album.get('name', 'Unknown Album') - artist_name = album.get('artist', 'Unknown Artist') - artist_id = album.get('artist_id', '') - album_id = album.get('id', '') + album_name = album.get('name', album.get('album_name', 'Unknown Album')) + artist_name = album.get('artist', album.get('artist_name', 'Unknown Artist')) + album_id = album.get('id', album.get('album_id', '')) + source = str(album.get('source') or data.get('source') or '').strip().lower() - # Get artist genres from Spotify if possible - artist_genres = album.get('genres', []) - if not artist_genres and artist_id and not _spotify_rate_limited(): - try: - from core.api_call_tracker import api_call_tracker - if hasattr(spotify_client, 'sp') and spotify_client.sp: - api_call_tracker.record_call('spotify', endpoint='artist') - sp_artist = spotify_client.sp.artist(artist_id) if hasattr(spotify_client, 'sp') and spotify_client.sp else None - if sp_artist: - artist_genres = sp_artist.get('genres', []) - except Exception: - pass - - # Compute total_discs across all matched tracks for multi-disc subfolder support total_discs = max( - (m['spotify_track'].get('disc_number', 1) for m in matches if m.get('spotify_track')), - default=1 + ( + match.get('track', {}).get('disc_number', 1) + for match in matches + if match.get('track') + ), + default=1, ) + artist_context = resolve_album_artist_context(album, source=source) for match in matches: staging_file = match.get('staging_file') - spotify_track = match.get('spotify_track') - if not staging_file or not spotify_track: + track = match.get('track') or {} + if not staging_file or not track: continue file_path = staging_file.get('full_path', '') @@ -52470,49 +48140,18 @@ def import_album_process(): errors.append(f"File not found: {staging_file.get('filename', '?')}") continue - track_name = spotify_track.get('name', 'Unknown Track') - track_number = spotify_track.get('track_number', 1) - disc_number = spotify_track.get('disc_number', 1) - track_artists = spotify_track.get('artists', [artist_name]) + track_name = track.get('name', 'Unknown Track') + track_number = track.get('track_number', 1) + disc_number = track.get('disc_number', 1) context_key = f"import_album_{album_id}_{track_number}_{uuid.uuid4().hex[:8]}" - context = { - 'spotify_artist': { - 'name': artist_name, - 'id': artist_id, - 'genres': artist_genres - }, - 'spotify_album': { - 'id': album_id, - 'name': album_name, - 'release_date': album.get('release_date', ''), - 'total_tracks': album.get('total_tracks', len(matches)), - 'total_discs': total_discs, - 'image_url': album.get('image_url', '') - }, - 'track_info': { - 'name': track_name, - 'id': spotify_track.get('id', ''), - 'track_number': track_number, - 'disc_number': disc_number, - 'duration_ms': spotify_track.get('duration_ms', 0), - 'artists': [{'name': a} if isinstance(a, str) else a for a in track_artists], - 'uri': spotify_track.get('uri', '') - }, - 'original_search_result': { - 'title': track_name, - 'artist': artist_name, - 'album': album_name, - 'track_number': track_number, - 'disc_number': disc_number, - 'spotify_clean_title': track_name, - 'spotify_clean_album': album_name, - 'artists': [{'name': a} if isinstance(a, str) else a for a in track_artists] - }, - 'is_album_download': True, - 'has_clean_spotify_data': True, - 'has_full_spotify_metadata': True - } + context = build_album_import_context( + album, + track, + artist_context=artist_context, + total_discs=total_discs, + source=source, + ) try: _post_process_matched_download(context_key, context, file_path) @@ -52561,35 +48200,18 @@ def import_album_process(): @app.route('/api/import/search/tracks', methods=['GET']) def import_search_tracks(): - """Search Spotify for individual tracks (used for manual singles identification).""" + """Search tracks using the configured metadata provider priority order.""" try: query = request.args.get('q', '').strip() if not query: return jsonify({'success': False, 'error': 'Missing query parameter'}), 400 limit = min(int(request.args.get('limit', 10)), 30) + if get_primary_source() == 'hydrabase' and hydrabase_worker and dev_mode_enabled: + hydrabase_worker.enqueue(query, 'tracks') - if _is_hydrabase_active(): - tracks = hydrabase_client.search_tracks(query, limit=limit) - else: - if hydrabase_worker and dev_mode_enabled: - hydrabase_worker.enqueue(query, 'tracks') - tracks = spotify_client.search_tracks(query, limit=limit) - - results = [] - for t in tracks: - results.append({ - 'id': t.id, - 'name': t.name, - 'artist': ', '.join(t.artists) if hasattr(t, 'artists') and t.artists else 'Unknown Artist', - 'album': t.album if hasattr(t, 'album') else '', - 'album_id': t.album_id if hasattr(t, 'album_id') else '', - 'duration_ms': t.duration_ms if hasattr(t, 'duration_ms') else 0, - 'image_url': t.image_url if hasattr(t, 'image_url') else '', - 'track_number': t.track_number if hasattr(t, 'track_number') else 1, - }) - - return jsonify({'success': True, 'tracks': results}) + tracks = search_import_tracks(query, limit=limit) + return jsonify({'success': True, 'tracks': tracks}) except Exception as e: logger.error(f"Error searching tracks for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @@ -52616,180 +48238,51 @@ def import_singles_process(): title = file_info.get('title', '') artist = file_info.get('artist', '') - spotify_override = file_info.get('spotify_override', None) + manual_match = file_info.get('manual_match') + if manual_match is not None and not isinstance(manual_match, dict): + manual_match = None + + manual_match_source = '' + manual_match_id = None + if manual_match: + manual_match_source = str(manual_match.get('source') or '').strip().lower() + manual_match_id = str(manual_match.get('id') or '').strip() + if not manual_match_id or not manual_match_source: + errors.append(f"Malformed manual match for file: {file_info.get('filename', '?')}") + continue # Fallback to filename parsing if no metadata - if not title and not spotify_override: - parsed = _parse_filename_metadata(file_info.get('filename', '')) - title = parsed.get('title', os.path.splitext(file_info.get('filename', 'Unknown'))[0]) + if not title and not manual_match: + parsed = parse_filename_metadata(file_info.get('filename', '')) + title = parsed.get('title') or os.path.splitext(file_info.get('filename', 'Unknown'))[0] if not artist: artist = parsed.get('artist', '') - # Search Spotify for rich metadata - spotify_track_data = None - spotify_artist_data = None - spotify_album_data = None + from core.metadata_service import get_single_track_import_context - # If a manual spotify_override is provided, look up that specific track - if spotify_override and spotify_override.get('id'): - try: - override_id = spotify_override['id'] - sp_track = None - if hasattr(spotify_client, 'sp') and spotify_client.sp and not _spotify_rate_limited(): - from core.api_call_tracker import api_call_tracker - api_call_tracker.record_call('spotify', endpoint='track') - sp_track = spotify_client.sp.track(override_id) - if sp_track: - sp_track_artists = sp_track.get('artists', []) - spotify_track_data = { - 'name': sp_track.get('name', ''), - 'id': override_id, - 'track_number': sp_track.get('track_number', 1), - 'disc_number': sp_track.get('disc_number', 1), - 'duration_ms': sp_track.get('duration_ms', 0), - 'artists': [{'name': a.get('name', '')} for a in sp_track_artists], - 'uri': sp_track.get('uri', f"spotify:track:{override_id}") - } - title = sp_track.get('name', title) - artist = sp_track_artists[0].get('name', artist) if sp_track_artists else artist - # Get album info - sp_album_info = sp_track.get('album', {}) - if sp_album_info: - spotify_album_data = { - 'id': sp_album_info.get('id', ''), - 'name': sp_album_info.get('name', ''), - 'release_date': sp_album_info.get('release_date', ''), - 'total_tracks': sp_album_info.get('total_tracks', 1), - 'image_url': (sp_album_info.get('images', [{}])[0].get('url') if sp_album_info.get('images') else ''), - 'album_type': sp_album_info.get('album_type', 'album'), - } - album_artists = sp_album_info.get('artists', []) - if album_artists: - spotify_artist_data = { - 'name': album_artists[0].get('name', artist), - 'id': album_artists[0].get('id', ''), - 'genres': [] - } - try: - sp_a = None - if hasattr(spotify_client, 'sp') and spotify_client.sp and not _spotify_rate_limited(): - from core.api_call_tracker import api_call_tracker - api_call_tracker.record_call('spotify', endpoint='artist') - sp_a = spotify_client.sp.artist(album_artists[0]['id']) - if sp_a: - spotify_artist_data['genres'] = sp_a.get('genres', []) - except Exception: - pass - except Exception as override_err: - logger.warning(f"Spotify override lookup failed for track {spotify_override.get('id')}: {override_err}") - - if not spotify_track_data and title: - try: - search_q = f"{title} {artist}" if artist else title - tracks = spotify_client.search_tracks(search_q, limit=1) - if tracks: - t = tracks[0] - spotify_track_data = { - 'name': t.name, - 'id': t.id, - 'track_number': t.track_number if hasattr(t, 'track_number') else 1, - 'disc_number': 1, - 'duration_ms': t.duration_ms if hasattr(t, 'duration_ms') else 0, - 'artists': [{'name': a} for a in (t.artists if hasattr(t, 'artists') else [artist])], - 'uri': f"spotify:track:{t.id}" - } - # Get album info from the track's album - if hasattr(t, 'album_id') and t.album_id: - sp_album = spotify_client.get_album(t.album_id) - if sp_album: - spotify_album_data = { - 'id': t.album_id, - 'name': sp_album.get('name', ''), - 'release_date': sp_album.get('release_date', ''), - 'total_tracks': sp_album.get('total_tracks', 1), - 'image_url': (sp_album.get('images', [{}])[0].get('url') if sp_album.get('images') else ''), - 'album_type': sp_album.get('album_type', 'album'), - } - # Get artist genres - sp_artists = sp_album.get('artists', []) - if sp_artists: - spotify_artist_data = { - 'name': sp_artists[0].get('name', artist), - 'id': sp_artists[0].get('id', ''), - 'genres': [] - } - try: - sp_a = None - if hasattr(spotify_client, 'sp') and spotify_client.sp and not _spotify_rate_limited(): - from core.api_call_tracker import api_call_tracker - api_call_tracker.record_call('spotify', endpoint='artist') - sp_a = spotify_client.sp.artist(sp_artists[0]['id']) - if sp_a: - spotify_artist_data['genres'] = sp_a.get('genres', []) - except Exception: - pass - - # Fallback artist data from track - if not spotify_artist_data: - track_artists = t.artists if hasattr(t, 'artists') else [artist] - spotify_artist_data = { - 'name': track_artists[0] if track_artists else artist, - 'id': '', - 'genres': [] - } - - # Fallback album data - if not spotify_album_data: - spotify_album_data = { - 'id': '', - 'name': t.album if hasattr(t, 'album') else '', - 'release_date': '', - 'total_tracks': 1, - 'image_url': t.image_url if hasattr(t, 'image_url') else '', - 'album_type': t.album_type if hasattr(t, 'album_type') else 'album', - } - except Exception as sp_err: - logger.warning(f"Spotify lookup failed for '{title}': {sp_err}") - - # Build context — use Spotify data if found, else use file metadata - if not spotify_artist_data: - spotify_artist_data = {'name': artist or 'Unknown Artist', 'id': '', 'genres': []} - if not spotify_album_data: - spotify_album_data = {'id': '', 'name': '', 'release_date': '', 'total_tracks': 1, 'image_url': '', 'album_type': 'album'} - if not spotify_track_data: - spotify_track_data = { - 'name': title, 'id': '', 'track_number': 1, 'disc_number': 1, - 'duration_ms': 0, 'artists': [{'name': artist or 'Unknown Artist'}], 'uri': '' - } - - final_title = spotify_track_data.get('name', title) - final_artist = spotify_artist_data.get('name', artist) - final_album = spotify_album_data.get('name', '') + resolved = get_single_track_import_context( + title, + artist, + override_id=manual_match_id, + override_source=manual_match_source, + ) + context = normalize_import_context(resolved['context']) + artist_data = get_import_context_artist(context) + track_data = get_import_track_info(context) + final_title = track_data.get('name', title) + final_artist = artist_data.get('name', artist) context_key = f"import_single_{uuid.uuid4().hex[:8]}" - context = { - 'spotify_artist': spotify_artist_data, - 'spotify_album': spotify_album_data, - 'track_info': spotify_track_data, - 'original_search_result': { - 'title': final_title, - 'artist': final_artist, - 'album': final_album, - 'track_number': spotify_track_data.get('track_number', 1), - 'disc_number': 1, - 'spotify_clean_title': final_title, - 'spotify_clean_album': final_album, - 'artists': spotify_track_data.get('artists', [{'name': final_artist}]) - }, - 'is_album_download': False, - 'has_clean_spotify_data': bool(spotify_track_data.get('id')), - 'has_full_spotify_metadata': bool(spotify_track_data.get('id')) - } try: _post_process_matched_download(context_key, context, file_path) processed += 1 - logger.info(f"Import single processed: {final_title} by {final_artist}") + logger.info( + "Import single processed: %s by %s (source=%s)", + final_title, + final_artist, + resolved.get('source') or 'local', + ) except Exception as proc_err: err_msg = f"{title}: {str(proc_err)}" errors.append(err_msg) @@ -52830,126 +48323,6 @@ def import_singles_process(): logger.error(f"Error processing singles import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 -# --- Import Suggestion Cache (server-side background builder) --- - -_import_suggestions_cache = { - 'suggestions': [], - 'building': False, - 'built': False, -} - -def _build_import_suggestions_background(): - """Background thread: extract hints from staging folder, search Spotify, cache results.""" - cache = _import_suggestions_cache - if cache['building']: - return - cache['building'] = True - - try: - staging_path = _get_staging_path() - if not os.path.isdir(staging_path): - cache['suggestions'] = [] - cache['built'] = True - cache['building'] = False - return - - # Reuse the hint extraction logic - tag_albums = {} - folder_hints = {} - - for root, _dirs, filenames in os.walk(staging_path): - audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS] - if not audio_files: - continue - rel_dir = os.path.relpath(root, staging_path) - if rel_dir != '.': - top_folder = rel_dir.split(os.sep)[0] - folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files) - for fname in audio_files: - full_path = os.path.join(root, fname) - try: - from mutagen import File as MutagenFile - tags = MutagenFile(full_path, easy=True) - if tags: - album = (tags.get('album') or [None])[0] - artist = (tags.get('artist') or (tags.get('albumartist') or [None]))[0] - if album: - key = (album.strip(), (artist or '').strip()) - tag_albums[key] = tag_albums.get(key, 0) + 1 - except Exception: - pass - - queries = [] - seen_lower = set() - for (album, artist), _ in sorted(tag_albums.items(), key=lambda x: -x[1]): - q = f"{album} {artist}".strip() if artist else album - if q.lower() not in seen_lower: - seen_lower.add(q.lower()) - queries.append(q) - for folder, _ in sorted(folder_hints.items(), key=lambda x: -x[1]): - q = folder.replace('_', ' ') - if q.lower() not in seen_lower: - seen_lower.add(q.lower()) - queries.append(q) - queries = queries[:5] - - if not queries: - cache['suggestions'] = [] - cache['built'] = True - cache['building'] = False - return - - suggestions = [] - seen_ids = set() - for q in queries: - try: - if not spotify_client: - break - albums = spotify_client.search_albums(q, limit=2) - for a in albums: - if a.id not in seen_ids: - seen_ids.add(a.id) - suggestions.append({ - 'id': a.id, - 'name': a.name, - 'artist': ', '.join(a.artists) if a.artists else 'Unknown Artist', - 'release_date': a.release_date or '', - 'total_tracks': a.total_tracks, - 'image_url': a.image_url, - 'album_type': a.album_type or 'album', - }) - except Exception as e: - logger.warning(f"Import suggestion search failed for '{q}': {e}") - - cache['suggestions'] = suggestions[:8] - cache['built'] = True - logger.info(f"Import suggestions cache built: {len(cache['suggestions'])} suggestions from {len(queries)} hints") - except Exception as e: - logger.error(f"Error building import suggestions cache: {e}") - cache['suggestions'] = [] - cache['built'] = True - finally: - cache['building'] = False - - -def start_import_suggestions_cache(): - """Start building the import suggestions cache in a background thread (called on server startup).""" - threading.Thread( - target=_build_import_suggestions_background, - daemon=True, - name='import-suggestions-cache' - ).start() - - -def refresh_import_suggestions_cache(): - """Invalidate and rebuild the suggestions cache (called after imports change staging contents).""" - _import_suggestions_cache['built'] = False - threading.Thread( - target=_build_import_suggestions_background, - daemon=True, - name='import-suggestions-cache' - ).start() - # ── Auto-Import Worker ── auto_import_worker = None @@ -53089,7 +48462,7 @@ def auto_import_clear_completed(): @app.route('/api/import/staging/suggestions', methods=['GET']) def import_staging_suggestions(): """Return cached import suggestions. If cache isn't built yet, returns partial/empty with a flag.""" - cache = _import_suggestions_cache + cache = get_import_suggestions_cache() return jsonify({ 'success': True, 'suggestions': cache['suggestions'],