Merge pull request #378 from kettui/refactor/extract-import-pipelines

Refactor import and related post-processing pipelines
This commit is contained in:
BoulderBadgeDad 2026-04-27 13:19:59 -07:00 committed by GitHub
commit f7b01f476a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
53 changed files with 9559 additions and 5173 deletions

View file

@ -5,6 +5,7 @@ Download management endpoints — list, cancel active downloads.
from flask import request, current_app
from .auth import require_api_key
from .helpers import api_success, api_error
from core.runtime_state import download_tasks, tasks_lock
def _serialize_download(task_id, task):
@ -53,8 +54,6 @@ def register_routes(bp):
descending so newest/in-flight tasks appear first.
"""
try:
from web_server import download_tasks, tasks_lock
# Parse pagination params
try:
limit = int(request.args.get("limit", 100))

View file

@ -55,7 +55,7 @@ def register_routes(bp):
def system_activity():
"""Recent activity feed."""
try:
from web_server import activity_feed
from core.runtime_state import activity_feed
items = list(activity_feed) if activity_feed else []
return api_success({"activities": items})
except Exception as e:
@ -74,7 +74,7 @@ def register_routes(bp):
# Active download count
download_count = 0
try:
from web_server import download_tasks, tasks_lock
from core.runtime_state import download_tasks, tasks_lock
with tasks_lock:
download_count = sum(
1 for t in download_tasks.values()

1
core/imports/__init__.py Normal file
View file

@ -0,0 +1 @@
"""Import flow helpers package."""

478
core/imports/album.py Normal file
View file

@ -0,0 +1,478 @@
"""Album import helpers for staging matching and post-processing context."""
from __future__ import annotations
from typing import Any, Dict, Iterable, List, Optional, Set
from core.imports.context import normalize_import_context
from core.imports.staging import collect_staging_files
from utils.logging_config import get_logger
logger = get_logger("imports.album")
def get_client_for_source(source: str):
from core.metadata_service import get_client_for_source as _get_client_for_source
return _get_client_for_source(source)
def get_artist_album_tracks(
album_id: str,
artist_name: str = "",
album_name: str = "",
source: Optional[str] = None,
):
from core.metadata_service import get_artist_album_tracks as _get_artist_album_tracks
return _get_artist_album_tracks(
album_id,
artist_name=artist_name,
album_name=album_name,
source_override=source,
)
try:
from core.matching_engine import MusicMatchingEngine
_MATCHING_ENGINE_IMPORT_ERROR = None
except Exception as exc: # pragma: no cover - only hits in stripped-down environments
MusicMatchingEngine = None # type: ignore[assignment]
_MATCHING_ENGINE_IMPORT_ERROR = exc
_MATCHING_ENGINE = None
def _get_matching_engine() -> Any:
global _MATCHING_ENGINE
if _MATCHING_ENGINE is None:
if MusicMatchingEngine is None:
raise RuntimeError("Music matching engine is unavailable") from _MATCHING_ENGINE_IMPORT_ERROR
_MATCHING_ENGINE = MusicMatchingEngine()
return _MATCHING_ENGINE
def _normalize_artist_entries(artists: Any) -> List[Dict[str, Any]]:
if not artists:
return []
if isinstance(artists, (str, bytes)):
artists = [artists]
elif isinstance(artists, dict):
artists = [artists]
else:
try:
artists = list(artists)
except TypeError:
artists = [artists]
normalized: List[Dict[str, Any]] = []
for artist in artists:
if isinstance(artist, dict):
entry: Dict[str, Any] = {}
name = artist.get("name") or artist.get("artist_name") or artist.get("title") or ""
artist_id = artist.get("id") or artist.get("artist_id") or ""
if name:
entry["name"] = str(name)
if artist_id:
entry["id"] = str(artist_id)
genres = artist.get("genres")
if genres is not None:
entry["genres"] = genres
if entry:
normalized.append(entry)
continue
name = str(artist).strip()
if name:
normalized.append({"name": name})
return normalized
def _normalize_album_source(album: Dict[str, Any], source: str = "") -> str:
album_source = source or album.get("source") or ""
return str(album_source).strip().lower()
def _strip_legacy_source_fields(payload: Any) -> Any:
if not isinstance(payload, dict):
return payload
cleaned = dict(payload)
cleaned.pop("_source", None)
cleaned.pop("provider", None)
return cleaned
def _extract_track_artist_name(track: Dict[str, Any]) -> str:
artists = track.get("artists") or []
if isinstance(artists, (str, bytes)):
artists = [artists]
elif isinstance(artists, dict):
artists = [artists]
else:
try:
artists = list(artists)
except TypeError:
artists = [artists]
if not artists:
return ""
first = artists[0]
if isinstance(first, dict):
return str(first.get("name") or first.get("artist_name") or first.get("title") or "").strip()
return str(first or "").strip()
def _coerce_track_int(value: Any, default: int = 1) -> int:
if value in (None, ""):
return default
try:
return int(str(value).split("/")[0].strip() or default)
except (TypeError, ValueError):
return default
def _normalize_match_track(track: Dict[str, Any], source: str, album: Dict[str, Any]) -> Dict[str, Any]:
track_album = track.get("album") if isinstance(track.get("album"), dict) else album
if isinstance(track_album, dict):
track_album = _strip_legacy_source_fields(track_album)
track_source = _normalize_album_source(track, source)
track_artists = _normalize_artist_entries(track.get("artists") or [])
if not track_artists and album.get("artists"):
track_artists = _normalize_artist_entries(album.get("artists"))
return {
"id": track.get("id", ""),
"name": track.get("name", "Unknown Track"),
"track_number": _coerce_track_int(track.get("track_number", 1), default=1),
"disc_number": _coerce_track_int(track.get("disc_number", 1), default=1),
"duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0),
"artists": track_artists,
"uri": track.get("uri", ""),
"album": track_album,
"source": track_source,
}
def _score_album_track_match(track: Dict[str, Any], staging_file: Dict[str, Any], album_name: str) -> float:
engine = _get_matching_engine()
track_name = track.get("name", "")
staging_title = staging_file.get("title", "")
score = 0.0
title_sim = engine.similarity_score(
engine.normalize_string(track_name),
engine.normalize_string(staging_title or ""),
)
score += title_sim * 0.45
track_artist_name = _extract_track_artist_name(track)
staging_artist = staging_file.get("artist") or ""
if track_artist_name and staging_artist:
artist_sim = engine.similarity_score(
engine.normalize_string(track_artist_name),
engine.normalize_string(staging_artist),
)
score += artist_sim * 0.15
else:
score += 0.075
track_number = _coerce_track_int(track.get("track_number", 1), default=1)
staging_track_number = _coerce_track_int(staging_file.get("track_number", 1), default=1)
if staging_track_number and track_number:
if staging_track_number == track_number:
score += 0.30
elif abs(staging_track_number - track_number) <= 1:
score += 0.12
staging_album = staging_file.get("album") or ""
if staging_album and album_name:
album_sim = engine.similarity_score(
engine.normalize_string(staging_album),
engine.normalize_string(album_name),
)
score += album_sim * 0.10
return score
def _fetch_artist_data_for_source(client: Any, artist_id: str, source: str) -> Any:
if source == "spotify":
try:
return client.get_artist(artist_id, allow_fallback=False)
except TypeError:
return client.get_artist(artist_id)
return client.get_artist(artist_id)
def resolve_album_artist_context(album: Dict[str, Any], source: str = "") -> Dict[str, Any]:
"""Build a neutral artist context for album import processing."""
album = dict(album or {})
source = _normalize_album_source(album, source)
artists = _normalize_artist_entries(album.get("artists") or [])
if not artists:
artist_name = album.get("artist") or album.get("artist_name") or ""
artist_id = album.get("artist_id") or ""
if artist_name or artist_id:
artist_entry: Dict[str, Any] = {}
if artist_name:
artist_entry["name"] = str(artist_name)
if artist_id:
artist_entry["id"] = str(artist_id)
artists = [artist_entry]
primary_artist = artists[0] if artists else {}
artist_name = str(
primary_artist.get("name")
or album.get("artist")
or album.get("artist_name")
or "Unknown Artist"
).strip()
artist_id = str(primary_artist.get("id") or album.get("artist_id") or "").strip()
genres: List[Any] = []
if artist_id and source:
client = get_client_for_source(source)
if client and hasattr(client, "get_artist"):
try:
artist_data = _fetch_artist_data_for_source(client, artist_id, source)
raw_genres = artist_data.get("genres") if isinstance(artist_data, dict) else getattr(artist_data, "genres", [])
if isinstance(raw_genres, str):
genres = [raw_genres]
elif raw_genres:
try:
genres = list(raw_genres)
except TypeError:
genres = [raw_genres]
except Exception as exc:
logger.debug("Could not resolve artist genres for %s on %s: %s", artist_id, source, exc)
return {
"id": artist_id,
"name": artist_name,
"genres": genres,
"source": source,
}
def build_album_import_context(
album: Dict[str, Any],
track: Dict[str, Any],
*,
artist_context: Optional[Dict[str, Any]] = None,
total_discs: int = 1,
source: str = "",
) -> Dict[str, Any]:
"""Build a neutral post-processing context for one album track."""
album = dict(album or {})
track = dict(track or {})
source = _normalize_album_source(album, source)
album_artists = _normalize_artist_entries(album.get("artists") or [])
if not album_artists and artist_context:
album_artists = _normalize_artist_entries([artist_context])
if artist_context:
artist_ctx = dict(artist_context)
else:
artist_ctx = resolve_album_artist_context(album, source)
artist_ctx = _strip_legacy_source_fields(artist_ctx)
artist_ctx.setdefault("genres", [])
artist_ctx.setdefault("source", source)
artist_ctx["genres"] = artist_ctx.get("genres") or []
track_artists = _normalize_artist_entries(track.get("artists") or [])
if not track_artists:
track_artists = album_artists or [artist_ctx]
track_album_value = track.get("album")
if isinstance(track_album_value, dict):
track_album_name = (
track_album_value.get("name")
or track_album_value.get("title")
or album.get("name")
or album.get("album_name")
or ""
)
track_album_id = str(track_album_value.get("id") or track_album_value.get("album_id") or "").strip()
track_album_type = track_album_value.get("album_type") or album.get("album_type") or "album"
track_album_release = track_album_value.get("release_date") or album.get("release_date") or ""
track_album_image = track_album_value.get("image_url") or album.get("image_url") or ""
else:
track_album_name = str(track_album_value or album.get("name") or album.get("album_name") or "").strip()
track_album_id = str(album.get("id") or album.get("album_id") or "").strip()
track_album_type = album.get("album_type") or "album"
track_album_release = album.get("release_date") or ""
track_album_image = album.get("image_url") or ""
album_name = str(album.get("name") or album.get("album_name") or track_album_name or "Unknown Album").strip()
artist_name = str(
artist_ctx.get("name")
or album.get("artist")
or album.get("artist_name")
or "Unknown Artist"
).strip()
track_number = _coerce_track_int(track.get("track_number", 1), default=1)
disc_number = _coerce_track_int(track.get("disc_number", 1), default=1)
normalized_track = {
"id": str(track.get("id") or track.get("track_id") or "").strip(),
"name": str(track.get("name") or "Unknown Track").strip(),
"track_number": track_number,
"disc_number": disc_number,
"duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0),
"artists": track_artists,
"uri": str(track.get("uri") or "").strip(),
"album": track_album_name,
"album_id": track_album_id,
"album_type": track_album_type,
"release_date": track_album_release,
"source": source,
}
normalized_album = {
"id": str(album.get("id") or album.get("album_id") or track_album_id or "").strip(),
"name": album_name,
"artist": artist_name,
"artist_name": artist_name,
"artist_id": str(artist_ctx.get("id") or album.get("artist_id") or "").strip(),
"artists": album_artists,
"release_date": str(album.get("release_date") or track_album_release or "").strip(),
"total_tracks": int(album.get("total_tracks") or track.get("total_tracks") or 0) or 1,
"total_discs": int(total_discs or 1) if str(total_discs or 1).isdigit() else total_discs or 1,
"album_type": str(album.get("album_type") or track_album_type or "album").strip() or "album",
"image_url": str(album.get("image_url") or track_album_image or "").strip(),
"images": album.get("images") or ([] if not track_album_image else [{"url": track_album_image}]),
"source": source,
}
original_search = {
"title": normalized_track["name"],
"artist": artist_name,
"album": album_name,
"track_number": track_number,
"disc_number": disc_number,
"clean_title": normalized_track["name"],
"clean_album": album_name,
"clean_artist": artist_name,
"artists": track_artists,
"duration_ms": normalized_track["duration_ms"],
"id": normalized_track["id"],
"source": source,
}
context = {
"artist": artist_ctx,
"album": normalized_album,
"track_info": normalized_track,
"original_search_result": original_search,
"is_album_download": True,
"has_clean_metadata": bool(normalized_track["id"]),
"has_full_metadata": bool(normalized_track["id"]),
"source": source,
}
normalized_context = normalize_import_context(context)
normalized_context["artist"] = _strip_legacy_source_fields(normalized_context.get("artist"))
normalized_context["album"] = _strip_legacy_source_fields(normalized_context.get("album"))
normalized_context["track_info"] = _strip_legacy_source_fields(normalized_context.get("track_info"))
normalized_context["original_search_result"] = _strip_legacy_source_fields(normalized_context.get("original_search_result"))
return normalized_context
def build_album_import_match_payload(
album_id: str,
*,
album_name: str = "",
album_artist: str = "",
file_paths: Optional[Iterable[str]] = None,
source: Optional[str] = None,
) -> Dict[str, Any]:
"""Build the album import match payload using provider-priority metadata lookup."""
album_response = get_artist_album_tracks(
album_id,
artist_name=album_artist,
album_name=album_name,
source=source,
)
album = _strip_legacy_source_fields(dict(album_response.get("album") or {}))
source = _normalize_album_source(album, album_response.get("source") or source or "")
tracks = list(album_response.get("tracks") or [])
if not album_response.get("success") or not tracks:
return {
"success": False,
"error": album_response.get("error", "Album not found"),
"status_code": album_response.get("status_code", 404),
"album": {
"id": album_id,
"name": album_name or album_id,
"artist": album_artist or "Unknown Artist",
"artist_name": album_artist or "Unknown Artist",
"artist_id": "",
"artists": [],
"release_date": "",
"total_tracks": 0,
"total_discs": 1,
"album_type": "album",
"image_url": "",
"images": [],
"source": source,
},
"matches": [],
"unmatched_files": [],
"source": source,
"source_priority": album_response.get("source_priority", []),
"resolved_album_id": album_response.get("resolved_album_id") or album_id,
}
staging_files = collect_staging_files(file_paths)
album_name_for_match = album.get("name") or album_name or ""
matches: List[Dict[str, Any]] = []
used_files: Set[int] = set()
for track in tracks:
normalized_track = _normalize_match_track(track, source, album)
best_match = None
best_score = 0.0
for index, staging_file in enumerate(staging_files):
if index in used_files:
continue
score = _score_album_track_match(normalized_track, staging_file, album_name_for_match)
if score > best_score and score >= 0.4:
best_score = score
best_match = index
matches.append(
{
"track": normalized_track,
"staging_file": staging_files[best_match] if best_match is not None else None,
"confidence": round(best_score, 2) if best_match is not None else 0,
}
)
if best_match is not None:
used_files.add(best_match)
unmatched_files = [sf for index, sf in enumerate(staging_files) if index not in used_files]
return {
"success": True,
"album": album,
"matches": matches,
"unmatched_files": unmatched_files,
"source": source,
"source_priority": album_response.get("source_priority", []),
"resolved_album_id": album_response.get("resolved_album_id") or album_id,
}

View file

@ -0,0 +1,184 @@
"""Album naming and grouping helpers used by import flows."""
from __future__ import annotations
import re
import threading
from typing import Any, Dict
from core.imports.context import extract_artist_name
from utils.logging_config import get_logger
logger = get_logger("imports.album_naming")
_album_cache_lock = threading.Lock()
_album_editions: dict[str, str] = {}
_album_name_cache: dict[str, str] = {}
def clear_album_grouping_cache() -> None:
"""Clear cached album grouping decisions.
Useful for tests and for any future config reload flows.
"""
with _album_cache_lock:
_album_editions.clear()
_album_name_cache.clear()
def get_base_album_name(album_name: str) -> str:
"""Extract the base album name without edition indicators."""
base_name = album_name or ""
base_name = re.sub(
r"\s*[\[\(][^)\]]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^)\]]*[\]\)]\s*$",
"",
base_name,
flags=re.IGNORECASE,
)
base_name = re.sub(r"\s*[\[\(][^)\]]*\bedition\b[^)\]]*[\]\)]\s*$", "", base_name, flags=re.IGNORECASE)
base_name = re.sub(
r"\s+(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$",
"",
base_name,
flags=re.IGNORECASE,
)
return base_name.strip()
def detect_deluxe_edition(album_name: str) -> bool:
"""Detect if an album name indicates a deluxe/special edition."""
if not album_name:
return False
album_lower = album_name.lower()
deluxe_indicators = [
"deluxe",
"deluxe edition",
"special edition",
"expanded edition",
"extended edition",
"bonus",
"remastered",
"anniversary",
"collectors edition",
"limited edition",
"silver edition",
"gold edition",
"platinum edition",
]
for indicator in deluxe_indicators:
if indicator in album_lower:
logger.info("Detected deluxe edition: %r contains %r", album_name, indicator)
return True
return False
def normalize_base_album_name(base_album: str, artist_name: str) -> str:
"""Normalize the base album name to handle case variations and known corrections."""
normalized_lower = (base_album or "").lower().strip()
known_corrections = {
# Add specific album name corrections here as needed.
}
for variant, correction in known_corrections.items():
if normalized_lower == variant.lower():
logger.info("Album correction applied: %r -> %r", base_album, correction)
return correction
normalized = base_album or ""
normalized = re.sub(r"\s*&\s*", " & ", normalized)
normalized = re.sub(r"\s+", " ", normalized)
normalized = normalized.strip()
logger.info("Album variant normalization: %r -> %r", base_album, normalized)
return normalized
def clean_album_title(album_title: str, artist_name: str) -> str:
"""Clean up album title by removing common prefixes, suffixes, and artist redundancy."""
original = (album_title or "").strip()
cleaned = original
logger.info("Album Title Cleaning: %r (artist: %r)", original, artist_name)
cleaned = re.sub(r"^Album\s*-\s*", "", cleaned, flags=re.IGNORECASE)
artist_pattern = re.escape(artist_name or "") + r"\s*-\s*"
cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"\s*[\[\(]\d{4}[\]\)]\s*", " ", cleaned)
quality_patterns = [
r"\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*",
r"\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*",
r"\s*[\[\(].*?flac.*?[\]\)]\s*",
r"\s*[\[\(].*?mp3.*?[\]\)]\s*",
r"\s*[\[\(].*?itunes.*?[\]\)]\s*",
r"\s*[\[\(].*?web.*?[\]\)]\s*",
r"\s*[\[\(].*?cd.*?[\]\)]\s*",
]
for pattern in quality_patterns:
cleaned = re.sub(pattern, " ", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"\s*[\[\(][^\]\)]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"\s*[\[\(][^\]\)]*\bedition\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"\s*(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", "", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"^[-\s\.]+", "", cleaned)
cleaned = re.sub(r"[-\s\.]+$", "", cleaned)
cleaned = re.sub(r"\s+", " ", cleaned).strip()
return cleaned if cleaned else original
def resolve_album_group(artist_context: dict, album_info: dict, original_album: str = None) -> str:
"""Smart album grouping: upgrade to deluxe if any track is deluxe."""
try:
with _album_cache_lock:
artist_name = extract_artist_name(artist_context)
detected_album = (album_info or {}).get("album_name", "")
if detected_album:
base_album = get_base_album_name(detected_album)
elif original_album:
cleaned_original = clean_album_title(original_album, artist_name)
base_album = get_base_album_name(cleaned_original)
else:
base_album = get_base_album_name(detected_album)
base_album = normalize_base_album_name(base_album, artist_name)
album_key = f"{artist_name}::{base_album}"
is_deluxe_track = False
if detected_album:
is_deluxe_track = detect_deluxe_edition(detected_album)
elif original_album:
is_deluxe_track = detect_deluxe_edition(original_album)
if album_key in _album_name_cache:
cached_name = _album_name_cache[album_key]
current_edition = _album_editions.get(album_key, "standard")
if is_deluxe_track and current_edition == "standard":
final_album_name = f"{base_album} (Deluxe Edition)"
_album_editions[album_key] = "deluxe"
_album_name_cache[album_key] = final_album_name
logger.info("Album cache upgrade: %r -> %r", album_key, final_album_name)
return final_album_name
logger.info("Using cached album name for %r: %r", album_key, cached_name)
return cached_name
logger.info("Album grouping - Key: %r, Detected: %r", album_key, detected_album)
current_edition = _album_editions.get(album_key, "standard")
if is_deluxe_track and current_edition == "standard":
logger.info("UPGRADE: Album %r upgraded from standard to deluxe!", base_album)
_album_editions[album_key] = "deluxe"
current_edition = "deluxe"
if current_edition == "deluxe":
final_album_name = f"{base_album} (Deluxe Edition)"
else:
final_album_name = base_album
_album_name_cache[album_key] = final_album_name
logger.info("Album resolution: %r -> %r (edition: %s)", detected_album, final_album_name, current_edition)
return final_album_name
except Exception as e:
logger.error("Error resolving album group: %s", e)
album_name = (album_info or {}).get("album_name", "Unknown Album")
return album_name

391
core/imports/context.py Normal file
View file

@ -0,0 +1,391 @@
"""Helpers for normalizing and reading import contexts.
These functions keep the single-import pipeline source-agnostic while still
accepting legacy `spotify_*` payloads from older callers.
"""
from __future__ import annotations
from typing import Any, Dict, Optional
def _as_dict(value: Any) -> Dict[str, Any]:
return value if isinstance(value, dict) else {}
def _first_value(mapping: Dict[str, Any], *keys: str, default: Any = "") -> Any:
for key in keys:
if key in mapping:
value = mapping.get(key)
if value not in (None, ""):
return value
return default
def _first_id_value(*values: Any) -> str:
for value in values:
if value in (None, ""):
continue
text = str(value).strip()
if text:
return text
return ""
def extract_artist_name(artist: Any) -> str:
if isinstance(artist, dict):
return str(artist.get("name", "") or "")
if hasattr(artist, "name"):
return str(getattr(artist, "name") or "")
return str(artist) if artist else ""
def normalize_import_context(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
"""Normalize an import context to neutral fields in place and drop legacy aliases."""
if not isinstance(context, dict):
return {}
source = context.get("source") or context.get("_source") or ""
artist = _as_dict(context.get("artist") or context.get("spotify_artist"))
album = _as_dict(context.get("album") or context.get("spotify_album"))
track_info = _as_dict(context.get("track_info"))
original_search = _as_dict(context.get("original_search_result"))
search_result = _as_dict(context.get("search_result"))
normalized_search = original_search or search_result
if source:
context["source"] = source
context["artist"] = artist
context["album"] = album
context["track_info"] = track_info
context["original_search_result"] = normalized_search
context.pop("_source", None)
context.pop("spotify_artist", None)
context.pop("spotify_album", None)
for clean_key, legacy_key in (
("clean_title", "spotify_clean_title"),
("clean_album", "spotify_clean_album"),
("clean_artist", "spotify_clean_artist"),
):
if clean_key not in normalized_search or normalized_search.get(clean_key) in (None, ""):
legacy_value = normalized_search.get(legacy_key)
if legacy_value not in (None, ""):
normalized_search[clean_key] = legacy_value
normalized_search.pop(legacy_key, None)
has_clean = bool(context.get("has_clean_metadata", context.get("has_clean_spotify_data", False)))
has_full = bool(context.get("has_full_metadata", context.get("has_full_spotify_metadata", False)))
context["has_clean_metadata"] = has_clean
context["has_full_metadata"] = has_full
context.pop("has_clean_spotify_data", None)
context.pop("has_full_spotify_metadata", None)
return context
def get_import_context_artist(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if not isinstance(context, dict):
return {}
return _as_dict(context.get("artist") or context.get("spotify_artist"))
def get_import_context_album(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if not isinstance(context, dict):
return {}
return _as_dict(context.get("album") or context.get("spotify_album"))
def get_import_track_info(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if not isinstance(context, dict):
return {}
return _as_dict(context.get("track_info"))
def get_import_original_search(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if not isinstance(context, dict):
return {}
return _as_dict(context.get("original_search_result"))
def get_import_search_result(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if not isinstance(context, dict):
return {}
return _as_dict(context.get("search_result"))
def get_import_source(context: Optional[Dict[str, Any]]) -> str:
if not isinstance(context, dict):
return ""
source = context.get("source")
if source:
return str(source)
track_info = get_import_track_info(context)
source = _first_value(track_info, "source", default="")
if source:
return str(source)
original_search = get_import_original_search(context)
source = _first_value(original_search, "source", default="")
if source:
return str(source)
album = get_import_context_album(context)
source = _first_value(album, "source", default="")
if source:
return str(source)
artist = get_import_context_artist(context)
source = _first_value(artist, "source", default="")
return str(source) if source else ""
def get_import_clean_title(
context: Optional[Dict[str, Any]],
album_info: Optional[Dict[str, Any]] = None,
default: str = "Unknown Track",
) -> str:
original_search = get_import_original_search(context)
title = _first_value(
original_search,
"clean_title",
"title",
default="",
)
if not title and album_info:
title = _first_value(album_info, "clean_track_name", "track_name", default="")
if not title:
track_info = get_import_track_info(context)
title = _first_value(track_info, "name", "title", default="")
return str(title or default)
def get_import_clean_album(
context: Optional[Dict[str, Any]],
album_info: Optional[Dict[str, Any]] = None,
default: str = "Unknown Album",
) -> str:
original_search = get_import_original_search(context)
album = _first_value(
original_search,
"clean_album",
"album",
default="",
)
if not album and album_info:
album = _first_value(album_info, "album_name", "clean_album_name", default="")
if not album:
album_ctx = get_import_context_album(context)
album = _first_value(album_ctx, "name", default="")
return str(album or default)
def get_import_clean_artist(context: Optional[Dict[str, Any]], default: str = "Unknown Artist") -> str:
original_search = get_import_original_search(context)
artist = _first_value(
original_search,
"clean_artist",
"artist",
default="",
)
if not artist:
artist_ctx = get_import_context_artist(context)
artist = _first_value(artist_ctx, "name", default="")
return str(artist or default)
def get_import_has_clean_metadata(context: Optional[Dict[str, Any]]) -> bool:
if not isinstance(context, dict):
return False
return bool(context.get("has_clean_metadata", False))
def get_import_has_full_metadata(context: Optional[Dict[str, Any]]) -> bool:
if not isinstance(context, dict):
return False
return bool(context.get("has_full_metadata", False))
def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]:
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
search_result = get_import_search_result(context)
artist = get_import_context_artist(context)
album = get_import_context_album(context)
return {
"track_id": _first_id_value(
_first_value(track_info, "id", "track_id", "trackId", "source_track_id", default=""),
_first_value(track_info, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""),
_first_value(original_search, "id", "track_id", "source_track_id", default=""),
_first_value(original_search, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""),
_first_value(search_result, "id", "track_id", "source_track_id", default=""),
_first_value(search_result, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""),
),
"artist_id": _first_id_value(
_first_value(artist, "id", "artist_id", "source_artist_id", default=""),
_first_value(artist, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""),
_first_value(original_search, "artist_id", "source_artist_id", default=""),
_first_value(original_search, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""),
_first_value(search_result, "artist_id", "source_artist_id", default=""),
_first_value(search_result, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""),
),
"album_id": _first_id_value(
_first_value(album, "id", "album_id", "collectionId", "source_album_id", default=""),
_first_value(album, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""),
_first_value(original_search, "album_id", "source_album_id", default=""),
_first_value(original_search, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""),
_first_value(track_info, "album_id", "source_album_id", default=""),
_first_value(track_info, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""),
_first_value(search_result, "album_id", "source_album_id", default=""),
_first_value(search_result, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""),
),
}
def get_source_tag_names(source: str) -> Dict[str, Optional[str]]:
source_name = (source or "").strip().lower()
if source_name == "spotify":
return {"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"}
if source_name == "itunes":
return {"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"}
if source_name == "deezer":
return {"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None}
if source_name == "hydrabase":
return {"track": None, "artist": None, "album": None}
if source_name == "discogs":
return {"track": None, "artist": None, "album": None}
return {"track": None, "artist": None, "album": None}
def get_library_source_id_columns(source: str) -> Dict[str, Optional[str]]:
source_name = (source or "").strip().lower()
if source_name == "spotify":
return {"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"}
if source_name == "itunes":
return {"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"}
if source_name == "deezer":
return {"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"}
if source_name == "hydrabase":
return {"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"}
if source_name == "discogs":
return {"artist": "discogs_id", "album": "discogs_id", "track": None}
return {}
def build_import_album_info(
context: Optional[Dict[str, Any]],
*,
album_info: Optional[Dict[str, Any]] = None,
force_album: bool = False,
) -> Dict[str, Any]:
"""Build the album-info payload used by post-processing."""
album_ctx = get_import_context_album(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
artist_ctx = get_import_context_artist(context)
track_number = (
(album_info or {}).get("track_number")
or track_info.get("track_number")
or original_search.get("track_number")
or 1
)
disc_number = (
(album_info or {}).get("disc_number")
or track_info.get("disc_number")
or original_search.get("disc_number")
or 1
)
clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track"))
album_name = get_import_clean_album(context, album_info=album_info, default=original_search.get("album", "Unknown Album"))
album_image_url = (
(album_info or {}).get("album_image_url")
or album_ctx.get("image_url")
or ""
)
total_tracks = (
album_ctx.get("total_tracks")
or track_info.get("total_tracks")
or (album_info or {}).get("total_tracks")
or 0
)
album_type = (album_ctx.get("album_type") or track_info.get("album_type") or "album")
source = get_import_source(context)
artist_name = artist_ctx.get("name") or original_search.get("artist") or get_import_clean_artist(context)
normalized_album = str(album_name or "").strip().lower()
normalized_title = str(clean_track_name or "").strip().lower()
normalized_artist = str(artist_name or "").strip().lower()
is_album = bool(
force_album
or (
normalized_album
and total_tracks
and int(total_tracks) > 1
and normalized_album != normalized_title
and normalized_album != normalized_artist
)
)
return {
"is_album": is_album,
"album_name": album_name,
"track_number": int(track_number) if str(track_number).isdigit() else track_number,
"disc_number": int(disc_number) if str(disc_number).isdigit() else disc_number,
"clean_track_name": clean_track_name,
"album_image_url": album_image_url,
"confidence": (album_info or {}).get("confidence", 1.0 if is_album or force_album else 0.0),
"source": source,
"album_type": album_type,
"total_tracks": int(total_tracks) if str(total_tracks).isdigit() else total_tracks,
}
def detect_album_info_web(context, artist_context=None):
"""Best-effort album detection for single-track downloads."""
context = normalize_import_context(context)
if artist_context is None:
artist_context = context.get("artist") or {}
album_info = build_import_album_info(context)
if album_info.get("is_album"):
return album_info
album_ctx = get_import_context_album(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
album_name = (
album_ctx.get("name")
or track_info.get("album")
or original_search.get("album")
or ""
)
track_name = (
track_info.get("name")
or original_search.get("title")
or ""
)
artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="")
if album_name and track_name and album_name.strip().lower() not in {
track_name.strip().lower(),
artist_name.strip().lower(),
}:
return build_import_album_info(
context,
album_info={
"album_name": album_name,
"track_number": track_info.get("track_number", 1),
"disc_number": track_info.get("disc_number", 1),
"album_image_url": album_ctx.get("image_url", ""),
"confidence": 0.5,
},
force_album=True,
)
return None

368
core/imports/file_ops.py Normal file
View file

@ -0,0 +1,368 @@
"""File operation helpers for the import flow."""
from __future__ import annotations
import logging
import os
import shutil
import subprocess
import time
from pathlib import Path
from config.settings import config_manager
logger = logging.getLogger("imports.file_ops")
def safe_move_file(src, dst):
"""Move a file safely across filesystems."""
src = Path(src)
dst = Path(dst)
dst.parent.mkdir(parents=True, exist_ok=True)
if not src.exists():
if dst.exists():
logger.info(f"Source gone but destination exists, file already transferred: {dst.name}")
return
raise FileNotFoundError(f"Source file not found and destination does not exist: {src}")
if dst.exists():
for _attempt in range(3):
try:
dst.unlink()
break
except PermissionError:
if _attempt < 2:
time.sleep(1)
else:
logger.warning(f"Could not remove locked destination after 3 attempts: {dst.name}")
except Exception:
break
try:
shutil.move(str(src), str(dst))
return
except FileNotFoundError:
if dst.exists():
logger.info(f"Source moved by another thread, destination exists: {dst.name}")
return
raise
except (OSError, PermissionError) as e:
error_msg = str(e).lower()
if dst.exists() and dst.stat().st_size > 0:
logger.warning(f"Move raised {type(e).__name__} but destination exists, treating as success: {e}")
try:
src.unlink()
except Exception:
logger.info(f"Could not delete source file (may be owned by another process): {src}")
return
if "cross-device" in error_msg or "operation not permitted" in error_msg or "permission denied" in error_msg:
logger.warning(f"Cross-device move detected, using fallback copy method: {e}")
try:
with open(src, "rb") as f_src:
with open(dst, "wb") as f_dst:
shutil.copyfileobj(f_src, f_dst)
f_dst.flush()
os.fsync(f_dst.fileno())
try:
src.unlink()
except PermissionError:
logger.info(f"Could not delete source file (may be owned by another process): {src}")
logger.info(f"Successfully moved file using fallback method: {src} -> {dst}")
return
except Exception as fallback_error:
logger.error(f"Fallback copy also failed: {fallback_error}")
raise
raise
def cleanup_empty_directories(download_path, moved_file_path):
"""Remove empty directories after a move, ignoring hidden files."""
try:
current_dir = os.path.dirname(moved_file_path)
while current_dir != download_path and current_dir.startswith(download_path):
is_empty = not any(not f.startswith(".") for f in os.listdir(current_dir))
if is_empty:
logger.warning(f"Removing empty directory: {current_dir}")
os.rmdir(current_dir)
current_dir = os.path.dirname(current_dir)
else:
break
except Exception as e:
logger.error(f"An error occurred during directory cleanup: {e}")
def get_audio_quality_string(file_path):
"""Return a compact audio quality string for the given file."""
try:
ext = os.path.splitext(file_path)[1].lower()
if ext == ".flac":
from mutagen.flac import FLAC
audio = FLAC(file_path)
return f"FLAC {audio.info.bits_per_sample}bit"
if ext == ".mp3":
from mutagen.mp3 import MP3, BitrateMode
audio = MP3(file_path)
bitrate_kbps = audio.info.bitrate // 1000
if audio.info.bitrate_mode == BitrateMode.VBR:
return "MP3-VBR"
return f"MP3-{bitrate_kbps}"
if ext in (".m4a", ".aac", ".mp4"):
from mutagen.mp4 import MP4
audio = MP4(file_path)
return f"M4A-{audio.info.bitrate // 1000}"
if ext == ".ogg":
from mutagen.oggvorbis import OggVorbis
audio = OggVorbis(file_path)
return f"OGG-{audio.info.bitrate // 1000}"
if ext == ".opus":
from mutagen.oggopus import OggOpus
audio = OggOpus(file_path)
return f"OPUS-{audio.info.bitrate // 1000}"
return ""
except Exception as e:
logger.debug(f"Could not determine audio quality for {file_path}: {e}")
return ""
def get_quality_tier_from_extension(file_path):
"""Classify a file extension into a quality tier."""
if not file_path:
return ("unknown", 999)
ext = os.path.splitext(file_path)[1].lower()
quality_tiers = {
"lossless": {
"extensions": [".flac", ".ape", ".wav", ".alac", ".dsf", ".dff", ".aiff", ".aif"],
"tier": 1,
},
"high_lossy": {
"extensions": [".opus", ".ogg"],
"tier": 2,
},
"standard_lossy": {
"extensions": [".m4a", ".aac"],
"tier": 3,
},
"low_lossy": {
"extensions": [".mp3", ".wma"],
"tier": 4,
},
}
for tier_name, tier_data in quality_tiers.items():
if ext in tier_data["extensions"]:
return (tier_name, tier_data["tier"])
return ("unknown", 999)
def downsample_hires_flac(final_path, context):
"""Downsample a hi-res FLAC to 16-bit/44.1kHz if enabled."""
from mutagen.flac import FLAC
if not config_manager.get("lossy_copy.downsample_hires", False):
return None
if os.path.splitext(final_path)[1].lower() != ".flac":
return None
try:
audio = FLAC(final_path)
original_bits = audio.info.bits_per_sample
original_rate = audio.info.sample_rate
except Exception as e:
logger.error(f"[Downsample] Could not read FLAC info: {e}")
return None
if original_bits <= 16 and original_rate <= 44100:
return None
logger.info(f"[Downsample] Converting {original_bits}-bit/{original_rate}Hz -> 16-bit/44100Hz: {os.path.basename(final_path)}")
ffmpeg_bin = shutil.which("ffmpeg")
if not ffmpeg_bin:
local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg")
if os.path.isfile(local):
ffmpeg_bin = local
else:
logger.warning("[Downsample] ffmpeg not found - skipping hi-res conversion")
return None
temp_path = final_path + ".tmp.flac"
try:
result = subprocess.run(
[
ffmpeg_bin, "-i", final_path,
"-sample_fmt", "s16",
"-ar", "44100",
"-map_metadata", "0",
"-compression_level", "8",
"-y", temp_path,
],
capture_output=True,
text=True,
timeout=300,
)
if result.returncode != 0:
logger.error(f"[Downsample] ffmpeg failed: {result.stderr[:200]}")
if os.path.exists(temp_path):
os.remove(temp_path)
return None
if not os.path.isfile(temp_path) or os.path.getsize(temp_path) == 0:
logger.warning("[Downsample] Output file missing or empty")
if os.path.exists(temp_path):
os.remove(temp_path)
return None
verify_audio = FLAC(temp_path)
if verify_audio.info.bits_per_sample != 16:
logger.info(f"[Downsample] Output not 16-bit ({verify_audio.info.bits_per_sample}-bit), aborting")
os.remove(temp_path)
return None
os.replace(temp_path, final_path)
logger.info(f"[Downsample] Converted to 16-bit/44.1kHz: {os.path.basename(final_path)}")
new_quality = "FLAC 16bit"
try:
updated_audio = FLAC(final_path)
updated_audio["QUALITY"] = new_quality
updated_audio.save()
except Exception as tag_err:
logger.error(f"[Downsample] Could not update QUALITY tag: {tag_err}")
old_quality = context.get("_audio_quality", "")
context["_audio_quality"] = new_quality
if old_quality and old_quality != new_quality and old_quality in os.path.basename(final_path):
new_basename = os.path.basename(final_path).replace(old_quality, new_quality)
new_path = os.path.join(os.path.dirname(final_path), new_basename)
try:
os.rename(final_path, new_path)
logger.info(f"[Downsample] Renamed: {os.path.basename(final_path)} -> {new_basename}")
for lyrics_ext in (".lrc", ".txt"):
old_lyrics = os.path.splitext(final_path)[0] + lyrics_ext
if os.path.isfile(old_lyrics):
new_lyrics = os.path.splitext(new_path)[0] + lyrics_ext
os.rename(old_lyrics, new_lyrics)
return new_path
except Exception as rename_err:
logger.error(f"[Downsample] Could not rename file: {rename_err}")
return final_path
except subprocess.TimeoutExpired:
logger.info(f"[Downsample] Conversion timed out for: {os.path.basename(final_path)}")
if os.path.exists(temp_path):
os.remove(temp_path)
except Exception as e:
logger.error(f"[Downsample] Conversion error: {e}")
if os.path.exists(temp_path):
try:
os.remove(temp_path)
except Exception:
pass
return None
def create_lossy_copy(final_path):
"""Convert a FLAC file to a lossy copy using the configured codec."""
from mutagen.flac import FLAC
if not config_manager.get("lossy_copy.enabled", False):
return None
if os.path.splitext(final_path)[1].lower() != ".flac":
return None
codec = config_manager.get("lossy_copy.codec", "mp3").lower()
bitrate = config_manager.get("lossy_copy.bitrate", "320")
if codec == "opus" and int(bitrate) > 256:
bitrate = "256"
codec_map = {
"mp3": ("libmp3lame", ".mp3", f"MP3-{bitrate}", ["-vn", "-id3v2_version", "3"]),
"opus": ("libopus", ".opus", f"OPUS-{bitrate}", ["-vn", "-map", "0:a", "-vbr", "on"]),
"aac": ("aac", ".m4a", f"AAC-{bitrate}", ["-vn", "-movflags", "+faststart"]),
}
if codec not in codec_map:
logger.info(f"[Lossy Copy] Unknown codec '{codec}' - skipping conversion")
return None
ffmpeg_codec, out_ext, quality_label, extra_args = codec_map[codec]
out_path = os.path.splitext(final_path)[0] + out_ext
original_quality = get_audio_quality_string(final_path)
if original_quality:
out_basename = os.path.basename(out_path)
if original_quality in out_basename:
out_basename = out_basename.replace(original_quality, quality_label)
out_path = os.path.join(os.path.dirname(out_path), out_basename)
ffmpeg_bin = shutil.which("ffmpeg")
if not ffmpeg_bin:
local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg")
if os.path.isfile(local):
ffmpeg_bin = local
else:
logger.warning(f"[Lossy Copy] ffmpeg not found - skipping {codec.upper()} conversion")
return None
try:
logger.info(f"[Lossy Copy] Converting to {quality_label}: {os.path.basename(final_path)}")
cmd = [
ffmpeg_bin, "-i", final_path,
"-codec:a", ffmpeg_codec,
"-b:a", f"{bitrate}k",
"-map_metadata", "0",
] + extra_args + ["-y", out_path]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
if result.returncode == 0:
logger.info(f"[Lossy Copy] Created {quality_label} copy: {os.path.basename(out_path)}")
try:
from mutagen import File as MutagenFile
audio = MutagenFile(out_path)
if audio is not None:
if codec == "mp3":
from mutagen.id3 import TXXX
audio.tags.add(TXXX(encoding=3, desc="QUALITY", text=[quality_label]))
elif codec == "opus":
audio["QUALITY"] = [quality_label]
elif codec == "aac":
from mutagen.mp4 import MP4FreeForm
audio["----:com.apple.iTunes:QUALITY"] = [MP4FreeForm(quality_label.encode("utf-8"))]
audio.save()
except Exception as tag_err:
logger.error(f"[Lossy Copy] Could not update QUALITY tag: {tag_err}")
return out_path
logger.error(f"[Lossy Copy] ffmpeg failed: {result.stderr[:200]}")
if os.path.exists(out_path):
try:
os.remove(out_path)
except Exception:
pass
return None
except subprocess.TimeoutExpired:
logger.warning(f"[Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}")
except Exception as e:
logger.error(f"[Lossy Copy] Conversion error: {e}")
return None

92
core/imports/filename.py Normal file
View file

@ -0,0 +1,92 @@
"""Filename parsing helpers used by import flows."""
from __future__ import annotations
import os
import re
from typing import Any, Dict
_TRACK_PATTERNS = (
r"^(\d+)\s*[-\.]\s*(.+?)\s*[-]\s*(.+)$",
r"^(.+?)\s*[-]\s*(.+)$",
r"^(\d+)\s*[-\.]\s*(.+)$",
)
def extract_track_number_from_filename(filename: str, title: str = None) -> int:
"""Extract track number from a filename. Returns 1 if not found."""
basename = os.path.splitext(os.path.basename(filename))[0].strip()
match = re.match(r"^\d[\-\.](\d{1,2})\s*[\-\.]\s*", basename)
if match:
num = int(match.group(1))
if 1 <= num <= 99:
return num
match = re.match(r"^\(?(\d{1,3})\)?\s*[\-\.)\]]\s*", basename)
if match:
num = int(match.group(1))
if 1 <= num <= 999:
return num
return 1
def parse_filename_metadata(filename: str) -> Dict[str, Any]:
"""Extract artist/title/album hints from a loose filename."""
raw_path = str(filename or "")
normalized_path = raw_path.replace("\\", "/")
base_name = os.path.splitext(os.path.basename(normalized_path))[0]
result: Dict[str, Any] = {
"artist": "",
"title": "",
"album": "",
"track_number": None,
}
if not base_name:
return result
for pattern in _TRACK_PATTERNS:
match = re.match(pattern, base_name)
if not match:
continue
groups = match.groups()
if len(groups) == 3:
try:
result["track_number"] = int(groups[0])
result["artist"] = result["artist"] or groups[1].strip()
result["title"] = result["title"] or groups[2].strip()
except ValueError:
result["artist"] = result["artist"] or groups[0].strip()
result["title"] = result["title"] or f"{groups[1]} - {groups[2]}".strip()
elif len(groups) == 2:
if groups[0].isdigit():
try:
result["track_number"] = int(groups[0])
result["title"] = result["title"] or groups[1].strip()
except ValueError:
pass
else:
result["artist"] = result["artist"] or groups[0].strip()
result["title"] = result["title"] or groups[1].strip()
break
if not result["title"]:
result["title"] = base_name
if not result["album"] and "/" in normalized_path:
path_parts = normalized_path.split("/")
for part in reversed(path_parts[:-1]):
if not part or part.startswith("@"):
continue
cleaned = re.sub(r"^\d+\s*[-\.]\s*", "", part).strip()
if len(cleaned) > 3:
result["album"] = cleaned
break
return result

118
core/imports/guards.py Normal file
View file

@ -0,0 +1,118 @@
"""Import post-processing guards and quarantine helpers."""
from __future__ import annotations
import json
import os
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Optional
from config.settings import config_manager
from core.imports.context import (
get_import_clean_artist,
get_import_clean_title,
get_import_context_artist,
get_import_original_search,
get_import_track_info,
normalize_import_context,
)
from core.imports.file_ops import safe_move_file
from database.music_database import MusicDatabase
from utils.logging_config import get_logger
logger = get_logger("imports.guards")
def _get_config_manager():
return config_manager
def move_to_quarantine(file_path: str, context: dict, reason: str, automation_engine=None) -> str:
"""Move a file to the quarantine folder and write a metadata sidecar."""
download_dir = _get_config_manager().get("soulseek.download_path", "./downloads")
quarantine_dir = Path(download_dir) / "ss_quarantine"
quarantine_dir.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
original_name = Path(file_path).stem
file_ext = Path(file_path).suffix
quarantine_filename = f"{timestamp}_{original_name}{file_ext}.quarantined"
quarantine_path = quarantine_dir / quarantine_filename
safe_move_file(file_path, str(quarantine_path))
metadata_path = quarantine_dir / f"{timestamp}_{original_name}.json"
context = normalize_import_context(context)
original_search = get_import_original_search(context)
artist_context = get_import_context_artist(context)
metadata = {
"original_filename": Path(file_path).name,
"quarantine_reason": reason,
"timestamp": datetime.now().isoformat(),
"expected_track": get_import_clean_title(context, default=original_search.get("title", "Unknown")),
"expected_artist": get_import_clean_artist(context, default=(artist_context.get("name", "") if isinstance(artist_context, dict) else "Unknown")),
"context_key": context.get("context_key", "unknown"),
}
try:
with open(metadata_path, "w", encoding="utf-8") as f:
json.dump(metadata, f, indent=2, ensure_ascii=False)
except Exception as exc:
logger.warning("Failed to write quarantine metadata: %s", exc)
logger.warning("File quarantined: %s - Reason: %s", quarantine_path, reason)
if automation_engine:
try:
ti = context.get("track_info", {})
artists = ti.get("artists", [])
artist_name = ""
if artists:
first = artists[0]
artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first)
automation_engine.emit(
"download_quarantined",
{
"artist": artist_name,
"title": ti.get("name", ""),
"reason": reason or "Unknown",
},
)
except Exception:
pass
return str(quarantine_path)
def check_flac_bit_depth(file_path: str, context: dict) -> Optional[str]:
"""Return a rejection message if a FLAC file violates the configured bit depth."""
if not context.get("_audio_quality", "").startswith("FLAC"):
return None
quality_profile = MusicDatabase().get_quality_profile()
flac_config = quality_profile.get("qualities", {}).get("flac", {})
flac_pref = flac_config.get("bit_depth", "any")
if flac_pref == "any":
return None
actual_bits = context["_audio_quality"].replace("FLAC ", "").replace("bit", "")
if actual_bits == flac_pref:
return None
flac_fallback = flac_config.get("bit_depth_fallback", True)
downsample_enabled = _get_config_manager().get("lossy_copy.downsample_hires", False)
track_info = context.get("track_info", {})
track_name = track_info.get("name", os.path.basename(file_path))
if flac_fallback or downsample_enabled:
if downsample_enabled:
logger.info("[FLAC Downsample] Accepted %s-bit FLAC (will be downsampled to %s-bit): %s", actual_bits, flac_pref, track_name)
else:
logger.warning("[FLAC Fallback] Accepted %s-bit FLAC (preferred %s-bit): %s", actual_bits, flac_pref, track_name)
return None
return f"FLAC bit depth mismatch: file is {actual_bits}-bit, preference is {flac_pref}-bit"

593
core/imports/paths.py Normal file
View file

@ -0,0 +1,593 @@
"""Shared path and naming helpers for import processing."""
from __future__ import annotations
import json
import logging
import os
import re
from pathlib import Path
from typing import Any
# Album grouping lives in core.imports.album_naming; this module keeps the
# imported helper because the path builder still needs it.
from core.imports.album_naming import resolve_album_group
from core.imports.context import (
extract_artist_name,
get_import_clean_title,
get_import_context_album,
get_import_original_search,
get_import_source,
get_import_track_info,
normalize_import_context,
)
logger = logging.getLogger("imports.paths")
def _get_config_manager():
try:
from config.settings import config_manager
return config_manager
except Exception:
class _FallbackConfig:
@staticmethod
def get(key, default=None):
return default
return _FallbackConfig()
def _get_itunes_client():
try:
from core.metadata_service import get_itunes_client
return get_itunes_client()
except Exception:
return None
def _get_album_tracks_for_source(source: str, album_id: str):
try:
from core.metadata_service import get_album_tracks_for_source
return get_album_tracks_for_source(source, album_id)
except Exception:
return None
def docker_resolve_path(path_str: str) -> str:
"""Resolve Docker-hosted Windows paths into container paths."""
if os.path.exists("/.dockerenv") and len(path_str) >= 3 and path_str[1] == ":" and path_str[0].isalpha():
drive_letter = path_str[0].lower()
rest_of_path = path_str[2:].replace("\\", "/")
return f"/host/mnt/{drive_letter}{rest_of_path}"
return path_str
def build_simple_download_destination(context, file_path: str):
"""Build the destination path for a simple download into Transfer."""
context = normalize_import_context(context)
search_result = context.get("search_result", {}) or {}
if not isinstance(search_result, dict):
search_result = {}
transfer_dir = Path(docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer")))
album_name = None
original_filename = search_result.get("filename", "")
if "/" in original_filename or "\\" in original_filename:
path_parts = original_filename.replace("\\", "/").split("/")
if len(path_parts) >= 2:
album_name = path_parts[-2]
if not album_name:
album_value = search_result.get("album")
if isinstance(album_value, dict):
album_name = album_value.get("name", "")
else:
album_name = album_value
filename = Path(file_path).name
if album_name and str(album_name).lower() not in {"unknown", "unknown album", ""}:
album_name = sanitize_filename(str(album_name))
destination_dir = transfer_dir / album_name
else:
album_name = ""
destination_dir = transfer_dir
destination_dir.mkdir(parents=True, exist_ok=True)
return destination_dir / filename, album_name, filename
def sanitize_filename(filename: str) -> str:
"""Sanitize filename for file system compatibility."""
sanitized = re.sub(r'[<>:"/\\|?*]', "_", filename)
sanitized = re.sub(r"\s+", " ", sanitized).strip()
sanitized = sanitized.rstrip(". ") or "_"
if re.match(r"^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)", sanitized, re.IGNORECASE):
sanitized = "_" + sanitized
return sanitized[:200]
def sanitize_context_values(context: dict) -> dict:
"""Sanitize all string values in a template context for path safety."""
sanitized = {}
for key, value in context.items():
if isinstance(value, str) and value:
sanitized[key] = sanitize_filename(value)
else:
sanitized[key] = value
return sanitized
def clean_track_title(track_title: str, artist_name: str) -> str:
"""Clean up track title by removing artist prefix and other noise."""
original = (track_title or "").strip()
cleaned = original
cleaned = re.sub(r"^\d{1,2}[\.\s\-]+", "", cleaned)
artist_pattern = re.escape(artist_name or "") + r"\s*-\s*"
cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"^[A-Za-z0-9\.]+\s*-\s*\d{1,2}\s*-\s*", "", cleaned)
quality_patterns = [
r"\s*[\[\(][0-9]+\s*kbps[\]\)]\s*",
r"\s*[\[\(]flac[\]\)]\s*",
r"\s*[\[\(]mp3[\]\)]\s*",
]
for pattern in quality_patterns:
cleaned = re.sub(pattern, "", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"^[-\s\.]+", "", cleaned)
cleaned = re.sub(r"[-\s\.]+$", "", cleaned)
cleaned = re.sub(r"\s+", " ", cleaned).strip()
return cleaned if cleaned else original
def get_album_type_display(raw_type, track_count) -> str:
"""Return the display form of an album's type for the $albumtype template variable."""
raw = (raw_type or "").strip().lower()
try:
tc = int(track_count or 0)
except (TypeError, ValueError):
tc = 0
if raw in ("compilation", "compile"):
return "Compilation"
if raw == "album":
return "Album"
if raw in ("single", "ep"):
if tc <= 3:
return "Single"
if tc <= 6:
return "EP"
return "Album"
if tc <= 0:
return "Album"
if tc <= 3:
return "Single"
if tc <= 6:
return "EP"
return "Album"
def _replace_template_variables(template: str, context: dict) -> str:
clean_context = sanitize_context_values(context)
result = template
album_artist_value = clean_context.get("albumartist", clean_context.get("artist", "Unknown Artist"))
collab_mode = _get_config_manager().get("file_organization.collab_artist_mode", "first")
if collab_mode == "first" and album_artist_value:
artists_list = context.get("_artists_list")
if artists_list and len(artists_list) > 1:
first = artists_list[0]
album_artist_value = first.get("name", first) if isinstance(first, dict) else str(first)
elif artists_list and len(artists_list) == 1:
itunes_artist_id = context.get("_itunes_artist_id")
if itunes_artist_id and ("," in album_artist_value or " & " in album_artist_value):
try:
resolved_client = _get_itunes_client()
if resolved_client and hasattr(resolved_client, "resolve_primary_artist"):
resolved = resolved_client.resolve_primary_artist(itunes_artist_id)
if resolved and resolved != album_artist_value:
album_artist_value = resolved
except Exception:
pass
bracket_map = {
"albumartist": album_artist_value,
"albumtype": clean_context.get("albumtype", "Album"),
"playlist": clean_context.get("playlist_name", ""),
"artistletter": (clean_context.get("artist", "U") or "U")[0].upper(),
"artist": clean_context.get("artist", "Unknown Artist"),
"album": clean_context.get("album", "Unknown Album"),
"title": clean_context.get("title", "Unknown Track"),
"track": f"{_coerce_int(clean_context.get('track_number', 1), 1):02d}",
"disc": str(_coerce_int(clean_context.get("disc_number", 1), 1)),
"discnum": str(_coerce_int(clean_context.get("disc_number", 1), 1)),
"year": str(clean_context.get("year", "")),
"quality": clean_context.get("quality", ""),
}
for var_name, val in bracket_map.items():
result = result.replace("${" + var_name + "}", val)
result = result.replace("$albumartist", album_artist_value)
result = result.replace("$albumtype", clean_context.get("albumtype", "Album"))
result = result.replace("$playlist", clean_context.get("playlist_name", ""))
result = result.replace("$artistletter", (clean_context.get("artist", "U") or "U")[0].upper())
result = result.replace("$artist", clean_context.get("artist", "Unknown Artist"))
result = result.replace("$album", clean_context.get("album", "Unknown Album"))
result = result.replace("$title", clean_context.get("title", "Unknown Track"))
result = result.replace("$track", f"{clean_context.get('track_number', 1):02d}")
result = result.replace("$year", str(clean_context.get("year", "")))
result = re.sub(r"\s+", " ", result)
result = re.sub(r"\s*-\s*-\s*", " - ", result)
result = result.strip()
return result
def apply_path_template(template: str, context: dict) -> str:
"""Apply a template to build a path string."""
return _replace_template_variables(template, context)
def get_file_path_from_template_raw(template: str, context: dict) -> tuple[str, str]:
"""Build file path using a user-provided template string directly."""
full_path = apply_path_template(template, context)
quality_value = context.get("quality", "")
disc_number = _coerce_int(context.get("disc_number", 1), 1)
disc_value = f"{disc_number:02d}"
disc_value_raw = str(disc_number)
path_parts = full_path.split("/")
if len(path_parts) > 1:
folder_parts = path_parts[:-1]
filename_base = path_parts[-1]
cleaned_folders = []
for part in folder_parts:
part = part.replace("$quality", "")
part = part.replace("$discnum", "")
part = part.replace("$disc", "")
part = re.sub(r"\s*\[\s*\]", "", part)
part = re.sub(r"\s*\(\s*\)", "", part)
part = re.sub(r"\s*\{\s*\}", "", part)
part = re.sub(r"\s*-\s*$", "", part)
part = re.sub(r"^\s*-\s*", "", part)
part = re.sub(r"\s+", " ", part).strip()
if part:
cleaned_folders.append(part)
filename_base = filename_base.replace("$quality", quality_value)
filename_base = filename_base.replace("$discnum", disc_value_raw)
filename_base = filename_base.replace("$disc", disc_value)
filename_base = re.sub(r"\s*\[\s*\]", "", filename_base)
filename_base = re.sub(r"\s*\(\s*\)", "", filename_base)
filename_base = re.sub(r"\s*\{\s*\}", "", filename_base)
filename_base = re.sub(r"\s*-\s*$", "", filename_base)
filename_base = re.sub(r"\s+", " ", filename_base).strip()
sanitized_folders = [sanitize_filename(part) for part in cleaned_folders]
folder_path = os.path.join(*sanitized_folders) if sanitized_folders else ""
return folder_path, sanitize_filename(filename_base)
full_path = full_path.replace("$quality", quality_value)
full_path = full_path.replace("$discnum", disc_value_raw)
full_path = full_path.replace("$disc", disc_value)
full_path = re.sub(r"\s*\[\s*\]", "", full_path)
full_path = re.sub(r"\s*\(\s*\)", "", full_path)
full_path = re.sub(r"\s*\{\s*\}", "", full_path)
full_path = re.sub(r"\s*-\s*$", "", full_path)
full_path = re.sub(r"\s+", " ", full_path).strip()
return "", sanitize_filename(full_path)
def get_file_path_from_template(context: dict, template_type: str = "album_path") -> tuple[str, str]:
"""Build complete file path using configured templates."""
if not _get_config_manager().get("file_organization.enabled", True):
return None, None
templates = _get_config_manager().get("file_organization.templates", {})
template = templates.get(template_type)
if not template:
default_templates = {
"album_path": "$albumartist/$albumartist - $album/$track - $title",
"single_path": "$artist/$artist - $title/$title",
"compilation_path": "Compilations/$album/$track - $artist - $title",
"playlist_path": "$playlist/$artist - $title",
}
template = default_templates.get(template_type, "$artist/$album/$track - $title")
full_path = apply_path_template(template, context)
path_parts = full_path.split("/")
quality_value = context.get("quality", "")
disc_number = _coerce_int(context.get("disc_number", 1), 1)
disc_value = f"{disc_number:02d}"
disc_value_raw = str(disc_number)
if len(path_parts) > 1:
folder_parts = path_parts[:-1]
filename_base = path_parts[-1]
cleaned_folders = []
for part in folder_parts:
part = part.replace("$quality", "")
part = part.replace("$discnum", "")
part = part.replace("$disc", "")
part = re.sub(r"\s*\[\s*\]", "", part)
part = re.sub(r"\s*\(\s*\)", "", part)
part = re.sub(r"\s*\{\s*\}", "", part)
part = re.sub(r"\s*-\s*$", "", part)
part = re.sub(r"^\s*-\s*", "", part)
part = re.sub(r"\s+", " ", part).strip()
if part:
cleaned_folders.append(part)
filename_base = filename_base.replace("$quality", quality_value)
filename_base = filename_base.replace("$discnum", disc_value_raw)
filename_base = filename_base.replace("$disc", disc_value)
filename_base = re.sub(r"\s*\[\s*\]", "", filename_base)
filename_base = re.sub(r"\s*\(\s*\)", "", filename_base)
filename_base = re.sub(r"\s*\{\s*\}", "", filename_base)
filename_base = re.sub(r"\s*-\s*$", "", filename_base)
filename_base = re.sub(r"\s+", " ", filename_base).strip()
sanitized_folders = [sanitize_filename(part) for part in cleaned_folders]
folder_path = os.path.join(*sanitized_folders) if sanitized_folders else ""
filename = sanitize_filename(filename_base)
return folder_path, filename
full_path = full_path.replace("$quality", quality_value)
full_path = full_path.replace("$discnum", disc_value_raw)
full_path = full_path.replace("$disc", disc_value)
full_path = re.sub(r"\s*\[\s*\]", "", full_path)
full_path = re.sub(r"\s*\(\s*\)", "", full_path)
full_path = re.sub(r"\s*\{\s*\}", "", full_path)
full_path = re.sub(r"\s*-\s*$", "", full_path)
full_path = re.sub(r"\s+", " ", full_path).strip()
return "", sanitize_filename(full_path)
def _max_disc_number(album_tracks: Any) -> int:
items = []
if isinstance(album_tracks, dict):
items = album_tracks.get("items") or album_tracks.get("tracks") or []
elif isinstance(album_tracks, list):
items = album_tracks
max_disc = 1
for track in items:
if not isinstance(track, dict):
continue
try:
disc_number = int(track.get("disc_number", 1) or 1)
except (TypeError, ValueError):
disc_number = 1
if disc_number > max_disc:
max_disc = disc_number
return max_disc
def _coerce_int(value: Any, default: int = 1) -> int:
try:
coerced = int(value)
except (TypeError, ValueError):
return default
return coerced if coerced > 0 else default
def build_final_path_for_track(context, artist_context, album_info, file_ext):
"""Shared path builder used by both post-processing and verification."""
transfer_dir = docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer"))
context = normalize_import_context(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
album_context = get_import_context_album(context)
source = get_import_source(context)
playlist_folder_mode = track_info.get("_playlist_folder_mode", False)
artist_name = extract_artist_name(artist_context)
source_info = track_info.get("source_info") or {}
if isinstance(source_info, str):
try:
source_info = json.loads(source_info)
except (json.JSONDecodeError, TypeError):
source_info = {}
if source_info.get("enhance") and source_info.get("original_file_path"):
original_path = source_info["original_file_path"]
original_dir = os.path.dirname(original_path)
original_stem = os.path.splitext(os.path.basename(original_path))[0]
final_path = os.path.join(original_dir, original_stem + file_ext)
os.makedirs(original_dir, exist_ok=True)
logger.info("[Enhance] Using original file location: %s", final_path)
return final_path, True
year = ""
if album_context and album_context.get("release_date"):
release_date = album_context["release_date"]
if release_date and len(release_date) >= 4:
year = release_date[:4]
raw_album_type = ""
if album_context:
raw_album_type = album_context.get("album_type", "") or ""
total_tracks = (album_context.get("total_tracks", 0) or 0) if album_context else 0
album_type_display = get_album_type_display(raw_album_type, total_tracks)
if playlist_folder_mode:
playlist_name = track_info.get("_playlist_name", "Unknown Playlist")
track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track"))
_artists = original_search.get("artists") or track_info.get("artists") or []
template_context = {
"artist": artist_name,
"albumartist": artist_name,
"album": track_name,
"title": track_name,
"playlist_name": playlist_name,
"track_number": 1,
"disc_number": 1,
"year": year,
"quality": context.get("_audio_quality", ""),
"albumtype": album_type_display,
"_artists_list": _artists,
"_itunes_artist_id": str(artist_context.get("id", "")) if isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source == "itunes" else None,
}
folder_path, filename_base = get_file_path_from_template(template_context, "playlist_path")
if folder_path and filename_base:
final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
return final_path, True
playlist_name_sanitized = sanitize_filename(playlist_name)
playlist_dir = os.path.join(transfer_dir, playlist_name_sanitized)
os.makedirs(playlist_dir, exist_ok=True)
artist_name_sanitized = sanitize_filename(template_context["artist"])
track_name_sanitized = sanitize_filename(track_name)
new_filename = f"{artist_name_sanitized} - {track_name_sanitized}{file_ext}"
return os.path.join(playlist_dir, new_filename), True
if album_info and album_info.get("is_album"):
clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track"))
track_number = _coerce_int(album_info.get("track_number", 1), 1)
disc_number = _coerce_int(album_info.get("disc_number", 1), 1)
_artists = original_search.get("artists") or track_info.get("artists") or []
_album_ctx = album_context
_itunes_aid = None
_is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer")
if _is_itunes and isinstance(artist_context, dict):
_aid = artist_context.get("id", "")
if str(_aid).isdigit():
_itunes_aid = str(_aid)
if not _itunes_aid and _album_ctx:
_ext = _album_ctx.get("external_urls", {})
if isinstance(_ext, dict) and _ext.get("itunes_artist_id"):
_itunes_aid = _ext["itunes_artist_id"]
_artist_name = artist_name
_album_artist_name = _artist_name
_album_artists_for_collab = None
_explicit_artist_ctx = track_info.get("_explicit_artist_context") if isinstance(track_info, dict) else None
if isinstance(_explicit_artist_ctx, dict) and _explicit_artist_ctx.get("name"):
_album_artist_name = _explicit_artist_ctx["name"]
_album_artists_for_collab = [_explicit_artist_ctx]
elif isinstance(_explicit_artist_ctx, str) and _explicit_artist_ctx:
_album_artist_name = _explicit_artist_ctx
_album_artists_for_collab = [{"name": _explicit_artist_ctx}]
else:
_sa_artists = _album_ctx.get("artists", []) if _album_ctx else []
if _sa_artists:
_first_sa = _sa_artists[0]
if isinstance(_first_sa, dict) and _first_sa.get("name"):
_album_artist_name = _first_sa["name"]
elif isinstance(_first_sa, str) and _first_sa:
_album_artist_name = _first_sa
_album_artists_for_collab = _sa_artists
template_context = {
"artist": _artist_name,
"albumartist": _album_artist_name,
"album": album_info["album_name"],
"title": clean_track_name,
"track_number": track_number,
"disc_number": disc_number,
"year": year,
"quality": context.get("_audio_quality", ""),
"albumtype": album_type_display,
"_artists_list": _album_artists_for_collab if _album_artists_for_collab else _artists,
"_itunes_artist_id": _itunes_aid,
}
total_discs = _coerce_int(album_context.get("total_discs", 1) if album_context else 1, 1)
if total_discs <= 1 and album_context and album_context.get("id"):
if disc_number > 1:
total_discs = disc_number
else:
try:
_album_tracks = _get_album_tracks_for_source(source, str(album_context["id"]))
if _album_tracks:
total_discs = _max_disc_number(_album_tracks)
if total_discs > 1:
album_context["total_discs"] = total_discs
logger.info(
"[Multi-Disc] Resolved %s discs for single-track download of %r",
total_discs,
album_context.get("name"),
)
except Exception as _disc_err:
logger.warning("[Multi-Disc] Could not resolve total_discs: %s", _disc_err)
album_template = _get_config_manager().get("file_organization.templates.album_path", "")
user_controls_disc = "$disc" in album_template
disc_label = _get_config_manager().get("file_organization.disc_label", "Disc")
folder_path, filename_base = get_file_path_from_template(template_context, "album_path")
if folder_path and filename_base:
if total_discs > 1 and not user_controls_disc:
disc_folder = f"{disc_label} {disc_number}"
final_path = os.path.join(transfer_dir, folder_path, disc_folder, filename_base + file_ext)
os.makedirs(os.path.join(transfer_dir, folder_path, disc_folder), exist_ok=True)
else:
final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
return final_path, True
artist_name_sanitized = sanitize_filename(template_context["albumartist"])
album_name_sanitized = sanitize_filename(album_info["album_name"])
artist_dir = os.path.join(transfer_dir, artist_name_sanitized)
album_folder_name = f"{artist_name_sanitized} - {album_name_sanitized}"
album_dir = os.path.join(artist_dir, album_folder_name)
if total_discs > 1:
album_dir = os.path.join(album_dir, f"{disc_label} {disc_number}")
os.makedirs(album_dir, exist_ok=True)
final_track_name_sanitized = sanitize_filename(clean_track_name)
new_filename = f"{track_number:02d} - {final_track_name_sanitized}{file_ext}"
return os.path.join(album_dir, new_filename), True
clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track"))
_artists = original_search.get("artists") or track_info.get("artists") or []
_album_ctx = album_context
_itunes_aid = None
_is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer")
if _is_itunes and isinstance(artist_context, dict):
_aid = artist_context.get("id", "")
if str(_aid).isdigit():
_itunes_aid = str(_aid)
if not _itunes_aid and _album_ctx:
_ext = _album_ctx.get("external_urls", {})
if isinstance(_ext, dict) and _ext.get("itunes_artist_id"):
_itunes_aid = _ext["itunes_artist_id"]
template_context = {
"artist": artist_name,
"albumartist": artist_name,
"album": album_info.get("album_name", clean_track_name) if album_info else clean_track_name,
"title": clean_track_name,
"track_number": 1,
"disc_number": 1,
"year": year,
"quality": context.get("_audio_quality", ""),
"albumtype": album_type_display,
"_artists_list": _artists,
"_itunes_artist_id": _itunes_aid,
}
folder_path, filename_base = get_file_path_from_template(template_context, "single_path")
if filename_base:
if folder_path:
final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
else:
final_path = os.path.join(transfer_dir, filename_base + file_ext)
os.makedirs(transfer_dir, exist_ok=True)
return final_path, True
artist_name_sanitized = sanitize_filename(template_context["artist"])
final_track_name_sanitized = sanitize_filename(clean_track_name)
artist_dir = os.path.join(transfer_dir, artist_name_sanitized)
single_folder_name = f"{artist_name_sanitized} - {final_track_name_sanitized}"
single_dir = os.path.join(artist_dir, single_folder_name)
os.makedirs(single_dir, exist_ok=True)
new_filename = f"{final_track_name_sanitized}{file_ext}"
return os.path.join(single_dir, new_filename), True

952
core/imports/pipeline.py Normal file
View file

@ -0,0 +1,952 @@
"""Import/post-processing pipeline for downloads and imported files."""
from __future__ import annotations
import json
import os
import threading
import time
from types import SimpleNamespace
from typing import Any
from config.settings import config_manager
from core.imports.file_ops import (
cleanup_empty_directories,
create_lossy_copy,
downsample_hires_flac,
get_audio_quality_string,
get_quality_tier_from_extension,
safe_move_file,
)
from core.imports.context import (
build_import_album_info,
detect_album_info_web,
extract_artist_name,
get_import_clean_artist,
get_import_clean_title,
get_import_context_artist,
get_import_has_clean_metadata,
get_import_original_search,
get_import_source,
get_import_track_info,
normalize_import_context,
)
from core.imports.filename import extract_track_number_from_filename
from core.imports.guards import check_flac_bit_depth, move_to_quarantine
from core.imports.side_effects import (
check_and_remove_from_wishlist,
emit_track_downloaded,
record_download_provenance,
record_library_history_download,
record_retag_download,
record_soulsync_library_entry,
)
from core.runtime_state import (
add_activity_item,
download_batches,
download_tasks,
matched_context_lock,
matched_downloads_context,
mark_task_completed as _mark_task_completed,
post_process_locks,
post_process_locks_lock,
processed_download_ids,
tasks_lock,
)
from core.metadata.artwork import download_cover_art
from core.metadata.common import wipe_source_tags
from core.metadata.enrichment import enhance_file_metadata
from core.imports.paths import (
build_final_path_for_track,
build_simple_download_destination,
docker_resolve_path,
)
from core.imports.album_naming import resolve_album_group
from core.metadata.lyrics import generate_lrc_file
from database.music_database import get_database
from utils.logging_config import get_logger
logger = get_logger("imports.pipeline")
pp_logger = get_logger("post_processing")
__all__ = [
"build_import_pipeline_runtime",
"post_process_matched_download",
"post_process_matched_download_with_verification",
]
def build_import_pipeline_runtime(
*,
automation_engine: Any | None = None,
on_download_completed: Any | None = None,
web_scan_manager: Any | None = None,
repair_worker: Any | None = None,
) -> SimpleNamespace:
"""Build the runtime object consumed by core.imports.pipeline."""
return SimpleNamespace(
automation_engine=automation_engine,
on_download_completed=on_download_completed,
web_scan_manager=web_scan_manager,
repair_worker=repair_worker,
)
def post_process_matched_download(context_key, context, file_path, runtime, metadata_runtime=None):
on_download_completed = getattr(runtime, "on_download_completed", None)
automation_engine = getattr(runtime, "automation_engine", None)
web_scan_manager = getattr(runtime, "web_scan_manager", None)
repair_worker = getattr(runtime, "repair_worker", None)
metadata_runtime = metadata_runtime or runtime
def _notify_download_completed(batch_id, task_id, success=True):
if on_download_completed:
on_download_completed(batch_id, task_id, success=success)
with post_process_locks_lock:
if context_key not in post_process_locks:
post_process_locks[context_key] = threading.Lock()
file_lock = post_process_locks[context_key]
file_lock.acquire()
try:
if not os.path.exists(file_path):
existing_final = context.get('_final_processed_path')
if existing_final and os.path.exists(existing_final):
logger.info(
f"[Race Guard] Source gone but destination exists — already processed by another thread: "
f"{os.path.basename(existing_final)}"
)
return
logger.error(
f"[Race Guard] Source file gone and no known destination — marking as failed: "
f"{os.path.basename(file_path)}"
)
context['_race_guard_failed'] = True
return
_basename = os.path.basename(file_path)
_prev_size = -1
for _stability_check in range(5):
try:
_cur_size = os.path.getsize(file_path)
except OSError:
_cur_size = -1
if _cur_size == _prev_size and _cur_size > 0:
break
_prev_size = _cur_size
if _stability_check == 0:
logger.info(f"Waiting for file to stabilise: {_basename} ({_cur_size} bytes)")
time.sleep(1.5)
else:
logger.info(f"File may still be writing after stability checks: {_basename} ({_prev_size} bytes)")
_skip_acoustid = False
try:
from core.acoustid_verification import AcoustIDVerification, VerificationResult
verifier = AcoustIDVerification()
available, available_reason = verifier.quick_check_available()
if available and not _skip_acoustid:
context = normalize_import_context(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
artist_context = get_import_context_artist(context)
expected_track = get_import_clean_title(context, default=original_search.get('title', ''))
expected_artist = ''
track_artists = track_info.get('artists', [])
if track_artists:
first = track_artists[0]
if isinstance(first, dict):
expected_artist = first.get('name', '')
elif isinstance(first, str):
expected_artist = first
if not expected_artist:
expected_artist = extract_artist_name(artist_context) or get_import_clean_artist(context, default='')
if expected_track and expected_artist:
logger.info(f"Running AcoustID verification for: '{expected_track}' by '{expected_artist}'")
verification_result, verification_msg = verifier.verify_audio_file(
file_path,
expected_track,
expected_artist,
context,
)
logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}")
context['_acoustid_result'] = verification_result.value
if verification_result == VerificationResult.FAIL:
try:
quarantine_path = move_to_quarantine(
file_path,
context,
verification_msg,
automation_engine,
)
logger.error(f"File quarantined due to verification failure: {quarantine_path}")
except Exception as quarantine_error:
logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}")
logger.error(f"Quarantine failed, deleting wrong file: {file_path}")
try:
os.remove(file_path)
except Exception as del_error:
logger.error(f"Could not delete wrong file either: {del_error}")
context['_acoustid_quarantined'] = True
context['_acoustid_failure_msg'] = verification_msg
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
task_id = context.get('task_id')
batch_id = context.get('batch_id')
if task_id:
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = (
f"AcoustID verification failed: {verification_msg}"
)
if task_id and batch_id:
_notify_download_completed(batch_id, task_id, success=False)
return
else:
logger.warning("AcoustID verification skipped: missing track/artist info")
context['_acoustid_result'] = 'skip'
else:
logger.info(f" AcoustID verification not available: {available_reason}")
context['_acoustid_result'] = 'disabled'
except Exception as verify_error:
logger.error(f"AcoustID verification error (continuing normally): {verify_error}")
context['_acoustid_result'] = 'error'
search_result = context.get('search_result', {}) or {}
if not isinstance(search_result, dict):
search_result = {}
is_simple_download = search_result.get('is_simple_download', False)
if is_simple_download:
logger.info(f"Processing simple download (no metadata enhancement): {file_path}")
destination, album_name, filename = build_simple_download_destination(context, file_path)
if album_name:
logger.info(f"Moving to album folder: {album_name}")
else:
logger.info("Moving to Transfer root (single track)")
safe_move_file(file_path, destination)
logger.info(f"Moved simple download to: {destination}")
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
if web_scan_manager:
threading.Thread(
target=lambda: web_scan_manager.request_scan("Simple download completed"),
daemon=True,
).start()
activity_target = f"{album_name}/{filename}" if album_name else filename
add_activity_item("", "Download Complete", activity_target, "Now")
logger.info(f"Simple download post-processing complete: {activity_target}")
context['_simple_download_completed'] = True
context['_final_path'] = str(destination)
emit_track_downloaded(context, automation_engine)
record_library_history_download(context)
record_download_provenance(context)
try:
check_and_remove_from_wishlist(context)
except Exception as wishlist_error:
logger.error(f"[Simple Download] Error checking wishlist removal: {wishlist_error}")
return
logger.info(f"Starting robust post-processing for: {context_key}")
context = normalize_import_context(context)
artist_context = get_import_context_artist(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
has_clean_metadata = get_import_has_clean_metadata(context)
if not artist_context:
logger.error("Post-processing failed: Missing artist context.")
return
_junk_artist_names = {'', 'unknown', 'unknown artist', 'various artists', 'none', 'null'}
_artist_name = (artist_context.get('name', '') if isinstance(artist_context, dict) else '').strip()
if _artist_name.lower() in _junk_artist_names:
logger.info(f"[Unknown Artist Guard] Artist name is '{_artist_name}' — attempting to resolve")
_resolved = False
track_info_guard = track_info or {}
original_search_guard = original_search or {}
_ti_artists = track_info_guard.get('artists', [])
if isinstance(_ti_artists, list) and _ti_artists:
_first = _ti_artists[0]
_name = _first.get('name', '') if isinstance(_first, dict) else str(_first)
if _name and _name.strip().lower() not in _junk_artist_names:
artist_context['name'] = _name.strip()
logger.info(f"[Unknown Artist Guard] Resolved from track_info.artists: '{_name}'")
_resolved = True
if not _resolved:
_os_artist = original_search_guard.get('artist') or original_search_guard.get('artist_name') or ''
if isinstance(_os_artist, str) and _os_artist.strip().lower() not in _junk_artist_names:
artist_context['name'] = _os_artist.strip()
logger.info(f"[Unknown Artist Guard] Resolved from original_search_result: '{_os_artist}'")
_resolved = True
if not _resolved:
_track_id = track_info_guard.get('id') or track_info_guard.get('track_id') or ''
if _track_id:
try:
from core.metadata_service import get_client_for_source, get_primary_source
_guard_source = get_import_source(context) or get_primary_source()
_fb_client = get_client_for_source(_guard_source) or get_client_for_source(get_primary_source())
if hasattr(_fb_client, 'get_track_details'):
_details = _fb_client.get_track_details(str(_track_id))
if _details and isinstance(_details, dict):
_d_artists = _details.get('artists', [])
if isinstance(_d_artists, list) and _d_artists:
_d_first = _d_artists[0]
_d_name = _d_first.get('name', '') if isinstance(_d_first, dict) else str(_d_first)
if _d_name and _d_name.strip().lower() not in _junk_artist_names:
artist_context['name'] = _d_name.strip()
logger.info(f"[Unknown Artist Guard] Resolved from metadata API: '{_d_name}'")
_resolved = True
except Exception as _guard_err:
logger.error(f"[Unknown Artist Guard] Metadata re-fetch failed: {_guard_err}")
if not _resolved:
logger.error(f"[Unknown Artist Guard] Could not resolve artist — proceeding with '{_artist_name}'")
context['artist'] = artist_context
playlist_folder_mode = track_info.get("_playlist_folder_mode", False)
logger.debug(f"[Debug] Post-processing - track_info type: {type(track_info)}, is None: {track_info is None}, is empty: {not track_info}")
logger.debug(f"[Debug] Post-processing - playlist_folder_mode: {playlist_folder_mode}")
if track_info:
logger.debug(f"[Debug] Post-processing - track_info keys: {list(track_info.keys())}")
if playlist_folder_mode:
playlist_name = track_info.get("_playlist_name", "Unknown Playlist")
logger.info(f"[Playlist Folder Mode] Organizing in playlist folder: {playlist_name}")
file_ext = os.path.splitext(file_path)[1]
final_path, _ = build_final_path_for_track(context, artist_context, None, file_ext)
logger.info(f"Playlist mode final path: '{final_path}'")
if not os.path.exists(file_path):
if os.path.exists(final_path):
logger.info(
f"[Playlist Folder Mode] Source gone but destination exists — already processed by another thread: "
f"{os.path.basename(final_path)}"
)
context['_final_processed_path'] = final_path
return
pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: Source file not found and destination does not exist: {file_path}")
raise FileNotFoundError(f"Source file not found and destination does not exist: {file_path}")
context['_audio_quality'] = get_audio_quality_string(file_path)
if context['_audio_quality']:
logger.info(f"Audio quality detected: {context['_audio_quality']}")
rejection_reason = check_flac_bit_depth(file_path, context)
if rejection_reason:
try:
quarantine_path = move_to_quarantine(
file_path,
context,
rejection_reason,
automation_engine,
)
logger.info(f"File quarantined due to bit depth filter: {quarantine_path}")
except Exception as quarantine_error:
logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}")
try:
os.remove(file_path)
except Exception:
pass
context['_bitdepth_rejected'] = True
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
task_id = context.get('task_id')
batch_id = context.get('batch_id')
if task_id:
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}"
if task_id and batch_id:
_notify_download_completed(batch_id, task_id, success=False)
return
try:
logger.warning(
f"[Metadata Input] Playlist mode - artist: '{artist_context.get('name', 'MISSING')}' "
f"(id: {artist_context.get('id', 'MISSING')})"
)
enhance_file_metadata(file_path, context, artist_context, None, runtime=metadata_runtime)
except Exception as meta_err:
import traceback
pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")
wipe_source_tags(file_path)
logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'")
safe_move_file(file_path, final_path)
context['_final_processed_path'] = final_path
if config_manager.get('post_processing.replaygain_enabled', False):
try:
from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF
if _rg_ffmpeg_ok():
lufs, peak_dbfs = _rg_analyze(final_path)
gain_db = _RG_REF - lufs
_rg_write(final_path, gain_db, peak_dbfs)
pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB — {os.path.basename(final_path)}")
except Exception as rg_err:
pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}")
downsampled_path = downsample_hires_flac(final_path, context)
if downsampled_path:
final_path = downsampled_path
context['_final_processed_path'] = final_path
blasphemy_path = create_lossy_copy(final_path)
if blasphemy_path:
context['_final_processed_path'] = blasphemy_path
downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
cleanup_empty_directories(downloads_path, file_path)
logger.info(f"[Playlist Folder Mode] Post-processing complete: {final_path}")
try:
check_and_remove_from_wishlist(context)
except Exception as wishlist_error:
logger.error(f"[Playlist Folder] Error checking wishlist removal: {wishlist_error}")
emit_track_downloaded(context, automation_engine)
record_library_history_download(context)
record_download_provenance(context)
task_id = context.get('task_id')
batch_id = context.get('batch_id')
if task_id and batch_id:
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['stream_processed'] = True
download_tasks[task_id]['status'] = 'completed'
logger.info(f"[Playlist Folder Mode] Marked task {task_id} as completed")
_notify_download_completed(batch_id, task_id, success=True)
return
is_album_download = bool(context.get("is_album_download", False))
album_info = build_import_album_info(context, force_album=is_album_download)
if is_album_download:
if has_clean_metadata:
logger.info("Album context with clean metadata found - using normalized album info")
else:
logger.warning("Album context found without clean metadata - using normalized album info")
elif not album_info.get('is_album'):
logger.info("Single track download - attempting album detection")
detected_album_info = detect_album_info_web(context, artist_context)
if detected_album_info:
album_info = detected_album_info
if album_info and album_info['is_album'] and not is_album_download:
logger.info(
"SMART ALBUM GROUPING for track=%r original_album=%r",
album_info.get('clean_track_name', 'Unknown'),
album_info.get('album_name', 'None'),
)
original_album = original_search.get("album") if original_search.get("album") else None
consistent_album_name = resolve_album_group(artist_context, album_info, original_album)
album_info['album_name'] = consistent_album_name
logger.info("Album grouping complete: final_album=%r", consistent_album_name)
elif album_info and album_info['is_album'] and is_album_download:
logger.info(
"EXPLICIT ALBUM DOWNLOAD - preserving album name=%r; skipping smart grouping",
album_info.get('album_name', 'None'),
)
context['_audio_quality'] = get_audio_quality_string(file_path)
if context['_audio_quality']:
logger.info(f"Audio quality detected: {context['_audio_quality']}")
rejection_reason = check_flac_bit_depth(file_path, context)
if rejection_reason:
try:
quarantine_path = move_to_quarantine(
file_path,
context,
rejection_reason,
automation_engine,
)
logger.info(f"File quarantined due to bit depth filter: {quarantine_path}")
except Exception as quarantine_error:
logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}")
try:
os.remove(file_path)
except Exception:
pass
context['_bitdepth_rejected'] = True
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
task_id = context.get('task_id')
batch_id = context.get('batch_id')
if task_id:
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}"
if task_id and batch_id:
_notify_download_completed(batch_id, task_id, success=False)
return
file_ext = os.path.splitext(file_path)[1]
clean_track_name = get_import_clean_title(
context,
album_info=album_info,
default=original_search.get('title', 'Unknown Track'),
)
track_number = album_info.get('track_number', 1)
logger.debug(
"Final track_number processing: source=%s album_info_track_number=%s track_number=%s",
album_info.get('source', 'unknown'),
album_info.get('track_number', 'NOT_FOUND'),
track_number,
)
if track_number is None:
track_number = extract_track_number_from_filename(file_path)
logger.info(
"Track number was None; extracted from filename=%r -> %s",
os.path.basename(file_path),
track_number,
)
if not isinstance(track_number, int) or track_number < 1:
logger.error(f"Invalid track number ({track_number}), defaulting to 1")
track_number = 1
logger.debug(f"FINAL track_number used for filename: {track_number}")
album_info['track_number'] = track_number
album_info['clean_track_name'] = clean_track_name
logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata")
final_path, _ = build_final_path_for_track(context, artist_context, album_info, file_ext)
logger.info(f"Resolved path: '{final_path}'")
context['_final_processed_path'] = final_path
try:
logger.warning(f"[Metadata Input] artist: '{artist_context.get('name', 'MISSING')}' (id: {artist_context.get('id', 'MISSING')})")
if album_info:
logger.warning(
f"[Metadata Input] album: '{album_info.get('album_name', 'MISSING')}', "
f"track#: {album_info.get('track_number', 'MISSING')}, disc#: {album_info.get('disc_number', 'MISSING')}, "
f"source: {album_info.get('source', 'unknown')}"
)
else:
logger.info("[Metadata Input] album_info: None (single track)")
enhance_file_metadata(file_path, context, artist_context, album_info, runtime=metadata_runtime)
except Exception as meta_err:
import traceback
pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")
wipe_source_tags(file_path)
_enhance_source_info = context.get('track_info', {}).get('source_info') or {}
if isinstance(_enhance_source_info, str):
try:
_enhance_source_info = json.loads(_enhance_source_info)
except (json.JSONDecodeError, TypeError):
_enhance_source_info = {}
is_enhance_download = _enhance_source_info.get('enhance', False)
logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'")
if os.path.exists(final_path):
if not os.path.exists(file_path):
logger.info(f"[Protection] Destination exists and source already gone - file already transferred: {os.path.basename(final_path)}")
return
try:
from mutagen import File as MutagenFile
existing_file = MutagenFile(final_path)
has_metadata = existing_file is not None and len(existing_file.tags or {}) > 2
if has_metadata and not is_enhance_download:
_replace_lower = config_manager.get('import.replace_lower_quality', False)
if _replace_lower:
_existing_tier = get_quality_tier_from_extension(final_path)
_incoming_tier = get_quality_tier_from_extension(file_path)
if _incoming_tier[1] < _existing_tier[1]:
logger.info(f"[Quality Replace] Replacing {_existing_tier[0]} with {_incoming_tier[0]}: {os.path.basename(final_path)}")
try:
os.remove(final_path)
except Exception as e:
logger.error(f"[Quality Replace] Could not remove existing file: {e}")
else:
logger.info(
f"[Protection] Existing file is same or better quality ({_existing_tier[0]} vs {_incoming_tier[0]}) - skipping: "
f"{os.path.basename(final_path)}"
)
try:
os.remove(file_path)
except FileNotFoundError:
pass
except Exception as e:
logger.error(f"[Protection] Error removing redundant file: {e}")
return
else:
logger.info(f"[Protection] Existing file already has metadata enhancement - skipping overwrite: {os.path.basename(final_path)}")
logger.info(f"[Protection] Removing redundant download file: {os.path.basename(file_path)}")
try:
os.remove(file_path)
except FileNotFoundError:
logger.error(f"[Protection] Could not remove redundant file (already gone): {file_path}")
except Exception as e:
logger.error(f"[Protection] Error removing redundant file: {e}")
return
elif is_enhance_download:
logger.info(f"[Enhance] Quality enhance mode — replacing existing file: {os.path.basename(final_path)}")
try:
os.remove(final_path)
except Exception as e:
logger.error(f"[Enhance] Could not remove existing file for replacement: {e}")
else:
logger.info(f"[Protection] Existing file lacks metadata - safe to overwrite: {os.path.basename(final_path)}")
try:
os.remove(final_path)
except FileNotFoundError:
pass
except Exception as check_error:
logger.error(f"[Protection] Error checking existing file metadata, proceeding with overwrite: {check_error}")
try:
if os.path.exists(final_path):
os.remove(final_path)
except Exception as e:
logger.error(f"[Protection] Failed to remove existing file for overwrite: {e}")
if not os.path.exists(file_path):
if os.path.exists(final_path):
logger.info(f"[Pre-Move] Source already gone and destination exists - another thread completed transfer: {os.path.basename(final_path)}")
download_cover_art(album_info, os.path.dirname(final_path), context)
generate_lrc_file(final_path, context, artist_context, album_info)
return
expected_dir = os.path.dirname(final_path)
expected_stem = os.path.splitext(os.path.basename(final_path))[0]
expected_ext = os.path.splitext(final_path)[1]
found_variant = None
check_exts = {expected_ext}
if expected_ext == '.flac' and config_manager.get('lossy_copy.enabled', False) and config_manager.get('lossy_copy.delete_original', False):
_lossy_ext_map = {'mp3': '.mp3', 'opus': '.opus', 'aac': '.m4a'}
_lossy_codec = config_manager.get('lossy_copy.codec', 'mp3')
check_exts.add(_lossy_ext_map.get(_lossy_codec, '.mp3'))
if os.path.exists(expected_dir):
for f in os.listdir(expected_dir):
f_ext = os.path.splitext(f)[1].lower()
if f_ext in check_exts and os.path.splitext(f)[0].startswith(expected_stem):
found_variant = os.path.join(expected_dir, f)
break
if found_variant:
logger.debug(f"[Pre-Move] Source gone but found variant in destination (stream processor handled it): {os.path.basename(found_variant)}")
context['_final_processed_path'] = found_variant
download_cover_art(album_info, expected_dir, context)
generate_lrc_file(found_variant, context, artist_context, album_info)
return
logger.warning(f"[Pre-Move] Source file gone and no matching file in destination: {os.path.basename(file_path)}")
raise FileNotFoundError(f"Source file vanished before move and destination does not exist: {file_path}")
safe_move_file(file_path, final_path)
if is_enhance_download and _enhance_source_info.get('original_file_path'):
original_enhance_path = _enhance_source_info['original_file_path']
if os.path.normpath(original_enhance_path) != os.path.normpath(final_path) and os.path.exists(original_enhance_path):
try:
os.remove(original_enhance_path)
old_fmt = os.path.splitext(original_enhance_path)[1]
new_fmt = os.path.splitext(final_path)[1]
logger.info(f"[Enhance] Upgraded {old_fmt}{new_fmt}: {os.path.basename(final_path)}")
except Exception as e:
logger.error(f"[Enhance] Could not remove old-format file: {e}")
elif is_enhance_download:
old_fmt = _enhance_source_info.get('original_format', 'unknown')
new_fmt = os.path.splitext(final_path)[1]
logger.info(f"[Enhance] Replaced in-place ({old_fmt}{new_fmt}): {os.path.basename(final_path)}")
download_cover_art(album_info, os.path.dirname(final_path), context)
generate_lrc_file(final_path, context, artist_context, album_info)
if config_manager.get('post_processing.replaygain_enabled', False):
try:
from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF
if _rg_ffmpeg_ok():
lufs, peak_dbfs = _rg_analyze(final_path)
gain_db = _RG_REF - lufs
_rg_write(final_path, gain_db, peak_dbfs)
pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB, peak {peak_dbfs:.2f} dBFS — {os.path.basename(final_path)}")
except Exception as rg_err:
pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}")
downsampled_path = downsample_hires_flac(final_path, context)
if downsampled_path:
final_path = downsampled_path
context['_final_processed_path'] = final_path
blasphemy_path = create_lossy_copy(final_path)
if blasphemy_path:
context['_final_processed_path'] = blasphemy_path
downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
cleanup_empty_directories(downloads_path, file_path)
logger.info(f"Post-processing complete for: {context.get('_final_processed_path', final_path)}")
emit_track_downloaded(context, automation_engine)
record_library_history_download(context)
record_download_provenance(context)
record_soulsync_library_entry(context, artist_context, album_info)
try:
if not playlist_folder_mode:
completed_path = context.get('_final_processed_path', final_path)
record_retag_download(context, artist_context, album_info, completed_path)
except Exception as retag_err:
logger.error(f"[Post-Process] Retag data capture failed (non-fatal): {retag_err}")
try:
completed_path = context.get('_final_processed_path', final_path)
batch_id_for_repair = context.get('batch_id')
if completed_path and batch_id_for_repair and repair_worker:
album_folder = os.path.dirname(str(completed_path))
if album_folder:
repair_worker.register_folder(batch_id_for_repair, album_folder)
except Exception as repair_err:
logger.error(f"[Post-Process] Repair folder registration failed: {repair_err}")
try:
completed_path = context.get('_final_processed_path', final_path)
batch_id_for_consistency = context.get('batch_id')
if completed_path and batch_id_for_consistency and album_info and album_info.get('is_album'):
_file_info = {
'path': str(completed_path),
'track_number': album_info.get('track_number', 1),
'disc_number': album_info.get('disc_number', 1),
'title': get_import_clean_title(
context,
album_info=album_info,
default=album_info.get('clean_track_name', ''),
),
}
with tasks_lock:
if batch_id_for_consistency in download_batches:
download_batches[batch_id_for_consistency].setdefault('_consistency_files', []).append(_file_info)
except Exception as cons_err:
logger.error(f"[Post-Process] Album consistency registration failed: {cons_err}")
try:
check_and_remove_from_wishlist(context)
except Exception as wishlist_error:
logger.error(f"[Post-Process] Error checking wishlist removal: {wishlist_error}")
task_id = context.get('task_id')
batch_id = context.get('batch_id')
if task_id and batch_id:
logger.info(f"[Post-Process] Calling completion callback for task {task_id} in batch {batch_id}")
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['stream_processed'] = True
download_tasks[task_id]['status'] = 'completed'
logger.info(f"[Post-Process] Marked task {task_id} as completed")
_notify_download_completed(batch_id, task_id, success=True)
except Exception as e:
import traceback
pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: {e}")
pp_logger.info(traceback.format_exc())
logger.error(f"\nCRITICAL ERROR in post-processing for {context_key}: {e}")
traceback.print_exc()
source_exists = os.path.exists(file_path) if file_path else False
if source_exists:
if context_key in processed_download_ids:
processed_download_ids.remove(context_key)
logger.warning(f"Removed {context_key} from processed set - will retry on next check")
with matched_context_lock:
if context_key not in matched_downloads_context:
matched_downloads_context[context_key] = context
logger.warning(f"Re-added {context_key} to context for retry")
else:
logger.warning(f"Source file gone, not retrying: {context_key}")
finally:
file_lock.release()
with post_process_locks_lock:
post_process_locks.pop(context_key, None)
def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime, metadata_runtime=None):
on_download_completed = getattr(runtime, "on_download_completed", None)
def _notify_download_completed(batch_id, task_id, success=True):
if on_download_completed:
on_download_completed(batch_id, task_id, success=success)
logger = pp_logger
try:
original_task_id = context.pop('task_id', None)
original_batch_id = context.pop('batch_id', None)
post_process_matched_download(context_key, context, file_path, runtime, metadata_runtime=metadata_runtime)
if original_task_id:
context['task_id'] = original_task_id
if original_batch_id:
context['batch_id'] = original_batch_id
if context.get('_race_guard_failed'):
logger.info(f"Race guard: source file gone for task {task_id} — marking as failed")
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = 'Source file was already processed or removed by another task'
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=False)
return
if context.get('_acoustid_quarantined'):
failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed')
logger.info(f"File was quarantined by AcoustID verification (task={task_id}): {failure_msg}")
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {failure_msg}"
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=False)
return
if context.get('_simple_download_completed'):
expected_final_path = context.get('_final_path')
if expected_final_path and os.path.exists(expected_final_path):
with tasks_lock:
if task_id in download_tasks:
_mark_task_completed(task_id, context.get('track_info'))
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=True)
return
logger.info(
f"FAILED simple download file not found at: {expected_final_path} "
f"(task={task_id}, context={context_key})"
)
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = (
f"Downloaded file not found at expected location: {os.path.basename(expected_final_path)}"
)
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=False)
return
expected_final_path = context.get('_final_processed_path')
if not expected_final_path:
logger.info(f"No _final_processed_path in context for task {task_id} — cannot verify, assuming success")
with tasks_lock:
if task_id in download_tasks:
_mark_task_completed(task_id, context.get('track_info'))
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=True)
return
if os.path.exists(expected_final_path):
redownload_ctx = None
with tasks_lock:
if task_id in download_tasks:
_mark_task_completed(task_id, context.get('track_info'))
download_tasks[task_id]['metadata_enhanced'] = True
redownload_ctx = download_tasks[task_id].get('_redownload_context')
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
if redownload_ctx:
try:
old_path = redownload_ctx.get('old_file_path')
lib_track_id = redownload_ctx.get('library_track_id')
if redownload_ctx.get('delete_old_file') and old_path and os.path.exists(old_path):
if os.path.normpath(old_path) != os.path.normpath(expected_final_path):
os.remove(old_path)
logger.info(f"[Redownload] Deleted old file: {old_path}")
if lib_track_id and expected_final_path:
_rd_db = get_database()
_rd_conn = _rd_db._get_connection()
_rd_cursor = _rd_conn.cursor()
_rd_cursor.execute(
"""
UPDATE tracks SET file_path = ?, updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(expected_final_path, lib_track_id),
)
_rd_conn.commit()
_rd_conn.close()
logger.info(f"[Redownload] Updated DB path for track {lib_track_id}")
except Exception as e:
logger.error(f"[Redownload] Post-processing hook error: {e}")
_notify_download_completed(batch_id, task_id, success=True)
else:
track_name = get_import_clean_title(context, default=context_key)
logger.info(f"FAILED verification for '{track_name}' (task={task_id})")
logger.info(f" expected_final_path: {expected_final_path}")
logger.info(f" file_path (source): {file_path}, exists={os.path.exists(file_path)}")
logger.info(
f" is_album={context.get('is_album_download', False)}, "
f"has_clean_data={get_import_has_clean_metadata(context)}"
)
expected_dir = os.path.dirname(expected_final_path)
if os.path.exists(expected_dir):
dir_contents = os.listdir(expected_dir)
logger.info(f" directory contains {len(dir_contents)} files: {dir_contents[:20]}")
else:
logger.info(f" directory does not exist: {expected_dir}")
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = (
f'File verification failed: expected file at {os.path.basename(expected_final_path)} but it was not found after processing'
)
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=False)
except Exception as e:
import traceback
logger.info(f"EXCEPTION in post-processing for '{context_key}' (task={task_id}): {e}")
logger.info(traceback.format_exc())
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = f"Post-processing verification failed: {str(e)}"
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=False)

412
core/imports/resolution.py Normal file
View file

@ -0,0 +1,412 @@
"""Single-track import lookup and context-building helpers."""
from __future__ import annotations
from typing import Any, Dict, List, Optional
from utils.logging_config import get_logger
logger = get_logger("imports.resolution")
def _get_metadata_service():
from core import metadata_service
return metadata_service
def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any:
if value is None:
return default
for name in names:
if isinstance(value, dict):
if name in value and value[name] is not None:
return value[name]
else:
candidate = getattr(value, name, None)
if candidate is not None:
return candidate
return default
def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]:
if not artists:
return []
if isinstance(artists, (str, bytes)):
artists = [artists]
elif isinstance(artists, dict):
artists = [artists]
else:
try:
artists = list(artists)
except TypeError:
artists = [artists]
normalized: List[Dict[str, Any]] = []
for artist in artists:
if isinstance(artist, dict):
name = _extract_lookup_value(artist, 'name', 'artist_name', 'title', default='') or ''
artist_id = _extract_lookup_value(artist, 'id', 'artist_id', default='') or ''
entry: Dict[str, Any] = {}
if name:
entry['name'] = str(name)
if artist_id:
entry['id'] = str(artist_id)
genres = _extract_lookup_value(artist, 'genres', default=None)
if genres is not None:
entry['genres'] = genres
if entry:
normalized.append(entry)
continue
name = str(artist).strip()
if name:
normalized.append({'name': name})
return normalized
def _get_source_chain_for_lookup(
source_override: Optional[str] = None,
allow_fallback: bool = True,
) -> List[str]:
metadata_service = _get_metadata_service()
primary_source = metadata_service.get_primary_source()
source_chain = list(metadata_service.get_source_priority(primary_source))
override = (source_override or '').strip().lower()
if override:
source_chain = [override] + [source for source in source_chain if source != override]
if not allow_fallback:
source_chain = source_chain[:1]
return source_chain
def _build_track_search_query(source: str, title: str, artist: str) -> str:
base_query = " ".join(part for part in (title, artist) if part).strip()
if source == 'deezer' and title:
if artist:
return f'artist:"{artist}" track:"{title}"'
return f'track:"{title}"'
return base_query or title or artist
def _pick_best_track_match(search_results: List[Any], title: str, artist: str = '') -> Optional[Any]:
if not search_results:
return None
target_title = str(title or '').strip().lower()
target_artist = str(artist or '').strip().lower()
for candidate in search_results:
candidate_title = str(_extract_lookup_value(candidate, 'name', 'title', 'track_name', default='') or '').strip().lower()
if candidate_title != target_title:
continue
if not target_artist:
return candidate
candidate_artists = _normalize_context_artists(_extract_lookup_value(candidate, 'artists', default=[]))
candidate_artist_name = candidate_artists[0]['name'].strip().lower() if candidate_artists else ''
if candidate_artist_name == target_artist:
return candidate
return search_results[0]
def search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]:
if not client or not hasattr(client, 'search_tracks'):
return []
try:
kwargs = {'limit': limit}
if source == 'spotify':
kwargs['allow_fallback'] = False
return client.search_tracks(query, **kwargs) or []
except Exception as exc:
logger.debug("Could not search %s for %s: %s", source, query, exc)
return []
def _build_single_import_context_payload(
track_data: Any,
source: Optional[str],
source_priority: List[str],
requested_title: str = '',
requested_artist: str = '',
) -> Dict[str, Any]:
album_data = _extract_lookup_value(track_data, 'album', default=None)
track_id = str(_extract_lookup_value(track_data, 'id', 'track_id', 'trackId', default='') or '')
track_name = _extract_lookup_value(track_data, 'name', 'title', 'trackName', default='') or requested_title or 'Unknown Track'
track_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'artists', default=[]))
if not track_artists and requested_artist:
track_artists = [{'name': requested_artist}]
primary_track_artist = track_artists[0] if track_artists else {}
primary_artist_name = primary_track_artist.get('name') or requested_artist or 'Unknown Artist'
primary_artist_id = str(primary_track_artist.get('id', '') or _extract_lookup_value(track_data, 'artist_id', 'artistId', default='') or '')
album_name = _extract_lookup_value(track_data, 'album_name', 'collectionName', default='') or ''
album_id = str(_extract_lookup_value(track_data, 'album_id', 'collectionId', 'albumId', default='') or '')
release_date = str(_extract_lookup_value(track_data, 'release_date', default='') or '')
album_type = str(_extract_lookup_value(track_data, 'album_type', default='album') or 'album')
total_tracks = int(_extract_lookup_value(track_data, 'total_tracks', 'track_count', default=0) or 0)
album_images: List[Dict[str, Any]] = []
album_image_url = str(_extract_lookup_value(track_data, 'image_url', 'thumb_url', default='') or '')
album_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'album_artists', 'artists', default=[]))
if isinstance(album_data, dict):
album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name
album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id)
release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date)
album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type)
total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks)
album_images = _extract_lookup_value(album_data, 'images', default=[]) or []
if not album_image_url:
album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '')
if not album_image_url and album_images:
album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '')
album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[]))
elif album_data:
album_name = album_name or str(album_data)
if not album_artists and primary_artist_name:
album_artists = [{'name': primary_artist_name}]
if not album_image_url and album_images:
album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '')
track_info = {
'id': track_id,
'name': track_name,
'track_number': int(_extract_lookup_value(track_data, 'track_number', 'trackNumber', default=1) or 1),
'disc_number': int(_extract_lookup_value(track_data, 'disc_number', 'discNumber', default=1) or 1),
'duration_ms': int(_extract_lookup_value(track_data, 'duration_ms', 'duration', 'trackTimeMillis', default=0) or 0),
'artists': track_artists or [{'name': primary_artist_name}],
'uri': str(_extract_lookup_value(track_data, 'uri', default='') or ''),
'album': album_name,
'album_id': album_id,
'album_type': album_type,
'release_date': release_date,
'_source': source or '',
}
album_payload = {
'id': album_id,
'name': album_name,
'release_date': release_date,
'total_tracks': total_tracks or 1,
'album_type': album_type,
'image_url': album_image_url,
'images': album_images,
'artists': album_artists,
'_source': source or '',
}
artist_payload = {
'id': primary_artist_id,
'name': primary_artist_name,
'genres': [],
'_source': source or '',
}
original_search = {
'title': track_name,
'artist': primary_artist_name,
'album': album_name,
'track_number': track_info['track_number'],
'disc_number': track_info['disc_number'],
'clean_title': track_name,
'clean_album': album_name,
'clean_artist': primary_artist_name,
'artists': track_info['artists'],
'duration_ms': track_info['duration_ms'],
'id': track_id,
'_source': source or '',
}
return {
'success': bool(track_id or track_name != requested_title or album_name),
'source': source,
'source_priority': source_priority,
'context': {
'artist': artist_payload,
'album': album_payload,
'track_info': track_info,
'original_search_result': original_search,
'is_album_download': False,
'has_clean_metadata': bool(track_id),
'has_full_metadata': bool(track_id),
'source': source,
'source_priority': source_priority,
},
}
def _build_single_import_fallback_context(
requested_title: str,
requested_artist: str,
source_priority: List[str],
) -> Dict[str, Any]:
artist_name = requested_artist or 'Unknown Artist'
title = requested_title or 'Unknown Track'
return {
'success': False,
'source': None,
'source_priority': source_priority,
'context': {
'artist': {
'id': '',
'name': artist_name,
'genres': [],
'_source': '',
},
'album': {
'id': '',
'name': '',
'release_date': '',
'total_tracks': 1,
'album_type': 'album',
'image_url': '',
'images': [],
'artists': [],
'_source': '',
},
'track_info': {
'id': '',
'name': title,
'track_number': 1,
'disc_number': 1,
'duration_ms': 0,
'artists': [{'name': artist_name}],
'uri': '',
'album': '',
'album_id': '',
'album_type': 'album',
'release_date': '',
'_source': '',
},
'original_search_result': {
'title': title,
'artist': artist_name,
'album': '',
'track_number': 1,
'disc_number': 1,
'clean_title': title,
'clean_album': '',
'clean_artist': artist_name,
'artists': [{'name': artist_name}],
'duration_ms': 0,
'id': '',
'_source': '',
},
'is_album_download': False,
'has_clean_metadata': False,
'has_full_metadata': False,
'source': None,
'source_priority': source_priority,
},
}
def get_single_track_import_context(
title: str,
artist: str = '',
override_id: Optional[str] = None,
override_source: str = 'spotify',
source_override: Optional[str] = None,
) -> Dict[str, Any]:
"""Build an import context for singles using source-priority metadata lookup."""
metadata_service = _get_metadata_service()
source_priority = _get_source_chain_for_lookup(source_override=source_override, allow_fallback=True)
title = (title or '').strip()
artist = (artist or '').strip()
if override_id:
chosen_source = (override_source or 'spotify').strip().lower() or 'spotify'
client = metadata_service.get_client_for_source(chosen_source)
if client and hasattr(client, 'get_track_details'):
try:
track_data = client.get_track_details(str(override_id))
if track_data:
payload = _build_single_import_context_payload(
track_data,
chosen_source,
source_priority,
requested_title=title,
requested_artist=artist,
)
if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'):
try:
artist_details = client.get_artist(payload['context']['artist']['id'])
if artist_details:
payload['context']['artist']['genres'] = _extract_lookup_value(
artist_details,
'genres',
default=[],
) or []
except Exception:
pass
return payload
except Exception as exc:
logger.debug("Override track lookup failed on %s for %s: %s", chosen_source, override_id, exc)
for source in source_priority:
client = metadata_service.get_client_for_source(source)
if not client:
continue
search_query = _build_track_search_query(source, title, artist)
if not search_query:
continue
search_results = search_tracks_for_source(source, client, search_query, limit=5)
if not search_results and search_query != title:
search_results = search_tracks_for_source(source, client, title, limit=5)
if not search_results and artist and search_query != artist:
search_results = search_tracks_for_source(source, client, artist, limit=5)
if not search_results:
continue
best_match = _pick_best_track_match(search_results, title or search_query, artist)
if not best_match:
continue
resolved_track_id = str(_extract_lookup_value(best_match, 'id', 'track_id', 'trackId', default='') or '')
resolved_data = best_match
if resolved_track_id and hasattr(client, 'get_track_details'):
try:
detailed = client.get_track_details(resolved_track_id)
if detailed:
resolved_data = detailed
except Exception as exc:
logger.debug("Track detail lookup failed on %s for %s: %s", source, resolved_track_id, exc)
payload = _build_single_import_context_payload(
resolved_data,
source,
source_priority,
requested_title=title,
requested_artist=artist,
)
if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'):
try:
artist_details = client.get_artist(payload['context']['artist']['id'])
if artist_details:
payload['context']['artist']['genres'] = _extract_lookup_value(
artist_details,
'genres',
default=[],
) or []
except Exception:
pass
return payload
return _build_single_import_fallback_context(title, artist, source_priority)

View file

@ -0,0 +1,574 @@
"""Import post-processing side effects that do not need web runtime state."""
from __future__ import annotations
import hashlib
import json
import os
from typing import Any, Dict, List, Optional
from config.settings import config_manager
from core.imports.context import (
extract_artist_name,
get_import_clean_album,
get_import_clean_artist,
get_import_clean_title,
get_import_context_album,
get_import_context_artist,
get_import_original_search,
get_import_search_result,
get_import_source,
get_import_source_ids,
get_import_track_info,
normalize_import_context,
get_library_source_id_columns,
)
from core.wishlist_service import get_wishlist_service
from database.music_database import get_database
from utils.logging_config import get_logger
logger = get_logger("imports.side_effects")
def _get_config_manager():
return config_manager
def _primary_track_artist_name(track_info: Dict[str, Any]) -> str:
artists = (track_info or {}).get("artists", [])
if isinstance(artists, list) and artists:
first = artists[0]
if isinstance(first, dict):
return str(first.get("name", "") or "")
return str(first or "")
if isinstance(artists, str):
return artists
return str((track_info or {}).get("artist", "") or "")
def _all_profile_wishlist_tracks(wishlist_service) -> List[Dict[str, Any]]:
database = get_database()
all_profiles = database.get_all_profiles()
wishlist_tracks: List[Dict[str, Any]] = []
for profile in all_profiles:
wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=profile["id"]))
return wishlist_tracks
def _stable_soulsync_id(text: str) -> str:
return str(abs(int(hashlib.md5(text.encode("utf-8", errors="replace")).hexdigest(), 16)) % (10 ** 9))
def emit_track_downloaded(context: Dict[str, Any], automation_engine=None) -> None:
"""Emit the track_downloaded automation event."""
try:
if not automation_engine:
return
ti = context.get("track_info") or context.get("search_result") or {}
artist_name = ""
artists = ti.get("artists", [])
if artists:
first = artists[0]
artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first)
automation_engine.emit(
"track_downloaded",
{
"artist": artist_name,
"title": ti.get("name", ti.get("title", "")),
"album": ti.get("album", ""),
"quality": context.get("_audio_quality", "Unknown"),
},
)
except Exception:
pass
def record_library_history_download(context: Dict[str, Any]) -> None:
"""Record a completed download to the library_history table."""
try:
search_result = context.get("original_search_result") or context.get("search_result") or {}
username = search_result.get("username", context.get("_download_username", ""))
source_map = {
"youtube": "YouTube",
"tidal": "Tidal",
"qobuz": "Qobuz",
"hifi": "HiFi",
"deezer_dl": "Deezer",
"lidarr": "Lidarr",
}
download_source = source_map.get(username, "Soulseek")
ti = context.get("track_info") or context.get("search_result") or {}
artist_name = _primary_track_artist_name(ti)
if not artist_name:
artist_name = ti.get("artist", "")
album_raw = ti.get("album", "")
album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "")
title = ti.get("name", ti.get("title", ""))
quality = context.get("_audio_quality", "")
file_path = context.get("_final_processed_path", context.get("_final_path", ""))
thumb_url = ""
album_context = get_import_context_album(context)
if album_context:
thumb_url = album_context.get("image_url", "")
if not thumb_url:
images = album_context.get("images", [])
if images:
thumb_url = images[0].get("url", "")
if not thumb_url:
album_info = context.get("album_info", {})
if isinstance(album_info, dict):
thumb_url = album_info.get("album_image_url", "")
source_filename = search_result.get("filename", "")
source_track_id = search_result.get("track_id", "") or search_result.get("id", "") or ti.get("id", "")
source_track_title = search_result.get("title", "") or search_result.get("name", "")
source_artist = search_result.get("artist", "")
if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr"):
stream_id = source_filename.split("||")[0]
if stream_id and not source_track_id:
source_track_id = stream_id
acoustid_result = context.get("_acoustid_result", "")
db = get_database()
db.add_library_history_entry(
event_type="download",
title=title,
artist_name=artist_name,
album_name=album_name,
quality=quality,
file_path=file_path,
thumb_url=thumb_url,
download_source=download_source,
source_track_id=source_track_id,
source_track_title=source_track_title,
source_filename=source_filename,
acoustid_result=acoustid_result,
source_artist=source_artist,
)
except Exception:
pass
def record_download_provenance(context: Dict[str, Any]) -> None:
"""Record source provenance for a completed download."""
try:
search_result = context.get("original_search_result") or context.get("search_result") or {}
username = search_result.get("username", context.get("_download_username", ""))
filename = search_result.get("filename", "")
source_service = {
"youtube": "youtube",
"tidal": "tidal",
"qobuz": "qobuz",
"hifi": "hifi",
"deezer_dl": "deezer",
"lidarr": "lidarr",
}.get(username, "soulseek")
ti = context.get("track_info") or context.get("search_result") or {}
artist_name = _primary_track_artist_name(ti)
if not artist_name:
artist_name = ti.get("artist", "")
album_raw = ti.get("album", "")
album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "")
title = ti.get("name", ti.get("title", ""))
file_path = context.get("_final_processed_path", context.get("_final_path", ""))
quality = context.get("_audio_quality", "")
size = search_result.get("size", 0)
bit_depth = None
sample_rate = None
bitrate = None
try:
if file_path and os.path.isfile(file_path):
from mutagen import File as MutagenFile
audio = MutagenFile(file_path)
if audio and audio.info:
sample_rate = getattr(audio.info, "sample_rate", None)
bitrate = getattr(audio.info, "bitrate", None)
bit_depth = getattr(audio.info, "bits_per_sample", None)
except Exception:
pass
db = get_database()
db.record_track_download(
file_path=file_path,
source_service=source_service,
source_username=username,
source_filename=filename,
source_size=size or 0,
audio_quality=quality,
track_title=title,
track_artist=artist_name,
track_album=album_name,
bit_depth=bit_depth,
sample_rate=sample_rate,
bitrate=bitrate,
)
except Exception:
pass
def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any]) -> None:
"""Write imported media to the SoulSync library tables when the active server is SoulSync."""
try:
if _get_config_manager().get_active_media_server() != "soulsync":
return
context = normalize_import_context(context)
final_path = context.get("_final_processed_path")
if not final_path:
return
album_ctx = get_import_context_album(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
source = get_import_source(context)
source_ids = get_import_source_ids(context)
source_columns = get_library_source_id_columns(source)
artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="")
if not artist_name or artist_name in ("Unknown", "Unknown Artist"):
return
album_name = ""
if album_info and isinstance(album_info, dict):
album_name = album_info.get("album_name", "")
if not album_name:
album_name = album_ctx.get("name", "") or original_search.get("album", "")
if not album_name:
album_name = track_info.get("name", "Unknown")
track_name = get_import_clean_title(
context,
album_info=album_info,
default=track_info.get("name", "") or original_search.get("title", ""),
)
track_number = (track_info.get("track_number") or (album_info.get("track_number") if isinstance(album_info, dict) else None)) or 1
duration_ms = track_info.get("duration_ms", 0) or 0
year = None
release_date = album_ctx.get("release_date", "")
if release_date and len(release_date) >= 4:
try:
year = int(release_date[:4])
except ValueError:
pass
image_url = album_ctx.get("image_url", "")
if not image_url:
images = album_ctx.get("images", [])
if images and isinstance(images, list) and len(images) > 0:
img = images[0]
image_url = img.get("url", "") if isinstance(img, dict) else str(img)
artist_source_id = source_ids.get("artist_id", "")
album_source_id = source_ids.get("album_id", "")
track_source_id = source_ids.get("track_id", "")
for key in ("auto_import", "from_sync_modal", "explicit_artist", "explicit_album", ""):
if artist_source_id == key:
artist_source_id = ""
if album_source_id == key:
album_source_id = ""
if track_source_id == key:
track_source_id = ""
genres = (artist_context or {}).get("genres", []) if isinstance(artist_context, dict) else []
if genres:
from core.genre_filter import filter_genres as _filter_genres
genres = _filter_genres(genres, _get_config_manager())
genres_json = json.dumps(genres) if genres else ""
bitrate = 0
try:
from mutagen import File as MutagenFile
audio = MutagenFile(final_path)
if audio and hasattr(audio, "info") and audio.info and hasattr(audio.info, "bitrate"):
bitrate = int(audio.info.bitrate / 1000) if audio.info.bitrate else 0
except Exception:
pass
artist_id = _stable_soulsync_id(artist_name.lower().strip())
album_id = _stable_soulsync_id(f"{artist_name}::{album_name}".lower().strip())
track_id = _stable_soulsync_id(final_path)
total_tracks = album_ctx.get("total_tracks", 0) or 0
db = get_database()
with db._get_connection() as conn:
cursor = conn.cursor()
cursor.execute("SELECT id FROM artists WHERE id = ? AND server_source = 'soulsync'", (artist_id,))
if not cursor.fetchone():
cursor.execute(
"SELECT id FROM artists WHERE name COLLATE NOCASE = ? AND server_source = 'soulsync' LIMIT 1",
(artist_name,),
)
existing_by_name = cursor.fetchone()
if existing_by_name:
artist_id = existing_by_name[0]
else:
cursor.execute("SELECT id FROM artists WHERE id = ?", (artist_id,))
if cursor.fetchone():
artist_id = _stable_soulsync_id(artist_name.lower().strip() + "::soulsync")
cursor.execute(
"""
INSERT INTO artists (id, name, genres, thumb_url, server_source, created_at, updated_at)
VALUES (?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
""",
(artist_id, artist_name, genres_json, image_url),
)
artist_source_col = source_columns.get("artist")
if artist_source_col and artist_source_id:
try:
cursor.execute(
f"UPDATE artists SET {artist_source_col} = ? WHERE id = ?",
(artist_source_id, artist_id),
)
except Exception:
pass
cursor.execute("SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'", (album_id,))
if not cursor.fetchone():
cursor.execute(
"SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? AND server_source = 'soulsync' LIMIT 1",
(album_name, artist_id),
)
existing_album_by_name = cursor.fetchone()
if existing_album_by_name:
album_id = existing_album_by_name[0]
else:
cursor.execute("SELECT id FROM albums WHERE id = ?", (album_id,))
if cursor.fetchone():
album_id = _stable_soulsync_id(f"{artist_name}::{album_name}::soulsync".lower().strip())
cursor.execute(
"""
INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, track_count,
duration, server_source, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
""",
(album_id, artist_id, album_name, year, image_url, genres_json, total_tracks, duration_ms),
)
album_source_col = source_columns.get("album")
if album_source_col and album_source_id:
try:
cursor.execute(
f"UPDATE albums SET {album_source_col} = ? WHERE id = ?",
(album_source_id, album_id),
)
except Exception:
pass
track_artist = None
track_artists_list = track_info.get("artists", []) or original_search.get("artists", [])
if track_artists_list:
first_track_artist = track_artists_list[0]
if isinstance(first_track_artist, dict):
ta_name = first_track_artist.get("name", "")
else:
ta_name = str(first_track_artist)
if ta_name and ta_name.lower() != artist_name.lower():
track_artist = ta_name
cursor.execute("SELECT id FROM tracks WHERE file_path = ?", (final_path,))
if not cursor.fetchone():
cursor.execute(
"""
INSERT INTO tracks (id, album_id, artist_id, title, track_number,
duration, file_path, bitrate, track_artist, server_source,
created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
""",
(
track_id,
album_id,
artist_id,
track_name,
track_number,
duration_ms,
final_path,
bitrate,
track_artist,
),
)
track_source_col = source_columns.get("track")
if track_source_col and track_source_id:
try:
cursor.execute(
f"UPDATE tracks SET {track_source_col} = ? WHERE id = ?",
(track_source_id, track_id),
)
track_album_col = source_columns.get("track_album")
if track_album_col and album_source_id:
cursor.execute(
f"UPDATE tracks SET {track_album_col} = ? WHERE id = ?",
(album_source_id, track_id),
)
except Exception:
pass
conn.commit()
logger.info("[SoulSync Library] Added: %s / %s / %s", artist_name, album_name, track_name)
except Exception as exc:
logger.error("[SoulSync Library] Could not record library entry: %s", exc)
def record_retag_download(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any], final_path: str) -> None:
"""Record a completed download for later re-tagging."""
try:
db = get_database()
context = normalize_import_context(context)
artist_context = get_import_context_artist(context) or (artist_context if isinstance(artist_context, dict) else {})
album_context = get_import_context_album(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
source = get_import_source(context)
source_ids = get_import_source_ids(context)
artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="Unknown Artist")
is_album = album_info and album_info.get("is_album", False)
group_type = "album" if is_album else "single"
album_name = album_info.get("album_name", "") if album_info else get_import_clean_album(context, default=original_search.get("album", "Unknown"))
image_url = album_info.get("album_image_url") if album_info else None
if not image_url:
image_url = album_context.get("image_url", "")
if not image_url and album_context.get("images"):
images = album_context.get("images", [])
if images and isinstance(images[0], dict):
image_url = images[0].get("url", "")
total_tracks = album_context.get("total_tracks", 1) if album_context else 1
release_date = album_context.get("release_date", "") if album_context else ""
spotify_album_id = None
itunes_album_id = None
if source == "spotify":
spotify_album_id = source_ids.get("album_id", "") or None
elif source == "itunes":
itunes_album_id = source_ids.get("album_id", "") or None
group_id = db.find_retag_group(artist_name, album_name)
if group_id is None:
group_id = db.add_retag_group(
group_type=group_type,
artist_name=artist_name,
album_name=album_name,
image_url=image_url,
spotify_album_id=spotify_album_id,
itunes_album_id=itunes_album_id,
total_tracks=total_tracks,
release_date=release_date,
)
if group_id is None:
return
track_number = album_info.get("track_number", 1) if album_info else (track_info.get("track_number", 1) or 1)
disc_number = original_search.get("disc_number") or (album_info.get("disc_number", 1) if album_info else track_info.get("disc_number", 1) or 1)
title = get_import_clean_title(
context,
album_info=album_info,
default=album_info.get("clean_track_name", "Unknown Track") if album_info else "Unknown Track",
)
file_format = os.path.splitext(str(final_path))[1].lstrip(".").lower()
source_track_id = None
itunes_track_id = None
if source == "spotify":
source_track_id = source_ids.get("track_id", "") or None
elif source == "itunes":
itunes_track_id = source_ids.get("track_id", "") or None
if not db.retag_track_exists(group_id, str(final_path)):
db.add_retag_track(
group_id=group_id,
track_number=track_number,
disc_number=disc_number,
title=title,
file_path=str(final_path),
file_format=file_format,
spotify_track_id=source_track_id,
itunes_track_id=itunes_track_id,
)
logger.info("[Retag] Recorded track for retag: '%s' in '%s'", title, album_name)
db.trim_retag_groups(100)
except Exception as exc:
logger.error("[Retag] Could not record track for retag: %s", exc)
def check_and_remove_from_wishlist(context: Dict[str, Any]) -> None:
"""Check whether a successful download should be removed from the wishlist."""
try:
wishlist_service = get_wishlist_service()
source = get_import_source(context)
source_ids = get_import_source_ids(context)
source_label = {
"spotify": "Spotify",
"itunes": "iTunes",
"deezer": "Deezer",
"discogs": "Discogs",
"hydrabase": "Hydrabase",
}.get(source, "Source")
track_info = get_import_track_info(context) or get_import_search_result(context)
search_result = get_import_original_search(context) or get_import_search_result(context)
track_id = None
if source == "spotify":
track_id = source_ids.get("track_id") or None
if track_id:
logger.info("[Wishlist] Found %s track ID from source_ids: %s", source_label, track_id)
elif "wishlist_id" in track_info:
wishlist_id = track_info["wishlist_id"]
logger.info("[Wishlist] Found wishlist_id in context: %s", wishlist_id)
wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service)
for wishlist_track in wishlist_tracks:
if wishlist_track.get("wishlist_id") == wishlist_id:
track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id")
logger.info("[Wishlist] Found track ID from wishlist entry: %s", track_id)
break
if not track_id:
track_name = track_info.get("name") or search_result.get("title", "")
artist_name = _primary_track_artist_name(track_info) or _primary_track_artist_name(search_result)
if track_name and artist_name:
logger.warning("[Wishlist] No track ID found, checking for fuzzy match: '%s' by '%s'", track_name, artist_name)
wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service)
for wishlist_track in wishlist_tracks:
wl_name = wishlist_track.get("name", "").lower()
wl_artists = wishlist_track.get("artists", [])
wl_artist_name = ""
if wl_artists:
if isinstance(wl_artists[0], dict):
wl_artist_name = wl_artists[0].get("name", "").lower()
else:
wl_artist_name = str(wl_artists[0]).lower()
if wl_name == track_name.lower() and wl_artist_name == artist_name.lower():
track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id")
logger.info("[Wishlist] Found fuzzy match - track ID: %s", track_id)
break
if track_id:
logger.info("[Wishlist] Attempting to remove track from wishlist: %s", track_id)
removed = wishlist_service.mark_track_download_result(track_id, success=True)
if removed:
logger.info("[Wishlist] Successfully removed track from wishlist: %s", track_id)
else:
logger.warning(" [Wishlist] Track not found in wishlist or already removed: %s", track_id)
else:
logger.warning(" [Wishlist] No track ID found for wishlist removal check")
except Exception as exc:
logger.error("[Wishlist] Error in wishlist removal check: %s", exc)

556
core/imports/staging.py Normal file
View file

@ -0,0 +1,556 @@
"""Shared staging folder and import suggestion helpers."""
from __future__ import annotations
import os
import threading
from typing import Any, Dict, Iterable, List, Optional, Tuple
from core.imports.paths import docker_resolve_path
from core.imports.filename import extract_track_number_from_filename
from utils.logging_config import get_logger
logger = get_logger("imports.staging")
AUDIO_EXTENSIONS = {".mp3", ".flac", ".ogg", ".opus", ".m4a", ".aac", ".wav", ".wma", ".aiff", ".aif", ".ape"}
_import_suggestions_cache_lock = threading.Lock()
_import_suggestions_cache: Dict[str, Any] = {
"suggestions": [],
"building": False,
"built": False,
}
def _get_config_manager():
try:
from config.settings import config_manager
return config_manager
except Exception:
class _FallbackConfig:
@staticmethod
def get(key, default=None):
return default
return _FallbackConfig()
def get_staging_path() -> str:
"""Resolve the configured staging folder path."""
raw = _get_config_manager().get("import.staging_path", "./Staging")
return docker_resolve_path(raw)
def get_import_suggestions_cache() -> Dict[str, Any]:
"""Expose the shared import suggestions cache."""
return _import_suggestions_cache
def get_primary_source() -> str:
from core.metadata_service import get_primary_source as _get_primary_source
return _get_primary_source()
def get_source_priority(preferred_source: str):
from core.metadata_service import get_source_priority as _get_source_priority
return _get_source_priority(preferred_source)
def get_client_for_source(source: str):
from core.metadata_service import get_client_for_source as _get_client_for_source
return _get_client_for_source(source)
def read_staging_file_metadata(file_path: str, filename: Optional[str] = None) -> Dict[str, Any]:
"""Read common audio tag metadata from a staging file."""
try:
from mutagen import File as MutagenFile
tags = MutagenFile(file_path, easy=True)
except Exception:
tags = None
filename = filename or os.path.basename(file_path)
stem = os.path.splitext(os.path.basename(filename))[0]
def _first_tag(*keys: str) -> str:
if not tags:
return ""
for key in keys:
try:
value = tags.get(key) # type: ignore[attr-defined]
except Exception:
value = None
if value:
if isinstance(value, (list, tuple)):
value = value[0] if value else ""
text = str(value).strip()
if text:
return text
return ""
title = _first_tag("title")
artist = _first_tag("artist")
albumartist = _first_tag("albumartist")
album = _first_tag("album")
if not title:
title = stem
if not albumartist:
albumartist = artist
track_number = extract_track_number_from_filename(filename or file_path)
try:
# Preserve tag-based numbers when present, but still fall back to the filename parser.
tag_track_number = _first_tag("tracknumber", "track_number")
if tag_track_number:
track_number = int(str(tag_track_number).split("/")[0].strip() or track_number)
except (TypeError, ValueError):
pass
disc_number = 1
try:
tag_disc_number = _first_tag("discnumber", "disc_number")
if tag_disc_number:
disc_number = int(str(tag_disc_number).split("/")[0].strip() or 1)
except (TypeError, ValueError):
pass
return {
"title": title,
"artist": artist,
"albumartist": albumartist,
"album": album,
"track_number": track_number,
"disc_number": disc_number,
}
def _search_albums_for_source(source: str, client: Any, query: str, limit: int = 5):
from core.metadata_service import _search_albums_for_source as _metadata_search_albums_for_source
return _metadata_search_albums_for_source(source, client, query, limit=limit)
def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 5):
from core.imports.resolution import search_tracks_for_source
return search_tracks_for_source(source, client, query, limit=limit)
def _extract_value(value: Any, *names: str, default: Any = None) -> Any:
if value is None:
return default
if isinstance(value, (str, bytes)):
return default
for name in names:
if isinstance(value, dict):
if name in value and value[name] is not None:
return value[name]
else:
candidate = getattr(value, name, None)
if candidate is not None:
return candidate
return default
def _extract_artist_names(artists: Any) -> List[str]:
if not artists:
return []
if isinstance(artists, (str, bytes)):
artist = str(artists).strip()
return [artist] if artist else []
try:
items = list(artists)
except TypeError:
items = [artists]
names: List[str] = []
for artist in items:
if isinstance(artist, dict):
name = str(_extract_value(artist, "name", "artist_name", "title", default="") or "").strip()
else:
candidate = getattr(artist, "name", None)
if candidate is None:
candidate = artist
name = str(candidate or "").strip()
if name:
names.append(name)
return names
def _normalize_album_result(album: Any, source: str) -> Dict[str, Any]:
album_id = str(_extract_value(album, "id", "album_id", "release_id", default="") or "").strip()
album_name = str(_extract_value(album, "name", "title", default="") or "").strip()
artists = _extract_artist_names(_extract_value(album, "artists", default=[]))
artist_name = ", ".join(artists) if artists else str(
_extract_value(album, "artist_name", "artist", default="Unknown Artist") or "Unknown Artist"
).strip()
release_date = str(_extract_value(album, "release_date", "releaseDate", default="") or "").strip()
album_type = str(_extract_value(album, "album_type", "type", default="album") or "album").strip() or "album"
total_tracks = _extract_value(album, "total_tracks", "track_count", default=0)
if isinstance(total_tracks, (list, tuple, set)):
total_tracks = len(total_tracks)
try:
total_tracks = int(total_tracks or 0)
except (TypeError, ValueError):
total_tracks = 0
image_url = _extract_value(album, "image_url", "thumb_url", "cover_image", "cover_url", default="")
if not image_url:
images = _extract_value(album, "images", default=[]) or []
if isinstance(images, dict):
images = [images]
elif isinstance(images, (str, bytes)):
images = [images]
try:
images = list(images)
except TypeError:
images = [images]
if images:
first_image = images[0]
if isinstance(first_image, (str, bytes)):
image_url = str(first_image).strip()
else:
image_url = _extract_value(first_image, "url", "image_url", "src", default="")
return {
"id": album_id or album_name or "unknown-album",
"name": album_name or album_id or "Unknown Album",
"artist": artist_name or "Unknown Artist",
"release_date": release_date,
"total_tracks": total_tracks,
"image_url": str(image_url or ""),
"album_type": album_type,
"source": source,
}
def _album_fingerprint(album: Dict[str, Any]) -> Tuple[str, str, str, str]:
return (
str(album.get("name", "") or "").strip().casefold(),
str(album.get("artist", "") or "").strip().casefold(),
str(album.get("release_date", "") or "").strip()[:10].casefold(),
str(album.get("album_type", "") or "").strip().casefold(),
)
def _normalize_track_result(track: Any, source: str) -> Dict[str, Any]:
track_id = str(_extract_value(track, "id", "track_id", "trackId", default="") or "").strip()
track_name = str(_extract_value(track, "name", "title", "track_name", default="") or "").strip()
artists = _extract_artist_names(_extract_value(track, "artists", default=[]))
artist_name = ", ".join(artists) if artists else str(
_extract_value(track, "artist", "artist_name", default="Unknown Artist") or "Unknown Artist"
).strip()
album_value = _extract_value(track, "album", default=None)
album_name = ""
album_id = str(_extract_value(track, "album_id", "collectionId", "albumId", default="") or "").strip()
if isinstance(album_value, dict):
album_name = str(_extract_value(album_value, "name", "title", default="") or "").strip()
album_id = album_id or str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip()
if not album_name:
album_name = album_id
elif isinstance(album_value, (str, bytes)):
album_name = str(album_value).strip()
elif album_value is not None:
album_name = str(_extract_value(album_value, "name", "title", default=album_value) or "").strip()
if not album_id:
album_id = str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip()
image_url = _extract_value(track, "image_url", "thumb_url", "cover_image", default="")
if not image_url:
images = _extract_value(track, "images", default=[]) or []
if isinstance(images, dict):
images = [images]
elif isinstance(images, (str, bytes)):
images = [images]
try:
images = list(images)
except TypeError:
images = [images]
if images:
first_image = images[0]
if isinstance(first_image, (str, bytes)):
image_url = str(first_image).strip()
else:
image_url = _extract_value(first_image, "url", "image_url", "src", default="")
if not image_url and album_value is not None:
album_images = _extract_value(album_value, "images", default=[]) or []
if isinstance(album_images, dict):
album_images = [album_images]
elif isinstance(album_images, (str, bytes)):
album_images = [album_images]
try:
album_images = list(album_images)
except TypeError:
album_images = [album_images]
if album_images:
first_album_image = album_images[0]
if isinstance(first_album_image, (str, bytes)):
image_url = str(first_album_image).strip()
else:
image_url = _extract_value(first_album_image, "url", "image_url", "src", default="")
duration_ms = _extract_value(track, "duration_ms", "duration", "trackTimeMillis", default=0)
try:
duration_ms = int(duration_ms or 0)
except (TypeError, ValueError):
duration_ms = 0
track_number = _extract_value(track, "track_number", "trackNumber", default=1)
try:
track_number = int(track_number or 1)
except (TypeError, ValueError):
track_number = 1
return {
"id": track_id or track_name or "unknown-track",
"name": track_name or track_id or "Unknown Track",
"artist": artist_name or "Unknown Artist",
"album": album_name or "",
"album_id": album_id or "",
"duration_ms": duration_ms,
"image_url": str(image_url or ""),
"track_number": track_number,
"source": source,
}
def _read_staging_audio_tags(file_path: str) -> Tuple[Optional[str], Optional[str]]:
try:
from mutagen import File as MutagenFile
tags = MutagenFile(file_path, easy=True)
if not tags:
return None, None
album = (tags.get("album") or [None])[0]
artist = (tags.get("artist") or (tags.get("albumartist") or [None]))[0]
album_text = str(album).strip() if album else ""
artist_text = str(artist).strip() if artist else ""
return (album_text or None, artist_text or None)
except Exception:
return None, None
def _collect_import_suggestion_queries(staging_path: str) -> List[str]:
tag_albums: Dict[Tuple[str, str], int] = {}
folder_hints: Dict[str, int] = {}
for root, _dirs, filenames in os.walk(staging_path):
audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS]
if not audio_files:
continue
rel_dir = os.path.relpath(root, staging_path)
if rel_dir != ".":
top_folder = rel_dir.split(os.sep)[0]
folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files)
for fname in audio_files:
full_path = os.path.join(root, fname)
album, artist = _read_staging_audio_tags(full_path)
if album:
key = (album.strip(), (artist or "").strip())
tag_albums[key] = tag_albums.get(key, 0) + 1
queries: List[str] = []
seen_lower = set()
for (album, artist), _count in sorted(tag_albums.items(), key=lambda item: -item[1]):
q = f"{album} {artist}".strip() if artist else album
if q and q.lower() not in seen_lower:
seen_lower.add(q.lower())
queries.append(q)
for folder, _count in sorted(folder_hints.items(), key=lambda item: -item[1]):
q = folder.replace("_", " ")
if q and q.lower() not in seen_lower:
seen_lower.add(q.lower())
queries.append(q)
return queries[:5]
def search_import_albums(query: str, limit: int = 12) -> List[Dict[str, Any]]:
"""Search albums using the configured metadata provider first."""
query = (query or "").strip()
if not query:
return []
results: List[Dict[str, Any]] = []
seen = set()
source_chain = get_source_priority(get_primary_source())
for source in source_chain:
client = get_client_for_source(source)
if not client:
continue
source_results = _search_albums_for_source(source, client, query, limit=limit)
if not source_results:
continue
added_for_source = False
for album in source_results:
suggestion = _normalize_album_result(album, source)
fingerprint = _album_fingerprint(suggestion)
if fingerprint in seen:
continue
seen.add(fingerprint)
results.append(suggestion)
added_for_source = True
if len(results) >= limit:
return results[:limit]
if added_for_source:
break
return results[:limit]
def search_import_tracks(query: str, limit: int = 30) -> List[Dict[str, Any]]:
"""Search tracks using the configured metadata provider priority order."""
query = (query or "").strip()
if not query:
return []
results: List[Dict[str, Any]] = []
source_chain = get_source_priority(get_primary_source())
for source in source_chain:
client = get_client_for_source(source)
if not client:
continue
source_results = _search_tracks_for_source(source, client, query, limit=limit)
if not source_results:
continue
for track in source_results:
results.append(_normalize_track_result(track, source))
if len(results) >= limit:
return results[:limit]
break
return results[:limit]
def _build_import_suggestions_background():
cache = _import_suggestions_cache
with _import_suggestions_cache_lock:
if cache["building"]:
return
cache["building"] = True
try:
staging_path = get_staging_path()
if not os.path.isdir(staging_path):
with _import_suggestions_cache_lock:
cache["suggestions"] = []
cache["built"] = True
return
queries = _collect_import_suggestion_queries(staging_path)
if not queries:
with _import_suggestions_cache_lock:
cache["suggestions"] = []
cache["built"] = True
return
suggestions: List[Dict[str, Any]] = []
seen = set()
for query in queries:
try:
albums = search_import_albums(query, limit=2)
for album in albums:
fingerprint = _album_fingerprint(album)
if fingerprint in seen:
continue
seen.add(fingerprint)
suggestions.append(album)
except Exception as exc:
logger.warning("Import suggestion search failed for %r: %s", query, exc)
with _import_suggestions_cache_lock:
cache["suggestions"] = suggestions[:8]
cache["built"] = True
logger.info(
"Import suggestions cache built: %s suggestions from %s hints",
len(cache["suggestions"]),
len(queries),
)
except Exception as exc:
logger.error("Error building import suggestions cache: %s", exc)
with _import_suggestions_cache_lock:
cache["suggestions"] = []
cache["built"] = True
finally:
with _import_suggestions_cache_lock:
cache["building"] = False
def start_import_suggestions_cache():
"""Start building the import suggestions cache in a background thread."""
threading.Thread(
target=_build_import_suggestions_background,
daemon=True,
name="import-suggestions-cache",
).start()
def refresh_import_suggestions_cache():
"""Invalidate and rebuild the suggestions cache."""
with _import_suggestions_cache_lock:
_import_suggestions_cache["built"] = False
start_import_suggestions_cache()
def collect_staging_files(file_paths: Optional[Iterable[str]] = None) -> List[Dict[str, Any]]:
"""Collect audio files from the staging area with normalized metadata."""
staging_path = get_staging_path()
file_filter: Optional[set[str]] = set(file_paths) if file_paths else None
staging_files: List[Dict[str, Any]] = []
if not os.path.isdir(staging_path):
return staging_files
for root, _dirs, filenames in os.walk(staging_path):
for filename in filenames:
ext = os.path.splitext(filename)[1].lower()
if ext not in AUDIO_EXTENSIONS:
continue
full_path = os.path.join(root, filename)
if file_filter is not None and full_path not in file_filter:
continue
meta = read_staging_file_metadata(full_path, filename)
staging_files.append(
{
"filename": filename,
"full_path": full_path,
"title": meta.get("title", ""),
"artist": meta.get("albumartist") or meta.get("artist") or "",
"album": meta.get("album", ""),
"albumartist": meta.get("albumartist") or meta.get("artist") or "",
"track_number": meta.get("track_number", 1),
"disc_number": meta.get("disc_number", 1),
}
)
return staging_files

View file

@ -0,0 +1,2 @@
"""Metadata helper package."""

157
core/metadata/artwork.py Normal file
View file

@ -0,0 +1,157 @@
"""Album artwork helpers for metadata enrichment."""
from __future__ import annotations
import os
import re
import urllib.request
from core.imports.context import get_import_context_album
from core.metadata.common import (
get_config_manager,
get_image_dimensions,
get_mutagen_symbols,
)
from utils.logging_config import get_logger as _create_logger
__all__ = [
"embed_album_art_metadata",
"download_cover_art",
]
logger = _create_logger("metadata.artwork")
def embed_album_art_metadata(audio_file, metadata: dict):
cfg = get_config_manager()
symbols = get_mutagen_symbols()
if not symbols:
return
try:
image_data = None
mime_type = None
release_mbid = metadata.get("musicbrainz_release_id")
if release_mbid and cfg.get("metadata_enhancement.prefer_caa_art", False):
try:
caa_url = f"https://coverartarchive.org/release/{release_mbid}/front"
req = urllib.request.Request(caa_url, headers={"Accept": "image/*"})
with urllib.request.urlopen(req, timeout=10) as response:
image_data = response.read()
mime_type = response.info().get_content_type() or "image/jpeg"
if not image_data or len(image_data) <= 1000:
image_data = None
except Exception:
image_data = None
if not image_data:
art_url = metadata.get("album_art_url")
if not art_url:
logger.warning("No album art URL available for embedding.")
return
with urllib.request.urlopen(art_url, timeout=10) as response:
image_data = response.read()
mime_type = response.info().get_content_type() or "image/jpeg"
if not image_data:
logger.error("Failed to download album art data.")
return
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.APIC(encoding=3, mime=mime_type, type=3, desc="Cover", data=image_data))
elif isinstance(audio_file, symbols.FLAC):
picture = symbols.Picture()
picture.data = image_data
picture.type = 3
picture.mime = mime_type
width, height = get_image_dimensions(image_data)
picture.width = width or 640
picture.height = height or 640
picture.depth = 24
audio_file.add_picture(picture)
elif isinstance(audio_file, symbols.MP4):
fmt = symbols.MP4Cover.FORMAT_JPEG if "jpeg" in mime_type else symbols.MP4Cover.FORMAT_PNG
audio_file["covr"] = [symbols.MP4Cover(image_data, imageformat=fmt)]
logger.info("Album art successfully embedded.")
except Exception as exc:
logger.error("Error embedding album art: %s", exc)
def download_cover_art(album_info: dict, target_dir: str, context: dict = None):
cfg = get_config_manager()
if cfg.get("metadata_enhancement.cover_art_download", True) is False:
return
try:
cover_path = os.path.join(target_dir, "cover.jpg")
album_info = album_info or {}
release_mbid = album_info.get("musicbrainz_release_id")
prefer_caa = cfg.get("metadata_enhancement.prefer_caa_art", False)
if os.path.exists(cover_path):
if release_mbid and prefer_caa:
try:
existing_size = os.path.getsize(cover_path)
if existing_size > 200_000:
return
is_upgrade = True
except Exception:
return
else:
return
else:
is_upgrade = False
image_data = None
if release_mbid and prefer_caa:
try:
caa_url = f"https://coverartarchive.org/release/{release_mbid}/front"
req = urllib.request.Request(caa_url, headers={"Accept": "image/*"})
with urllib.request.urlopen(req, timeout=10) as response:
image_data = response.read()
if not image_data or len(image_data) <= 1000:
image_data = None
except Exception:
image_data = None
if is_upgrade and not image_data:
logger.error("CAA upgrade failed - keeping existing cover.jpg")
return
if not image_data:
art_url = album_info.get("album_image_url")
if not art_url and context:
album_ctx = get_import_context_album(context)
art_url = album_ctx.get("image_url")
if not art_url and album_ctx.get("images"):
images = album_ctx.get("images", [])
if images and isinstance(images[0], dict):
art_url = images[0].get("url", "")
if art_url:
logger.info("Using cover art URL from album context")
if art_url and "i.scdn.co" in art_url:
try:
from core.spotify_client import _upgrade_spotify_image_url
art_url = _upgrade_spotify_image_url(art_url)
except Exception:
pass
elif art_url and "mzstatic.com" in art_url:
art_url = re.sub(r"\d+x\d+bb", "3000x3000bb", art_url)
if not art_url:
logger.warning("No cover art URL available for download.")
return
with urllib.request.urlopen(art_url, timeout=10) as response:
image_data = response.read()
if not image_data:
return
with open(cover_path, "wb") as handle:
handle.write(image_data)
logger.info("Cover art downloaded to: %s", cover_path)
except Exception as exc:
logger.error("Error downloading cover.jpg: %s", exc)

267
core/metadata/common.py Normal file
View file

@ -0,0 +1,267 @@
"""Shared low-level helpers for metadata enrichment."""
from __future__ import annotations
import os
import threading
import weakref
from types import SimpleNamespace
from typing import Any
from utils.logging_config import get_logger as _create_logger
logger = _create_logger("metadata.common")
__all__ = [
"get_logger",
"get_config_manager",
"get_mutagen_symbols",
"get_file_lock",
"is_ogg_opus",
"is_vorbis_like",
"save_audio_file",
"get_image_dimensions",
"strip_all_non_audio_tags",
"verify_metadata_written",
"wipe_source_tags",
]
_FILE_LOCKS: "weakref.WeakValueDictionary[str, threading.Lock]" = weakref.WeakValueDictionary()
_FILE_LOCKS_LOCK = threading.Lock()
class _NullConfigManager:
def get(self, _key: str, default: Any = None) -> Any:
return default
def get_logger():
return logger
def get_config_manager():
try:
from config.settings import config_manager as settings_config_manager
return settings_config_manager
except Exception:
return _NullConfigManager()
def get_mutagen_symbols():
"""Lazy mutagen import so tests can monkeypatch this without the package installed."""
try:
from mutagen import File as MutagenFile
from mutagen.apev2 import APEv2, APENoHeaderError
from mutagen.flac import FLAC, Picture
from mutagen.id3 import (
APIC,
ID3,
TBPM,
TCOP,
TDOR,
TDRC,
TCON,
TIT2,
TALB,
TPE1,
TPE2,
TPOS,
TPUB,
TRCK,
TSRC,
TXXX,
UFID,
TMED,
)
from mutagen.mp4 import MP4, MP4Cover, MP4FreeForm
from mutagen.oggvorbis import OggVorbis
try:
from mutagen.oggopus import OggOpus
except Exception:
OggOpus = None
except Exception as exc:
logger.debug("Mutagen unavailable for metadata enrichment: %s", exc)
return None
return SimpleNamespace(
File=MutagenFile,
APEv2=APEv2,
APENoHeaderError=APENoHeaderError,
FLAC=FLAC,
Picture=Picture,
ID3=ID3,
APIC=APIC,
TBPM=TBPM,
TCOP=TCOP,
TDOR=TDOR,
TDRC=TDRC,
TCON=TCON,
TIT2=TIT2,
TALB=TALB,
TPE1=TPE1,
TPE2=TPE2,
TPOS=TPOS,
TPUB=TPUB,
TRCK=TRCK,
TSRC=TSRC,
TXXX=TXXX,
UFID=UFID,
TMED=TMED,
MP4=MP4,
MP4Cover=MP4Cover,
MP4FreeForm=MP4FreeForm,
OggVorbis=OggVorbis,
OggOpus=OggOpus,
)
def get_file_lock(file_path: str) -> threading.Lock:
# Keep a per-path lock while it is actively referenced, but let it
# fall out of the cache once nobody is using it anymore.
with _FILE_LOCKS_LOCK:
lock = _FILE_LOCKS.get(file_path)
if lock is None:
lock = threading.Lock()
_FILE_LOCKS[file_path] = lock
return lock
def is_ogg_opus(audio_file: Any) -> bool:
return type(audio_file).__name__ == "OggOpus"
def is_vorbis_like(audio_file: Any, symbols: Any) -> bool:
vorbis_classes = tuple(
cls for cls in (
getattr(symbols, "FLAC", None),
getattr(symbols, "OggVorbis", None),
) if cls is not None
)
return bool(vorbis_classes) and isinstance(audio_file, vorbis_classes) or is_ogg_opus(audio_file)
def save_audio_file(audio_file: Any, symbols: Any) -> None:
if isinstance(audio_file.tags, symbols.ID3):
audio_file.save(v1=0, v2_version=4)
elif isinstance(audio_file, symbols.FLAC):
audio_file.save(deleteid3=True)
else:
audio_file.save()
def get_image_dimensions(data: bytes):
try:
if data[:8] == b"\x89PNG\r\n\x1a\n":
import struct
w, h = struct.unpack(">II", data[16:24])
return w, h
if data[:2] == b"\xff\xd8":
import struct
i = 2
while i < len(data) - 9:
if data[i] != 0xFF:
break
marker = data[i + 1]
if marker in (0xC0, 0xC2):
h, w = struct.unpack(">HH", data[i + 5 : i + 9])
return w, h
length = struct.unpack(">H", data[i + 2 : i + 4])[0]
i += 2 + length
except Exception:
pass
return None, None
def strip_all_non_audio_tags(file_path: str) -> dict:
summary = {"apev2_stripped": False, "apev2_tag_count": 0}
if os.path.splitext(file_path)[1].lower() != ".mp3":
return summary
symbols = get_mutagen_symbols()
if not symbols:
return summary
try:
apev2_tags = symbols.APEv2(file_path)
tag_count = len(apev2_tags)
tag_keys = list(apev2_tags.keys())
apev2_tags.delete(file_path)
summary["apev2_stripped"] = True
summary["apev2_tag_count"] = tag_count
logger.info("Stripped %s APEv2 tags: %s", tag_count, ", ".join(tag_keys[:10]))
except symbols.APENoHeaderError:
pass
except Exception as exc:
logger.error("Could not strip APEv2 tags (non-fatal): %s", exc)
return summary
def verify_metadata_written(file_path: str) -> bool:
symbols = get_mutagen_symbols()
if not symbols:
return False
try:
check = symbols.File(file_path)
if check is None or check.tags is None:
logger.info("[VERIFY] Tags are None after save: %s", file_path)
return False
title_found = False
artist_found = False
if isinstance(check.tags, symbols.ID3):
title_found = bool(check.tags.getall("TIT2"))
artist_found = bool(check.tags.getall("TPE1"))
try:
symbols.APEv2(file_path)
logger.info("[VERIFY] APEv2 tags still present after processing!")
return False
except symbols.APENoHeaderError:
pass
elif is_vorbis_like(check, symbols):
title_found = bool(check.get("title"))
artist_found = bool(check.get("artist"))
elif isinstance(check, symbols.MP4):
title_found = bool(check.get("\xa9nam"))
artist_found = bool(check.get("\xa9ART"))
if not title_found or not artist_found:
logger.warning("[VERIFY] Missing metadata - title:%s artist:%s", title_found, artist_found)
return False
logger.info("[VERIFY] Metadata verified OK")
return True
except Exception as exc:
logger.error("[VERIFY] Verification error (non-fatal): %s", exc)
return False
def wipe_source_tags(file_path: str) -> bool:
try:
strip_all_non_audio_tags(file_path)
symbols = get_mutagen_symbols()
if not symbols:
return False
audio = symbols.File(file_path)
if audio is None:
return False
if hasattr(audio, "clear_pictures"):
audio.clear_pictures()
if audio.tags is not None:
tag_count = len(audio.tags)
audio.tags.clear()
else:
audio.add_tags()
tag_count = 0
save_audio_file(audio, symbols)
if tag_count > 0:
logger.info("[Tag Wipe] Stripped %s source tags from: %s", tag_count, os.path.basename(file_path))
return True
except Exception as exc:
logger.error("[Tag Wipe] Failed (non-fatal): %s", exc)
return False

194
core/metadata/enrichment.py Normal file
View file

@ -0,0 +1,194 @@
"""Compatibility facade and orchestration for metadata enrichment."""
from __future__ import annotations
import os
from types import SimpleNamespace
from typing import Any
from core.metadata.artwork import embed_album_art_metadata
from core.metadata.common import (
get_config_manager,
get_file_lock,
get_mutagen_symbols,
is_vorbis_like,
save_audio_file,
strip_all_non_audio_tags,
verify_metadata_written,
)
from core.metadata.source import embed_source_ids, extract_source_metadata
from utils.logging_config import get_logger as _create_logger
__all__ = [
"build_metadata_enrichment_runtime",
"enhance_file_metadata",
"extract_source_metadata",
"embed_source_ids",
]
logger = _create_logger("metadata.enrichment")
def build_metadata_enrichment_runtime(
*,
mb_worker: Any | None = None,
deezer_worker: Any | None = None,
audiodb_worker: Any | None = None,
tidal_client: Any | None = None,
qobuz_enrichment_worker: Any | None = None,
lastfm_worker: Any | None = None,
genius_worker: Any | None = None,
spotify_enrichment_worker: Any | None = None,
itunes_enrichment_worker: Any | None = None,
) -> SimpleNamespace:
"""Build the runtime object consumed by core.metadata.enrichment/source."""
return SimpleNamespace(
mb_worker=mb_worker,
deezer_worker=deezer_worker,
audiodb_worker=audiodb_worker,
tidal_client=tidal_client,
qobuz_enrichment_worker=qobuz_enrichment_worker,
lastfm_worker=lastfm_worker,
genius_worker=genius_worker,
spotify_enrichment_worker=spotify_enrichment_worker,
itunes_enrichment_worker=itunes_enrichment_worker,
)
def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool:
cfg = get_config_manager()
if cfg.get("metadata_enhancement.enabled", True) is False:
logger.warning("Metadata enhancement disabled in config.")
return True
if album_info is None:
album_info = {}
symbols = get_mutagen_symbols()
if not symbols:
logger.error("Mutagen is unavailable, cannot enhance metadata.")
return False
file_lock = get_file_lock(file_path)
with file_lock:
logger.info("Enhancing metadata for: %s", os.path.basename(file_path))
try:
strip_all_non_audio_tags(file_path)
audio_file = symbols.File(file_path)
if audio_file is None:
logger.error("Could not load audio file with Mutagen: %s", file_path)
return False
if hasattr(audio_file, "clear_pictures"):
audio_file.clear_pictures()
if audio_file.tags is not None:
if len(audio_file.tags) > 0:
tag_keys = list(audio_file.tags.keys())[:15]
logger.info("Clearing %s existing tags: %s", len(audio_file.tags), ", ".join(str(k) for k in tag_keys))
audio_file.tags.clear()
else:
audio_file.add_tags()
save_audio_file(audio_file, symbols)
metadata = extract_source_metadata(context, artist, album_info)
if not metadata:
logger.error("Could not extract source metadata, saving with cleared tags.")
save_audio_file(audio_file, symbols)
return True
track_num_str = f"{metadata.get('track_number', 1)}/{metadata.get('total_tracks', 1)}"
write_multi = cfg.get("metadata_enhancement.tags.write_multi_artist", False)
artists_list = metadata.get("_artists_list", [])
if isinstance(audio_file.tags, symbols.ID3):
if metadata.get("title"):
audio_file.tags.add(symbols.TIT2(encoding=3, text=[metadata["title"]]))
if metadata.get("artist"):
audio_file.tags.add(symbols.TPE1(encoding=3, text=[metadata["artist"]]))
if write_multi and len(artists_list) > 1:
audio_file.tags.add(symbols.TPE1(encoding=3, text=artists_list))
if metadata.get("album_artist"):
audio_file.tags.add(symbols.TPE2(encoding=3, text=[metadata["album_artist"]]))
if metadata.get("album"):
audio_file.tags.add(symbols.TALB(encoding=3, text=[metadata["album"]]))
if metadata.get("date"):
audio_file.tags.add(symbols.TDRC(encoding=3, text=[metadata["date"]]))
if metadata.get("genre"):
audio_file.tags.add(symbols.TCON(encoding=3, text=[metadata["genre"]]))
audio_file.tags.add(symbols.TRCK(encoding=3, text=[track_num_str]))
if metadata.get("disc_number"):
audio_file.tags.add(symbols.TPOS(encoding=3, text=[str(metadata["disc_number"])]))
elif is_vorbis_like(audio_file, symbols):
if metadata.get("title"):
audio_file["title"] = [metadata["title"]]
if metadata.get("artist"):
audio_file["artist"] = [metadata["artist"]]
if write_multi and len(artists_list) > 1:
audio_file["artists"] = artists_list
if metadata.get("album_artist"):
audio_file["albumartist"] = [metadata["album_artist"]]
if metadata.get("album"):
audio_file["album"] = [metadata["album"]]
if metadata.get("date"):
audio_file["date"] = [metadata["date"]]
if metadata.get("genre"):
audio_file["genre"] = [metadata["genre"]]
audio_file["tracknumber"] = [track_num_str]
if metadata.get("disc_number"):
audio_file["discnumber"] = [str(metadata["disc_number"])]
elif isinstance(audio_file, symbols.MP4):
if metadata.get("title"):
audio_file["\xa9nam"] = [metadata["title"]]
if metadata.get("artist"):
audio_file["\xa9ART"] = artists_list if (write_multi and len(artists_list) > 1) else [metadata["artist"]]
if metadata.get("album_artist"):
audio_file["aART"] = [metadata["album_artist"]]
if metadata.get("album"):
audio_file["\xa9alb"] = [metadata["album"]]
if metadata.get("date"):
audio_file["\xa9day"] = [metadata["date"]]
if metadata.get("genre"):
audio_file["\xa9gen"] = [metadata["genre"]]
audio_file["trkn"] = [(metadata.get("track_number", 1), metadata.get("total_tracks", 1))]
if metadata.get("disc_number"):
audio_file["disk"] = [(metadata["disc_number"], 0)]
embed_source_ids(audio_file, metadata, context, runtime=runtime)
if album_info is not None and metadata.get("musicbrainz_release_id"):
album_info["musicbrainz_release_id"] = metadata["musicbrainz_release_id"]
if cfg.get("metadata_enhancement.embed_album_art", True):
embed_album_art_metadata(audio_file, metadata)
quality = context.get("_audio_quality", "")
if quality and cfg.get("metadata_enhancement.tags.quality_tag", True) is not False:
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.TXXX(encoding=3, desc="QUALITY", text=[quality]))
elif is_vorbis_like(audio_file, symbols):
audio_file["quality"] = [quality]
elif isinstance(audio_file, symbols.MP4):
audio_file["----:com.apple.iTunes:QUALITY"] = [symbols.MP4FreeForm(quality.encode("utf-8"))]
save_audio_file(audio_file, symbols)
verified = verify_metadata_written(file_path)
if verified:
logger.info("Metadata enhanced successfully.")
else:
logger.info("Metadata saved but verification found issues (see above).")
return True
except Exception as exc:
import traceback
logger.error("Error enhancing metadata for %s: %s", file_path, exc)
logger.error("[Metadata Debug] Exception type: %s", type(exc).__name__)
logger.info("[Metadata Debug] File exists: %s", os.path.exists(file_path))
logger.warning("[Metadata Debug] Artist: %s", artist.get("name", "MISSING") if artist else "None")
logger.warning("[Metadata Debug] Album info: %s", album_info.get("album_name", "MISSING") if album_info else "None")
logger.error("[Metadata Debug] Traceback:\n%s", traceback.format_exc())
return False

70
core/metadata/lyrics.py Normal file
View file

@ -0,0 +1,70 @@
"""Lyrics export helpers for metadata enrichment."""
from __future__ import annotations
from core.imports.context import (
get_import_clean_album,
get_import_clean_title,
get_import_context_album,
get_import_original_search,
normalize_import_context,
)
from core.metadata.common import get_config_manager
from utils.logging_config import get_logger as _create_logger
__all__ = [
"generate_lrc_file",
]
logger = _create_logger("metadata.lyrics")
def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool:
cfg = get_config_manager()
if cfg.get("metadata_enhancement.lrclib_enabled", True) is False:
return False
try:
from core.lyrics_client import lyrics_client
context = normalize_import_context(context)
original_search = get_import_original_search(context)
album_context = get_import_context_album(context)
track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track"))
if isinstance(artist, dict):
artist_name = artist.get("name", "Unknown Artist")
elif hasattr(artist, "name"):
artist_name = artist.name
else:
artist_name = str(artist) if artist else "Unknown Artist"
album_name = None
duration_seconds = None
if album_info and album_info.get("is_album"):
album_name = (
get_import_clean_album(context, album_info=album_info, default="")
or album_info.get("album_name")
or album_context.get("name")
)
if original_search.get("duration_ms"):
duration_seconds = int(original_search["duration_ms"] / 1000)
success = lyrics_client.create_lrc_file(
audio_file_path=file_path,
track_name=track_name,
artist_name=artist_name,
album_name=album_name,
duration_seconds=duration_seconds,
)
if success:
logger.info("LRC file generated for: %s", track_name)
else:
logger.warning("No lyrics found for: %s", track_name)
return success
except Exception as exc:
logger.error("Error generating LRC file for %s: %s", file_path, exc)
return False

974
core/metadata/source.py Normal file
View file

@ -0,0 +1,974 @@
"""Source metadata extraction and source-ID embedding helpers."""
from __future__ import annotations
import re
import socket
import threading
import time
from collections import OrderedDict
from typing import Any, Dict
import requests
from core.imports.context import (
extract_artist_name,
get_import_clean_artist,
get_import_clean_title,
get_import_context_album,
get_import_original_search,
get_import_source,
get_import_source_ids,
get_import_track_info,
get_source_tag_names,
normalize_import_context,
)
from core.metadata_service import get_itunes_client
from database.music_database import get_database
from core.metadata.common import (
get_config_manager,
get_mutagen_symbols,
is_vorbis_like,
)
from utils.logging_config import get_logger as _create_logger
__all__ = [
"extract_source_metadata",
"embed_source_ids",
"normalize_album_cache_key",
"mb_release_cache",
"mb_release_cache_lock",
"mb_release_detail_cache",
"mb_release_detail_cache_lock",
]
_MB_RELEASE_CACHE_MAX_ENTRIES = 4096
_MB_RELEASE_DETAIL_CACHE_MAX_ENTRIES = 4096
mb_release_cache: "OrderedDict[tuple, str]" = OrderedDict()
mb_release_cache_lock = threading.RLock()
mb_release_detail_cache: "OrderedDict[str, Dict[str, Any]]" = OrderedDict()
mb_release_detail_cache_lock = threading.RLock()
logger = _create_logger("metadata.source")
_SOURCE_NETWORK_EXCEPTIONS = (requests.RequestException, socket.timeout, TimeoutError)
_EDITION_PAREN_RE = re.compile(
r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|'
r'limited|bonus|platinum|gold|super\s*deluxe|standard)'
r'(?:\s+(?:edition|version))?[^)\]]*[\)\]]',
re.IGNORECASE,
)
_EDITION_BARE_RE = re.compile(
r'\s+(?:-\s+)?(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|'
r'limited|bonus|platinum|gold|super\s*deluxe|standard)'
r'(?:\s+(?:edition|version))?\s*$',
re.IGNORECASE,
)
def normalize_album_cache_key(album_name: str) -> str:
result = _EDITION_PAREN_RE.sub("", album_name or "")
result = _EDITION_BARE_RE.sub("", result)
return result.lower().strip()
def _bounded_cache_get(cache, key):
value = cache.get(key)
if value is not None and hasattr(cache, "move_to_end"):
cache.move_to_end(key)
return value
def _bounded_cache_set(cache, key, value, max_entries: int) -> None:
cache[key] = value
if hasattr(cache, "move_to_end"):
cache.move_to_end(key)
while len(cache) > max_entries:
cache.popitem(last=False)
def _call_source_lookup(label: str, func, *args, **kwargs):
try:
return func(*args, **kwargs)
except _SOURCE_NETWORK_EXCEPTIONS as exc:
logger.warning("%s lookup failed (network): %s", label, exc)
return None
SOURCE_TAG_CONFIG = {
"SPOTIFY_TRACK_ID": "spotify.tags.track_id",
"SPOTIFY_ARTIST_ID": "spotify.tags.artist_id",
"SPOTIFY_ALBUM_ID": "spotify.tags.album_id",
"ITUNES_TRACK_ID": "itunes.tags.track_id",
"ITUNES_ARTIST_ID": "itunes.tags.artist_id",
"ITUNES_ALBUM_ID": "itunes.tags.album_id",
"MUSICBRAINZ_RECORDING_ID": "musicbrainz.tags.recording_id",
"MUSICBRAINZ_ARTIST_ID": "musicbrainz.tags.artist_id",
"MUSICBRAINZ_RELEASE_ID": "musicbrainz.tags.release_id",
"MUSICBRAINZ_RELEASEGROUPID": "musicbrainz.tags.release_group_id",
"MUSICBRAINZ_ALBUMARTISTID": "musicbrainz.tags.album_artist_id",
"MUSICBRAINZ_RELEASETRACKID": "musicbrainz.tags.release_track_id",
"RELEASETYPE": "musicbrainz.tags.release_type",
"ORIGINALDATE": "musicbrainz.tags.original_date",
"RELEASESTATUS": "musicbrainz.tags.release_status",
"RELEASECOUNTRY": "musicbrainz.tags.release_country",
"BARCODE": "musicbrainz.tags.barcode",
"MEDIA": "musicbrainz.tags.media",
"TOTALDISCS": "musicbrainz.tags.total_discs",
"CATALOGNUMBER": "musicbrainz.tags.catalog_number",
"SCRIPT": "musicbrainz.tags.script",
"ASIN": "musicbrainz.tags.asin",
"DEEZER_TRACK_ID": "deezer.tags.track_id",
"DEEZER_ARTIST_ID": "deezer.tags.artist_id",
"AUDIODB_TRACK_ID": "audiodb.tags.track_id",
"TIDAL_TRACK_ID": "tidal.tags.track_id",
"TIDAL_ARTIST_ID": "tidal.tags.artist_id",
"QOBUZ_TRACK_ID": "qobuz.tags.track_id",
"QOBUZ_ARTIST_ID": "qobuz.tags.artist_id",
"GENIUS_TRACK_ID": "genius.tags.track_id",
}
DEFAULT_SOURCE_ORDER = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"]
ID3_TAG_MAP = {
"MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"),
"MUSICBRAINZ_ARTIST_ID": ("TXXX", "MusicBrainz Artist Id"),
"MUSICBRAINZ_RELEASE_ID": ("TXXX", "MusicBrainz Album Id"),
"MUSICBRAINZ_RELEASEGROUPID": ("TXXX", "MusicBrainz Release Group Id"),
"MUSICBRAINZ_ALBUMARTISTID": ("TXXX", "MusicBrainz Album Artist Id"),
"MUSICBRAINZ_RELEASETRACKID": ("TXXX", "MusicBrainz Release Track Id"),
"RELEASETYPE": ("TXXX", "MusicBrainz Album Type"),
"RELEASESTATUS": ("TXXX", "MusicBrainz Album Status"),
"RELEASECOUNTRY": ("TXXX", "MusicBrainz Album Release Country"),
"ORIGINALDATE": ("TDOR", None),
"MEDIA": ("TMED", None),
}
VORBIS_TAG_MAP = {
"MUSICBRAINZ_RECORDING_ID": "MUSICBRAINZ_TRACKID",
"MUSICBRAINZ_ARTIST_ID": "MUSICBRAINZ_ARTISTID",
"MUSICBRAINZ_RELEASE_ID": "MUSICBRAINZ_ALBUMID",
"MUSICBRAINZ_RELEASEGROUPID": "MUSICBRAINZ_RELEASEGROUPID",
"MUSICBRAINZ_ALBUMARTISTID": "MUSICBRAINZ_ALBUMARTISTID",
"MUSICBRAINZ_RELEASETRACKID": "MUSICBRAINZ_RELEASETRACKID",
}
MP4_TAG_MAP = {
"MUSICBRAINZ_RECORDING_ID": "MusicBrainz Track Id",
"MUSICBRAINZ_ARTIST_ID": "MusicBrainz Artist Id",
"MUSICBRAINZ_RELEASE_ID": "MusicBrainz Album Id",
"MUSICBRAINZ_RELEASEGROUPID": "MusicBrainz Release Group Id",
"MUSICBRAINZ_ALBUMARTISTID": "MusicBrainz Album Artist Id",
"MUSICBRAINZ_RELEASETRACKID": "MusicBrainz Release Track Id",
"RELEASETYPE": "MusicBrainz Album Type",
"RELEASESTATUS": "MusicBrainz Album Status",
"RELEASECOUNTRY": "MusicBrainz Album Release Country",
}
def _tag_enabled(cfg, path: str) -> bool:
return cfg.get(path, True) is not False
def _names_match(a: str, b: str, threshold: float = 0.75) -> bool:
if not a or not b:
return False
from difflib import SequenceMatcher
norm = lambda s: re.sub(r"[^a-z0-9 ]", "", re.sub(r"\(.*?\)", "", s).lower()).strip()
return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold
def _collect_source_ids(metadata: dict, cfg) -> dict:
source_ids = {}
source = (metadata.get("source") or "").strip().lower()
if source:
source_tag_names = get_source_tag_names(source)
source_track_id = metadata.get("source_track_id")
source_artist_id = metadata.get("source_artist_id")
source_album_id = metadata.get("source_album_id")
if cfg.get(f"{source}.embed_tags", True) is not False:
if source_tag_names.get("track") and source_track_id:
source_ids[source_tag_names["track"]] = source_track_id
if source_tag_names.get("artist") and source_artist_id:
source_ids[source_tag_names["artist"]] = source_artist_id
if source_tag_names.get("album") and source_album_id:
source_ids[source_tag_names["album"]] = source_album_id
if not source_ids:
if cfg.get("spotify.embed_tags", True) is not False:
if metadata.get("spotify_track_id"):
source_ids["SPOTIFY_TRACK_ID"] = metadata["spotify_track_id"]
if metadata.get("spotify_artist_id"):
source_ids["SPOTIFY_ARTIST_ID"] = metadata["spotify_artist_id"]
if metadata.get("spotify_album_id"):
source_ids["SPOTIFY_ALBUM_ID"] = metadata["spotify_album_id"]
if cfg.get("itunes.embed_tags", True) is not False:
if metadata.get("itunes_track_id"):
source_ids["ITUNES_TRACK_ID"] = metadata["itunes_track_id"]
if metadata.get("itunes_artist_id"):
source_ids["ITUNES_ARTIST_ID"] = metadata["itunes_artist_id"]
if metadata.get("itunes_album_id"):
source_ids["ITUNES_ALBUM_ID"] = metadata["itunes_album_id"]
return source_ids
def _process_musicbrainz_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
if cfg.get("musicbrainz.embed_tags", True) is False:
return
if not track_title or not artist_name:
return
mb_worker = getattr(runtime, "mb_worker", None)
mb_service = mb_worker.mb_service if mb_worker else None
if not mb_service:
return
result = _call_source_lookup("MusicBrainz recording", mb_service.match_recording, track_title, artist_name)
if result and result.get("mbid"):
pp["recording_mbid"] = result["mbid"]
pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"]
details = _call_source_lookup(
"MusicBrainz recording details",
mb_service.mb_client.get_recording,
pp["recording_mbid"],
includes=["isrcs", "genres"],
)
if details:
isrcs = details.get("isrcs", [])
if isrcs:
pp["isrc"] = isrcs[0]
pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)]
track_artist_name = metadata.get("artist", "") or artist_name
if ", " in track_artist_name:
track_artist_name = track_artist_name.split(", ")[0]
artist_result = _call_source_lookup("MusicBrainz artist", mb_service.match_artist, track_artist_name)
if artist_result and artist_result.get("mbid"):
pp["artist_mbid"] = artist_result["mbid"]
pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"]
album_name_for_mb = metadata.get("album", "")
if album_name_for_mb:
artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip()
rc_key_norm = (normalize_album_cache_key(album_name_for_mb), artist_key)
rc_key_exact = (album_name_for_mb.lower().strip(), artist_key)
release_mbid = None
with mb_release_cache_lock:
cached = _bounded_cache_get(mb_release_cache, rc_key_norm)
if cached is None:
cached = _bounded_cache_get(mb_release_cache, rc_key_exact)
if cached:
release_mbid = cached
else:
rc_result = _call_source_lookup("MusicBrainz release", mb_service.match_release, album_name_for_mb, artist_name)
if rc_result and rc_result.get("mbid"):
release_mbid = rc_result["mbid"]
if release_mbid:
_bounded_cache_set(mb_release_cache, rc_key_norm, release_mbid, _MB_RELEASE_CACHE_MAX_ENTRIES)
_bounded_cache_set(mb_release_cache, rc_key_exact, release_mbid, _MB_RELEASE_CACHE_MAX_ENTRIES)
pp["release_mbid"] = release_mbid or ""
if pp["release_mbid"]:
pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"]
if pp["release_mbid"]:
with mb_release_detail_cache_lock:
release_detail = _bounded_cache_get(mb_release_detail_cache, pp["release_mbid"])
if release_detail is None:
release_detail = _call_source_lookup(
"MusicBrainz release details",
mb_service.mb_client.get_release,
pp["release_mbid"],
includes=["release-groups", "labels", "media", "artist-credits", "recordings"],
) or {}
with mb_release_detail_cache_lock:
_bounded_cache_set(mb_release_detail_cache, pp["release_mbid"], release_detail, _MB_RELEASE_DETAIL_CACHE_MAX_ENTRIES)
if release_detail:
rg = release_detail.get("release-group", {})
if rg.get("id"):
pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"]
ac = release_detail.get("artist-credit", [])
if ac and isinstance(ac[0], dict):
aa = ac[0].get("artist", {})
if aa.get("id"):
pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"]
if rg.get("primary-type"):
pp["id_tags"]["RELEASETYPE"] = rg["primary-type"]
if rg.get("first-release-date"):
pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"]
if not pp["release_year"] and len(rg["first-release-date"]) >= 4:
year = rg["first-release-date"][:4]
if year.isdigit():
pp["release_year"] = year
if release_detail.get("status"):
pp["id_tags"]["RELEASESTATUS"] = release_detail["status"]
if release_detail.get("country"):
pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"]
if release_detail.get("barcode"):
pp["id_tags"]["BARCODE"] = release_detail["barcode"]
media_list = release_detail.get("media", [])
if media_list:
fmt = media_list[0].get("format", "")
if fmt:
pp["id_tags"]["MEDIA"] = fmt
pp["id_tags"]["TOTALDISCS"] = str(len(media_list))
label_info = release_detail.get("label-info", [])
if label_info and isinstance(label_info[0], dict):
cat = label_info[0].get("catalog-number", "")
if cat:
pp["id_tags"]["CATALOGNUMBER"] = cat
text_rep = release_detail.get("text-representation", {})
if isinstance(text_rep, dict) and text_rep.get("script"):
pp["id_tags"]["SCRIPT"] = text_rep["script"]
if release_detail.get("asin"):
pp["id_tags"]["ASIN"] = release_detail["asin"]
track_num = metadata.get("track_number")
disc_num = metadata.get("disc_number") or 1
if track_num and media_list:
try:
track_num_int = int(track_num)
disc_num_int = int(disc_num)
for medium in media_list:
if medium.get("position", 1) == disc_num_int:
for mtrack in (medium.get("tracks") or medium.get("track-list", [])):
if mtrack.get("position") == track_num_int:
if mtrack.get("id"):
pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"]
release_recording = mtrack.get("recording", {})
if release_recording.get("id"):
pp["recording_mbid"] = release_recording["id"]
pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"]
break
break
except (ValueError, TypeError):
pass
def _process_deezer_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
if cfg.get("deezer.embed_tags", True) is False:
return
if not track_title or not artist_name:
return
deezer_worker = getattr(runtime, "deezer_worker", None)
dz_client = deezer_worker.client if deezer_worker else None
if not dz_client:
return
dz_result = _call_source_lookup("Deezer track", dz_client.search_track, artist_name, track_title)
if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name):
dz_track_id = dz_result["id"]
pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id)
dz_artist_id = dz_result.get("artist", {}).get("id")
if dz_artist_id:
pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id)
dz_details = _call_source_lookup("Deezer track details", dz_client.get_track_details, dz_track_id)
if dz_details:
bpm_val = dz_details.get("bpm")
if bpm_val and bpm_val > 0:
pp["deezer_bpm"] = bpm_val
dz_isrc = dz_details.get("isrc")
if dz_isrc:
pp["deezer_isrc"] = dz_isrc
if not pp["release_year"]:
dz_album = dz_result.get("album", {})
dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or ""
if len(dz_release) >= 4 and dz_release[:4].isdigit():
pp["release_year"] = dz_release[:4]
def _process_audiodb_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
if cfg.get("audiodb.embed_tags", True) is False:
return
if not track_title or not artist_name:
return
audiodb_worker = getattr(runtime, "audiodb_worker", None)
adb_client = audiodb_worker.client if audiodb_worker else None
if not adb_client:
return
adb_result = _call_source_lookup("AudioDB track", adb_client.search_track, artist_name, track_title)
if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name):
adb_track_id = adb_result.get("idTrack")
if adb_track_id:
pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id)
adb_mb_track = adb_result.get("strMusicBrainzID")
if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]:
pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track
pp["recording_mbid"] = adb_mb_track
adb_mb_artist = adb_result.get("strMusicBrainzArtistID")
if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]:
pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist
pp["artist_mbid"] = adb_mb_artist
pp["audiodb_mood"] = adb_result.get("strMood") or None
pp["audiodb_style"] = adb_result.get("strStyle") or None
pp["audiodb_genre"] = adb_result.get("strGenre") or None
def _process_tidal_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
if cfg.get("tidal.embed_tags", True) is False:
return
if not track_title or not artist_name:
return
tidal_client = getattr(runtime, "tidal_client", None)
if not (tidal_client and tidal_client.is_authenticated()):
return
td_result = _call_source_lookup("Tidal track", tidal_client.search_track, artist_name, track_title)
if td_result and _names_match(td_result.get("title", ""), track_title):
td_track_id = td_result.get("id")
if td_track_id:
pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id)
td_artist = td_result.get("artist", {})
if isinstance(td_artist, dict) and td_artist.get("id"):
pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"])
if td_track_id:
td_details = _call_source_lookup("Tidal track details", tidal_client.get_track, str(td_track_id))
if td_details:
pp["tidal_isrc"] = td_details.get("isrc")
td_copyright = td_details.get("copyright")
if isinstance(td_copyright, dict):
td_copyright = td_copyright.get("text", td_copyright.get("name", ""))
pp["tidal_copyright"] = td_copyright or None
if not pp["release_year"]:
td_album = td_result.get("album", {})
td_release = ""
if isinstance(td_album, dict):
td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "")
if len(td_release) >= 4 and td_release[:4].isdigit():
pp["release_year"] = td_release[:4]
def _process_qobuz_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
if cfg.get("qobuz.embed_tags", True) is False:
return
if not track_title or not artist_name:
return
qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None)
qz_client = qobuz_worker.client if qobuz_worker else None
if not (qz_client and qz_client.is_authenticated()):
return
qz_result = _call_source_lookup("Qobuz track", qz_client.search_track, artist_name, track_title)
if qz_result:
qz_performer = qz_result.get("performer") or {}
if not isinstance(qz_performer, dict):
qz_performer = {}
qz_artist_name = qz_performer.get("name", "")
if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name):
qz_track_id = qz_result.get("id")
if qz_track_id:
pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id)
if qz_performer.get("id"):
pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"])
qz_isrc = qz_result.get("isrc")
if isinstance(qz_isrc, dict):
qz_isrc = qz_isrc.get("value", qz_isrc.get("id", ""))
if qz_isrc:
pp["qobuz_isrc"] = qz_isrc
qz_copyright = qz_result.get("copyright")
if isinstance(qz_copyright, dict):
qz_copyright = qz_copyright.get("text", qz_copyright.get("name", ""))
if isinstance(qz_copyright, str):
pp["qobuz_copyright"] = qz_copyright
qz_album = qz_result.get("album", {})
if isinstance(qz_album, dict):
qz_label_info = qz_album.get("label", {})
if isinstance(qz_label_info, dict) and qz_label_info.get("name"):
pp["qobuz_label"] = qz_label_info["name"]
if not pp["release_year"]:
qz_release = str(qz_album.get("release_date_original", "") or "")
if not qz_release:
qz_ts = qz_album.get("released_at")
if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0:
import datetime as _dt
qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year)
if len(qz_release) >= 4 and qz_release[:4].isdigit():
pp["release_year"] = qz_release[:4]
def _process_lastfm_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
if cfg.get("lastfm.embed_tags", True) is False:
return
if not track_title or not artist_name:
return
lastfm_worker = getattr(runtime, "lastfm_worker", None)
lf_client = lastfm_worker.client if lastfm_worker else None
if not lf_client:
return
lf_result = _call_source_lookup("Last.fm track", lf_client.get_track_info, artist_name, track_title)
if lf_result:
lf_url = lf_result.get("url")
if lf_url:
pp["lastfm_url"] = lf_url
lf_toptags = lf_result.get("toptags", {})
if isinstance(lf_toptags, dict):
tag_list = lf_toptags.get("tag", [])
if isinstance(tag_list, list):
pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")]
elif isinstance(tag_list, dict) and tag_list.get("name"):
pp["lastfm_tags"] = [tag_list["name"]]
def _process_genius_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
if cfg.get("genius.embed_tags", True) is False:
return
if not track_title or not artist_name:
return
import core.genius_client as _genius_module
if time.time() < _genius_module._rate_limit_until:
logger.info("Genius rate-limited, skipping (non-blocking)")
return
genius_worker = getattr(runtime, "genius_worker", None)
g_client = genius_worker.client if genius_worker else None
if not g_client:
return
g_result = _call_source_lookup("Genius track", g_client.search_song, artist_name, track_title)
if g_result:
g_id = g_result.get("id")
if g_id:
pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id)
g_url = g_result.get("url")
if g_url:
pp["genius_url"] = g_url
def _process_source_enrichment(source_name: str, pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
if source_name == "musicbrainz":
_process_musicbrainz_source(pp, metadata, cfg, runtime, track_title, artist_name)
elif source_name == "deezer":
_process_deezer_source(pp, metadata, cfg, runtime, track_title, artist_name)
elif source_name == "audiodb":
_process_audiodb_source(pp, metadata, cfg, runtime, track_title, artist_name)
elif source_name == "tidal":
_process_tidal_source(pp, metadata, cfg, runtime, track_title, artist_name)
elif source_name == "qobuz":
_process_qobuz_source(pp, metadata, cfg, runtime, track_title, artist_name)
elif source_name == "lastfm":
_process_lastfm_source(pp, metadata, cfg, runtime, track_title, artist_name)
elif source_name == "genius":
_process_genius_source(pp, metadata, cfg, runtime, track_title, artist_name)
def _write_embedded_metadata(audio_file, metadata: dict, pp: dict, cfg, symbols):
filtered_tags: Dict[str, str] = {}
for tag_name, value in pp["id_tags"].items():
config_path = SOURCE_TAG_CONFIG.get(tag_name)
if config_path and not _tag_enabled(cfg, config_path):
continue
filtered_tags[tag_name] = value
written = []
release_year = pp["release_year"]
if isinstance(audio_file.tags, symbols.ID3):
for tag_name, value in filtered_tags.items():
spec = ID3_TAG_MAP.get(tag_name)
if spec:
frame_type, desc = spec
if frame_type == "UFID":
audio_file.tags.add(symbols.UFID(owner=desc, data=str(value).encode("ascii")))
written.append(f"UFID:{desc}")
elif frame_type == "TDOR":
audio_file.tags.add(symbols.TDOR(encoding=3, text=[value]))
written.append("TDOR")
elif frame_type == "TMED":
audio_file.tags.add(symbols.TMED(encoding=3, text=[value]))
written.append("TMED")
else:
audio_file.tags.add(symbols.TXXX(encoding=3, desc=desc, text=[value]))
written.append(f"TXXX:{desc}")
else:
audio_file.tags.add(symbols.TXXX(encoding=3, desc=tag_name, text=[str(value)]))
written.append(f"TXXX:{tag_name}")
elif isinstance(audio_file, symbols.MP4):
for tag_name, value in filtered_tags.items():
key = f"----:com.apple.iTunes:{MP4_TAG_MAP.get(tag_name, tag_name)}"
audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))]
written.append(key)
elif is_vorbis_like(audio_file, symbols):
for tag_name, value in filtered_tags.items():
audio_file[VORBIS_TAG_MAP.get(tag_name, tag_name)] = [str(value)]
written.append(VORBIS_TAG_MAP.get(tag_name, tag_name))
if written:
logger.info("Embedded IDs: %s", ", ".join(written))
if release_year and not metadata.get("date"):
metadata["date"] = release_year
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year]))
elif is_vorbis_like(audio_file, symbols):
audio_file["date"] = [release_year]
elif isinstance(audio_file, symbols.MP4):
audio_file["\xa9day"] = [release_year]
logger.info("Date tag: %s", release_year)
if _tag_enabled(cfg, "deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0:
bpm_int = int(pp["deezer_bpm"])
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)]))
elif is_vorbis_like(audio_file, symbols):
audio_file["BPM"] = [str(bpm_int)]
elif isinstance(audio_file, symbols.MP4):
audio_file["tmpo"] = [bpm_int]
logger.info("BPM: %s", bpm_int)
if _tag_enabled(cfg, "audiodb.tags.mood") and pp["audiodb_mood"]:
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]]))
elif is_vorbis_like(audio_file, symbols):
audio_file["MOOD"] = [pp["audiodb_mood"]]
elif isinstance(audio_file, symbols.MP4):
audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))]
if _tag_enabled(cfg, "audiodb.tags.style") and pp["audiodb_style"]:
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]]))
elif is_vorbis_like(audio_file, symbols):
audio_file["STYLE"] = [pp["audiodb_style"]]
elif isinstance(audio_file, symbols.MP4):
audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))]
if _tag_enabled(cfg, "metadata_enhancement.tags.genre_merge"):
enrichment_genres = []
if _tag_enabled(cfg, "musicbrainz.tags.genres"):
enrichment_genres += pp["mb_genres"]
if pp["audiodb_genre"] and _tag_enabled(cfg, "audiodb.tags.genre"):
enrichment_genres.append(pp["audiodb_genre"])
if _tag_enabled(cfg, "lastfm.tags.genres"):
enrichment_genres += pp["lastfm_tags"]
if enrichment_genres:
from core.genre_filter import filter_genres as _filter_genres
enrichment_genres = _filter_genres(enrichment_genres, cfg)
source_genres = [g.strip() for g in str(metadata.get("genre", "")).split(",") if g.strip()]
seen = set()
merged = []
for genre in source_genres + enrichment_genres:
key = genre.strip().lower()
if key and key not in seen:
seen.add(key)
merged.append(genre.strip().title())
if len(merged) >= 5:
break
if merged:
genre_string = ", ".join(merged)
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string]))
elif is_vorbis_like(audio_file, symbols):
audio_file["GENRE"] = [genre_string]
elif isinstance(audio_file, symbols.MP4):
audio_file["\xa9gen"] = [genre_string]
logger.info("Genres merged: %s", genre_string)
isrc_candidates = []
if pp["isrc"] and _tag_enabled(cfg, "musicbrainz.tags.isrc"):
isrc_candidates.append(("MusicBrainz", pp["isrc"]))
if pp["deezer_isrc"] and _tag_enabled(cfg, "deezer.tags.isrc"):
isrc_candidates.append(("Deezer", pp["deezer_isrc"]))
if pp["tidal_isrc"] and _tag_enabled(cfg, "tidal.tags.isrc"):
isrc_candidates.append(("Tidal", pp["tidal_isrc"]))
if pp["qobuz_isrc"] and _tag_enabled(cfg, "qobuz.tags.isrc"):
isrc_candidates.append(("Qobuz", pp["qobuz_isrc"]))
if isrc_candidates:
isrc_source, final_isrc = isrc_candidates[0]
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc]))
elif is_vorbis_like(audio_file, symbols):
audio_file["ISRC"] = [final_isrc]
elif isinstance(audio_file, symbols.MP4):
audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))]
logger.info("ISRC (%s): %s", isrc_source, final_isrc)
copyright_candidates = []
if pp["tidal_copyright"] and _tag_enabled(cfg, "tidal.tags.copyright"):
copyright_candidates.append(("Tidal", pp["tidal_copyright"]))
if pp["qobuz_copyright"] and _tag_enabled(cfg, "qobuz.tags.copyright"):
copyright_candidates.append(("Qobuz", pp["qobuz_copyright"]))
if copyright_candidates:
copyright_source, final_copyright = copyright_candidates[0]
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright]))
elif is_vorbis_like(audio_file, symbols):
audio_file["COPYRIGHT"] = [final_copyright]
elif isinstance(audio_file, symbols.MP4):
audio_file["cprt"] = [final_copyright]
logger.info("Copyright (%s): %s", copyright_source, final_copyright[:60])
if _tag_enabled(cfg, "qobuz.tags.label") and pp["qobuz_label"]:
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]]))
elif is_vorbis_like(audio_file, symbols):
audio_file["LABEL"] = [pp["qobuz_label"]]
elif isinstance(audio_file, symbols.MP4):
audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))]
if _tag_enabled(cfg, "lastfm.tags.url") and pp["lastfm_url"]:
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]]))
elif is_vorbis_like(audio_file, symbols):
audio_file["LASTFM_URL"] = [pp["lastfm_url"]]
elif isinstance(audio_file, symbols.MP4):
audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))]
if _tag_enabled(cfg, "genius.tags.url") and pp["genius_url"]:
if isinstance(audio_file.tags, symbols.ID3):
audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]]))
elif is_vorbis_like(audio_file, symbols):
audio_file["GENIUS_URL"] = [pp["genius_url"]]
elif isinstance(audio_file, symbols.MP4):
audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))]
return release_year
def _update_album_year_in_database(db, metadata: dict, release_year) -> None:
if db is None:
return
try:
album_name_for_db = metadata.get("album", "")
album_artist_for_db = metadata.get("album_artist", "") or metadata.get("artist", "")
if album_name_for_db and album_artist_for_db:
conn = db._get_connection()
try:
cursor = conn.cursor()
cursor.execute(
"""
UPDATE albums SET year = ?
WHERE (year IS NULL OR year = 0)
AND id IN (
SELECT al.id FROM albums al
JOIN artists ar ON ar.id = al.artist_id
WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?)
)
""",
(int(release_year), album_name_for_db, album_artist_for_db),
)
if cursor.rowcount > 0:
conn.commit()
logger.info("Updated album year to %s in database", release_year)
else:
conn.rollback()
finally:
conn.close()
except Exception as exc:
logger.error("Could not update album year in DB: %s", exc)
def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> dict:
if album_info is None:
album_info = {}
cfg = get_config_manager()
context = normalize_import_context(context)
original_search = get_import_original_search(context)
album_ctx = get_import_context_album(context)
track_info = get_import_track_info(context)
source = get_import_source(context)
source_ids = get_import_source_ids(context)
artist_dict = artist if isinstance(artist, dict) else {
"name": extract_artist_name(artist),
"id": getattr(artist, "id", ""),
"genres": list(getattr(artist, "genres", []) or []),
}
metadata: Dict[str, Any] = {
"source": source,
"source_track_id": source_ids["track_id"],
"source_artist_id": source_ids["artist_id"],
"source_album_id": source_ids["album_id"],
}
metadata["title"] = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", ""))
if original_search.get("clean_title"):
logger.info("Metadata: Using clean title: '%s'", metadata["title"])
elif album_info.get("clean_track_name"):
logger.info("Metadata: Using album info clean name: '%s'", metadata["title"])
else:
logger.warning("Metadata: Using original title as fallback: '%s'", metadata["title"])
artists = original_search.get("artists")
if isinstance(artists, list) and artists:
all_artists = []
for artist_item in artists:
if isinstance(artist_item, dict) and artist_item.get("name"):
all_artists.append(artist_item["name"])
elif isinstance(artist_item, str):
all_artists.append(artist_item)
else:
all_artists.append(str(artist_item))
metadata["artist"] = ", ".join(all_artists)
logger.info("Metadata: Using all artists: '%s'", metadata["artist"])
else:
metadata["artist"] = artist_dict.get("name", "") or get_import_clean_artist(context)
logger.info("Metadata: Using primary artist: '%s'", metadata["artist"])
raw_album_artist = artist_dict.get("name", "") or metadata["artist"]
track_info_ctx = track_info or {}
explicit_artist = track_info_ctx.get("_explicit_artist_context") if isinstance(track_info_ctx, dict) else None
album_artists_for_collab = None
if isinstance(explicit_artist, dict) and explicit_artist.get("name"):
raw_album_artist = explicit_artist["name"]
album_artists_for_collab = [explicit_artist]
elif isinstance(explicit_artist, str) and explicit_artist:
raw_album_artist = explicit_artist
album_artists_for_collab = [{"name": explicit_artist}]
elif album_ctx and isinstance(album_ctx, dict):
album_artists = album_ctx.get("artists", [])
if album_artists:
first_album_artist = album_artists[0]
if isinstance(first_album_artist, dict) and first_album_artist.get("name"):
raw_album_artist = first_album_artist["name"]
elif isinstance(first_album_artist, str) and first_album_artist:
raw_album_artist = first_album_artist
album_artists_for_collab = album_artists
collab_mode = cfg.get("file_organization.collab_artist_mode", "first")
if collab_mode == "first" and raw_album_artist:
context_artists = album_artists_for_collab or original_search.get("artists") or track_info_ctx.get("artists") or []
if len(context_artists) > 1:
first = context_artists[0]
raw_album_artist = first.get("name", first) if isinstance(first, dict) else str(first)
elif len(context_artists) == 1 and ("," in raw_album_artist or " & " in raw_album_artist):
artist_id = str(artist_dict.get("id", ""))
if source == "itunes" and artist_id.isdigit():
try:
itunes_client = get_itunes_client()
if itunes_client and hasattr(itunes_client, "resolve_primary_artist"):
resolved = itunes_client.resolve_primary_artist(artist_id)
if resolved and resolved != raw_album_artist:
raw_album_artist = resolved
except Exception:
pass
metadata["album_artist"] = raw_album_artist
if album_info.get("is_album"):
metadata["album"] = album_info.get("album_name", "Unknown Album")
metadata["track_number"] = album_info.get("track_number", 1)
metadata["total_tracks"] = album_ctx.get("total_tracks", 1) if album_ctx else 1
logger.info("[METADATA] Album track - track_number: %s, album: %s", metadata["track_number"], metadata["album"])
else:
if album_ctx and album_ctx.get("name"):
logger.info("[SAFEGUARD] Using album context name instead of track title for album metadata")
metadata["album"] = album_ctx["name"]
metadata["track_number"] = album_info.get("track_number", 1) if album_info else 1
metadata["total_tracks"] = album_ctx.get("total_tracks", 1)
else:
metadata["album"] = metadata["title"]
metadata["track_number"] = 1
metadata["total_tracks"] = 1
disc_num = original_search.get("disc_number")
if disc_num is None and album_info:
disc_num = album_info.get("disc_number")
metadata["disc_number"] = disc_num if disc_num is not None else 1
if album_ctx and album_ctx.get("release_date"):
metadata["date"] = album_ctx["release_date"][:4]
genres = artist_dict.get("genres") or []
if genres:
from core.genre_filter import filter_genres
filtered = filter_genres(list(genres[:2]), cfg)
if filtered:
metadata["genre"] = ", ".join(filtered)
metadata["album_art_url"] = album_info.get("album_image_url") if album_info else None
if not metadata["album_art_url"] and album_ctx:
album_image = album_ctx.get("image_url")
if not album_image and album_ctx.get("images"):
first_image = album_ctx["images"][0]
album_image = first_image.get("url") if isinstance(first_image, dict) else None
metadata["album_art_url"] = album_image
logger.info(
"[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | track=%s/%s | disc=%s",
metadata.get("title"),
metadata.get("artist"),
metadata.get("album_artist"),
metadata.get("album"),
metadata.get("track_number"),
metadata.get("total_tracks"),
metadata.get("disc_number"),
)
return metadata
def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=None):
cfg = get_config_manager()
symbols = get_mutagen_symbols()
if not symbols:
return
try:
context = normalize_import_context(context)
source_ids = _collect_source_ids(metadata, cfg)
track_title = metadata.get("title", "")
artist_name = metadata.get("album_artist", "") or metadata.get("artist", "")
track_info = get_import_track_info(context)
explicit_artist = (track_info or {}).get("_explicit_artist_context") if isinstance(track_info, dict) else None
batch_artist_name = None
if isinstance(explicit_artist, dict) and explicit_artist.get("name"):
batch_artist_name = explicit_artist["name"]
elif isinstance(explicit_artist, str) and explicit_artist:
batch_artist_name = explicit_artist
pp = {
"id_tags": source_ids,
"track_title": track_title,
"artist_name": artist_name,
"batch_artist_name": batch_artist_name,
"metadata": metadata,
"recording_mbid": None,
"artist_mbid": None,
"release_mbid": "",
"mb_genres": [],
"isrc": None,
"deezer_bpm": None,
"deezer_isrc": None,
"audiodb_mood": None,
"audiodb_style": None,
"audiodb_genre": None,
"tidal_isrc": None,
"tidal_copyright": None,
"qobuz_isrc": None,
"qobuz_copyright": None,
"qobuz_label": None,
"lastfm_tags": [],
"lastfm_url": None,
"genius_url": None,
"release_year": None,
}
source_order = cfg.get("metadata_enhancement.post_process_order", None)
if not isinstance(source_order, list) or not source_order:
source_order = DEFAULT_SOURCE_ORDER
db = get_database()
for source_name in source_order:
_process_source_enrichment(source_name, pp, metadata, cfg, runtime, track_title, artist_name)
if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["audiodb_mood"] and not pp["audiodb_style"]:
return
release_year = _write_embedded_metadata(audio_file, metadata, pp, cfg, symbols)
release_id = pp["release_mbid"]
if release_id:
metadata["musicbrainz_release_id"] = release_id
_update_album_year_in_database(db, metadata, release_year)
except Exception as exc:
logger.error("Error embedding source IDs (non-fatal): %s", exc)

View file

@ -1,5 +1,5 @@
"""
Metadata Service - Centralized metadata source selection
Metadata Service - Centralized metadata source selection and provider access.
ALL metadata source decisions flow through this module. Other files import
get_primary_source() and get_primary_client() instead of reimplementing
@ -100,6 +100,35 @@ def get_source_priority(preferred_source: str):
return ordered
def _get_source_chain_for_lookup(options: MetadataLookupOptions) -> List[str]:
primary_source = get_primary_source()
source_chain = list(get_source_priority(primary_source))
override = (options.source_override or '').strip().lower()
if override:
source_chain = [override] + [source for source in source_chain if source != override]
if not options.allow_fallback:
source_chain = source_chain[:1]
return source_chain
def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any:
if value is None:
return default
for name in names:
if isinstance(value, dict):
if name in value and value[name] is not None:
return value[name]
else:
candidate = getattr(value, name, None)
if candidate is not None:
return candidate
return default
def get_client_for_source(source: str):
"""Get the client object for an exact metadata source.
@ -235,35 +264,6 @@ def get_artist_albums_for_source(
return None
def _get_source_chain_for_lookup(options: MetadataLookupOptions) -> List[str]:
primary_source = get_primary_source()
source_chain = list(get_source_priority(primary_source))
override = (options.source_override or '').strip().lower()
if override:
source_chain = [override] + [source for source in source_chain if source != override]
if not options.allow_fallback:
source_chain = source_chain[:1]
return source_chain
def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any:
if value is None:
return default
for name in names:
if isinstance(value, dict):
if name in value and value[name] is not None:
return value[name]
else:
candidate = getattr(value, name, None)
if candidate is not None:
return candidate
return default
def _normalize_artist_name(value: Any) -> str:
return (value or '').strip().casefold()
@ -719,6 +719,23 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti
if not isinstance(images, list):
images = list(images) if images else []
artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[]))
if not artists and artist_name:
artists = [{'name': artist_name}]
primary_artist = artists[0] if artists else {}
resolved_artist_name = (
_extract_lookup_value(primary_artist, 'name', default='')
or artist_name
or _extract_lookup_value(album_data, 'artist_name', 'artist', default='')
or ''
)
resolved_artist_id = str(
_extract_lookup_value(primary_artist, 'id', default='')
or _extract_lookup_value(album_data, 'artist_id', default='')
or ''
).strip()
image_url = None
if images:
image_url = _extract_lookup_value(images[0], 'url')
@ -728,12 +745,15 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti
return {
'id': _extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', 'release_id', default=album_id) or album_id,
'name': _extract_lookup_value(album_data, 'name', 'title', default=album_name or album_id) or album_name or album_id,
'artist': resolved_artist_name or '',
'artist_name': resolved_artist_name or '',
'artist_id': resolved_artist_id,
'artists': artists,
'image_url': image_url,
'images': images,
'release_date': _extract_lookup_value(album_data, 'release_date', default='') or '',
'album_type': _extract_lookup_value(album_data, 'album_type', default='album') or 'album',
'total_tracks': _extract_lookup_value(album_data, 'total_tracks', 'track_count', default=0) or 0,
'artist_name': artist_name or _extract_lookup_value(album_data, 'artist_name', default='') or '',
}
@ -754,6 +774,8 @@ def _build_album_track_entry(track_item: Any, album_info: Dict[str, Any], source
'external_urls': _extract_lookup_value(track_item, 'external_urls', default={}) or {},
'uri': _extract_lookup_value(track_item, 'uri', default='') or '',
'album': album_info,
'source': source,
'provider': source,
'_source': source,
}
@ -767,6 +789,9 @@ def _build_album_tracks_payload(
artist_name: str = '',
) -> Dict[str, Any]:
album_info = _build_album_info(album_data, album_id, album_name=album_name, artist_name=artist_name)
album_info['source'] = source
album_info['_source'] = source
album_info['provider'] = source
track_items = _extract_album_track_items(album_data, tracks_data)
tracks = [_build_album_track_entry(track, album_info, source) for track in track_items]
@ -778,6 +803,44 @@ def _build_album_tracks_payload(
}
def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]:
if not artists:
return []
if isinstance(artists, (str, bytes)):
artists = [artists]
elif isinstance(artists, dict):
artists = [artists]
else:
try:
artists = list(artists)
except TypeError:
artists = [artists]
normalized: List[Dict[str, Any]] = []
for artist in artists:
if isinstance(artist, dict):
name = _extract_lookup_value(artist, 'name', 'artist_name', 'title', default='') or ''
artist_id = _extract_lookup_value(artist, 'id', 'artist_id', default='') or ''
entry: Dict[str, Any] = {}
if name:
entry['name'] = str(name)
if artist_id:
entry['id'] = str(artist_id)
genres = _extract_lookup_value(artist, 'genres', default=None)
if genres is not None:
entry['genres'] = genres
if entry:
normalized.append(entry)
continue
name = str(artist).strip()
if name:
normalized.append({'name': name})
return normalized
def resolve_album_reference(
album_id: str,
preferred_source: Optional[str] = None,

80
core/runtime_state.py Normal file
View file

@ -0,0 +1,80 @@
"""Shared runtime state and tiny helpers for the app."""
from __future__ import annotations
import threading
import time
from functools import wraps
from typing import Any, Dict, Optional
matched_context_lock = threading.Lock()
matched_downloads_context: Dict[str, Dict[str, Any]] = {}
tasks_lock = threading.Lock()
download_tasks: Dict[str, Dict[str, Any]] = {}
download_batches: Dict[str, Dict[str, Any]] = {}
processed_download_ids = set()
post_process_locks: Dict[str, threading.Lock] = {}
post_process_locks_lock = threading.Lock()
activity_feed = []
activity_feed_lock = threading.Lock()
_activity_toast_emitter = None
def caller_must_hold_tasks_lock(func):
"""Best-effort guard for helpers that mutate download_tasks in place."""
@wraps(func)
def wrapper(*args, **kwargs):
if not tasks_lock.locked():
raise RuntimeError(f"{func.__name__}() requires tasks_lock to be held by the caller")
return func(*args, **kwargs)
return wrapper
def set_activity_toast_emitter(emitter) -> None:
"""Set the WebSocket-style emitter used by add_activity_item."""
global _activity_toast_emitter
_activity_toast_emitter = emitter
def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True):
"""Append an activity item and emit a toast if an emitter is configured."""
activity_item = {
"icon": icon,
"title": title,
"subtitle": subtitle,
"time": time_ago,
"timestamp": time.time(),
"show_toast": show_toast,
}
with activity_feed_lock:
activity_feed.append(activity_item)
if len(activity_feed) > 20:
activity_feed.pop(0)
if show_toast and _activity_toast_emitter is not None:
try:
_activity_toast_emitter("dashboard:toast", activity_item)
except Exception:
pass
return activity_item
@caller_must_hold_tasks_lock
def mark_task_completed(task_id: str, track_info: Optional[Dict[str, Any]] = None) -> bool:
"""Mark a download task as completed.
Callers must already hold `tasks_lock`.
"""
task = download_tasks.get(task_id)
if not task:
return False
task["status"] = "completed"
task["stream_processed"] = True
task["status_change_time"] = time.time()
if track_info is not None:
task["track_info"] = track_info
return True

View file

@ -8,6 +8,7 @@ import time
from pathlib import Path
from utils.logging_config import get_logger
from config.settings import config_manager
from core.imports.filename import parse_filename_metadata
logger = get_logger("soulseek_client")
@ -87,60 +88,17 @@ class TrackResult(SearchResult):
def _parse_filename_metadata(self):
"""Extract artist, title, album from filename patterns"""
import re
import os
# Get just the filename without extension and path
base_name = os.path.splitext(os.path.basename(self.filename))[0]
# Common patterns for track naming
patterns = [
r'^(\d+)\s*[-\.]\s*(.+?)\s*[-]\s*(.+)$', # "01 - Artist - Title" or "01. Artist - Title"
r'^(.+?)\s*[-]\s*(.+)$', # "Artist - Title"
r'^(\d+)\s*[-\.]\s*(.+)$', # "01 - Title" or "01. Title"
]
for pattern in patterns:
match = re.match(pattern, base_name)
if match:
groups = match.groups()
if len(groups) == 3: # Track number, artist, title
try:
self.track_number = int(groups[0])
self.artist = self.artist or groups[1].strip()
self.title = self.title or groups[2].strip()
except ValueError:
# First group might not be a number
self.artist = self.artist or groups[0].strip()
self.title = self.title or f"{groups[1]} - {groups[2]}".strip()
elif len(groups) == 2:
if groups[0].isdigit(): # Track number and title
try:
self.track_number = int(groups[0])
self.title = self.title or groups[1].strip()
except ValueError:
pass
else: # Artist and title
self.artist = self.artist or groups[0].strip()
self.title = self.title or groups[1].strip()
break
# Fallback: use filename as title if nothing was extracted
if not self.title:
self.title = base_name
# Try to extract album from directory path
if not self.album and '/' in self.filename:
path_parts = self.filename.split('/')
if len(path_parts) >= 2:
# Look for album-like directory names
for part in reversed(path_parts[:-1]): # Exclude filename
if part and not part.startswith('@'): # Skip system directories
# Clean up common patterns
cleaned = re.sub(r'^\d+\s*[-\.]\s*', '', part) # Remove leading numbers
if len(cleaned) > 3: # Must be substantial
self.album = cleaned
break
parsed = parse_filename_metadata(self.filename)
if not self.artist and parsed.get("artist"):
self.artist = parsed["artist"]
if not self.title and parsed.get("title"):
self.title = parsed["title"]
if not self.album and parsed.get("album"):
self.album = parsed["album"]
if self.track_number is None:
track_number = parsed.get("track_number")
if track_number is not None:
self.track_number = track_number
@dataclass
class AlbumResult:
@ -1829,4 +1787,4 @@ class SoulseekClient:
def __del__(self):
# No persistent session to clean up
pass
pass

View file

@ -2955,9 +2955,6 @@ class WatchlistScanner:
}
track_data = {
'spotify_track_id': track['id'],
'spotify_album_id': album_data['id'],
'spotify_artist_id': artist.spotify_artist_id,
'track_name': track['name'],
'artist_name': artist.artist_name,
'album_name': album_data.get('name', 'Unknown Album'),

View file

@ -0,0 +1,189 @@
import core.imports.album as import_album
class _FakeEngine:
def normalize_string(self, text):
return str(text or "").strip().lower()
def similarity_score(self, left, right):
return 1.0 if left == right else 0.0
def test_resolve_album_artist_context_uses_provider_genres(monkeypatch):
class _FakeClient:
def get_artist(self, artist_id):
assert artist_id == "artist-1"
return {"genres": ["rock", "indie"]}
monkeypatch.setattr(import_album, "get_client_for_source", lambda source: _FakeClient() if source == "spotify" else None)
context = import_album.resolve_album_artist_context(
{
"id": "album-1",
"name": "Album One",
"artist_id": "artist-1",
"artists": [{"name": "Artist One", "id": "artist-1"}],
},
source="spotify",
)
assert context == {
"id": "artist-1",
"name": "Artist One",
"genres": ["rock", "indie"],
"source": "spotify",
}
def test_build_album_import_context_is_neutral():
context = import_album.build_album_import_context(
{
"id": "album-1",
"name": "Album One",
"artist": "Artist One",
"artist_id": "artist-1",
"artists": [{"name": "Artist One", "id": "artist-1"}],
"release_date": "2024-01-01",
"total_tracks": 12,
"album_type": "album",
"image_url": "https://img.example/album.jpg",
"source": "deezer",
},
{
"id": "track-1",
"name": "Song One",
"track_number": 3,
"disc_number": 2,
"duration_ms": 180000,
"artists": [{"name": "Artist One"}],
"uri": "deezer:track:track-1",
"album": "Album One",
"source": "deezer",
},
artist_context={
"id": "artist-1",
"name": "Artist One",
"genres": ["rock"],
"source": "deezer",
},
total_discs=2,
source="deezer",
)
assert context["artist"]["name"] == "Artist One"
assert context["artist"]["genres"] == ["rock"]
assert context["album"]["name"] == "Album One"
assert context["album"]["total_discs"] == 2
assert context["track_info"]["name"] == "Song One"
assert context["track_info"]["track_number"] == 3
assert context["original_search_result"]["clean_title"] == "Song One"
assert context["source"] == "deezer"
assert "spotify_artist" not in context
assert "spotify_album" not in context
def test_build_album_import_match_payload_uses_generic_track_keys(monkeypatch, tmp_path):
staging_root = tmp_path / "Staging"
staging_root.mkdir()
(staging_root / "Song One.flac").write_text("fake")
monkeypatch.setattr(import_album, "_get_matching_engine", lambda: _FakeEngine())
monkeypatch.setattr(
import_album,
"collect_staging_files",
lambda file_paths=None: [
{
"filename": "Song One.flac",
"full_path": str(staging_root / "Song One.flac"),
"title": "Song One",
"artist": "Artist One",
"album": "Album One",
"albumartist": "Artist One",
"track_number": 1,
"disc_number": 1,
}
],
)
monkeypatch.setattr(
import_album,
"get_artist_album_tracks",
lambda album_id, artist_name="", album_name="", source=None: {
"success": True,
"album": {
"id": album_id,
"name": "Album One",
"artist": "Artist One",
"artist_name": "Artist One",
"artist_id": "artist-1",
"artists": [{"name": "Artist One", "id": "artist-1"}],
"release_date": "2024-01-01",
"total_tracks": 1,
"total_discs": 1,
"album_type": "album",
"image_url": "https://img.example/album.jpg",
"images": [{"url": "https://img.example/album.jpg"}],
"source": "spotify",
},
"tracks": [
{
"id": "track-1",
"name": "Song One",
"track_number": 1,
"disc_number": 1,
"duration_ms": 180000,
"artists": [{"name": "Artist One"}],
"uri": "spotify:track:track-1",
"album": {
"id": album_id,
"name": "Album One",
"artist": "Artist One",
},
"source": "spotify",
}
],
"source": "spotify",
"source_priority": ["spotify"],
"resolved_album_id": album_id,
},
)
result = import_album.build_album_import_match_payload(
"album-1",
album_name="Album One",
album_artist="Artist One",
source="spotify",
)
assert result["success"] is True
assert result["album"]["artist"] == "Artist One"
assert result["source"] == "spotify"
assert result["matches"] == [
{
"track": {
"id": "track-1",
"name": "Song One",
"track_number": 1,
"disc_number": 1,
"duration_ms": 180000,
"artists": [{"name": "Artist One"}],
"uri": "spotify:track:track-1",
"album": {
"id": "album-1",
"name": "Album One",
"artist": "Artist One",
},
"source": "spotify",
},
"staging_file": {
"filename": "Song One.flac",
"full_path": str(staging_root / "Song One.flac"),
"title": "Song One",
"artist": "Artist One",
"album": "Album One",
"albumartist": "Artist One",
"track_number": 1,
"disc_number": 1,
},
"confidence": 1.0,
}
]

View file

@ -0,0 +1,311 @@
import pytest
from core.imports.context import (
build_import_album_info,
detect_album_info_web,
get_import_clean_album,
get_import_clean_artist,
get_import_clean_title,
get_import_has_clean_metadata,
get_import_has_full_metadata,
get_import_source,
get_import_source_ids,
get_library_source_id_columns,
get_source_tag_names,
normalize_import_context,
)
def test_normalize_import_context_promotes_neutral_fields_without_legacy_aliases():
context = {
"source": "deezer",
"spotify_artist": {"name": "Artist One", "id": "artist-1"},
"spotify_album": {
"name": "Album One",
"id": "album-1",
"release_date": "2024-01-01",
"total_tracks": 12,
"album_type": "album",
"image_url": "https://img.example/album.jpg",
},
"track_info": {
"name": "Song One",
"id": "track-1",
"track_number": 3,
"disc_number": 2,
"artists": [{"name": "Artist One"}],
},
"original_search_result": {
"spotify_clean_title": "Song One",
"spotify_clean_album": "Album One",
"spotify_clean_artist": "Artist One",
},
"has_clean_spotify_data": True,
"has_full_spotify_metadata": False,
}
normalized = normalize_import_context(context)
assert normalized["artist"]["name"] == "Artist One"
assert normalized["album"]["name"] == "Album One"
assert "spotify_artist" not in normalized
assert "spotify_album" not in normalized
assert normalized["original_search_result"]["clean_title"] == "Song One"
assert normalized["original_search_result"]["clean_album"] == "Album One"
assert normalized["original_search_result"]["clean_artist"] == "Artist One"
assert "spotify_clean_title" not in normalized["original_search_result"]
assert "spotify_clean_album" not in normalized["original_search_result"]
assert "spotify_clean_artist" not in normalized["original_search_result"]
assert get_import_clean_title(normalized) == "Song One"
assert get_import_clean_album(normalized) == "Album One"
assert get_import_clean_artist(normalized) == "Artist One"
assert get_import_source(normalized) == "deezer"
assert get_import_has_clean_metadata(normalized) is True
assert get_import_has_full_metadata(normalized) is False
def test_normalize_import_context_promotes_legacy_source_alias():
context = {
"_source": "spotify",
"artist": {"name": "Artist One", "id": "artist-1"},
"album": {"name": "Album One", "id": "album-1"},
"track_info": {"name": "Song One", "id": "track-1"},
"original_search_result": {"title": "Song One"},
}
normalized = normalize_import_context(context)
assert normalized["source"] == "spotify"
assert "_source" not in normalized
assert get_import_source(normalized) == "spotify"
def test_normalize_import_context_promotes_search_result_when_original_search_missing():
context = {
"source": "spotify",
"track_info": {"name": "Song One", "id": "track-1"},
"search_result": {
"title": "Song One",
"album": "Album One",
"artist": "Artist One",
"spotify_clean_title": "Song One",
"spotify_clean_album": "Album One",
"spotify_clean_artist": "Artist One",
},
}
normalized = normalize_import_context(context)
assert normalized["original_search_result"]["clean_title"] == "Song One"
assert normalized["original_search_result"]["clean_album"] == "Album One"
assert normalized["original_search_result"]["clean_artist"] == "Artist One"
assert "spotify_clean_title" not in normalized["original_search_result"]
assert "spotify_clean_album" not in normalized["original_search_result"]
assert "spotify_clean_artist" not in normalized["original_search_result"]
assert get_import_clean_title(normalized) == "Song One"
assert get_import_clean_album(normalized) == "Album One"
assert get_import_clean_artist(normalized) == "Artist One"
def test_neutral_import_context_helpers_work_without_legacy_aliases():
context = {
"source": "deezer",
"artist": {"name": "Artist One", "id": "artist-1"},
"album": {
"name": "Album One",
"id": "album-1",
"release_date": "2024-01-01",
"total_tracks": 12,
"album_type": "album",
"image_url": "https://img.example/album.jpg",
},
"track_info": {
"name": "Song One",
"id": "track-1",
"track_number": 3,
"disc_number": 2,
"artists": [{"name": "Artist One"}],
},
"original_search_result": {
"title": "Song One",
"artist": "Artist One",
"album": "Album One",
"clean_title": "Song One",
"clean_album": "Album One",
"clean_artist": "Artist One",
},
"has_clean_metadata": True,
"has_full_metadata": True,
}
assert get_import_clean_title(context) == "Song One"
assert get_import_clean_album(context) == "Album One"
assert get_import_clean_artist(context) == "Artist One"
assert get_import_source(context) == "deezer"
normalized = normalize_import_context(context)
assert normalized["artist"]["name"] == "Artist One"
assert normalized["album"]["name"] == "Album One"
assert normalized["original_search_result"]["clean_title"] == "Song One"
assert "spotify_artist" not in normalized
assert "spotify_album" not in normalized
assert get_import_has_clean_metadata(normalized) is True
assert get_import_has_full_metadata(normalized) is True
@pytest.mark.parametrize(
"source,expected_tags,expected_columns",
[
(
"spotify",
{"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"},
{"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"},
),
(
"itunes",
{"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"},
{"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"},
),
(
"deezer",
{"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None},
{"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"},
),
(
"hydrabase",
{"track": None, "artist": None, "album": None},
{"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"},
),
(
"discogs",
{"track": None, "artist": None, "album": None},
{"artist": "discogs_id", "album": "discogs_id", "track": None},
),
],
)
def test_source_tag_and_library_column_mappings(source, expected_tags, expected_columns):
assert get_source_tag_names(source) == expected_tags
assert get_library_source_id_columns(source) == expected_columns
def test_get_import_source_ids_prefers_nested_source_specific_ids():
context = normalize_import_context(
{
"source": "deezer",
"artist": {"deezer_id": "deezer-artist-1"},
"album": {"deezer_id": "deezer-album-1"},
"track_info": {"deezer_id": "deezer-track-1"},
"original_search_result": {"source_artist_id": "deezer-artist-1"},
}
)
assert get_import_source_ids(context) == {
"track_id": "deezer-track-1",
"artist_id": "deezer-artist-1",
"album_id": "deezer-album-1",
}
def test_build_import_album_info_uses_normalized_album_context():
context = normalize_import_context(
{
"source": "deezer",
"artist": {"name": "Artist One"},
"album": {
"name": "Album One",
"image_url": "https://img.example/album.jpg",
"release_date": "2024-05-01",
"total_tracks": 8,
"album_type": "album",
},
"track_info": {
"name": "Song One",
"track_number": 4,
"disc_number": 2,
"duration_ms": 240000,
"artists": [{"name": "Artist One"}],
},
"original_search_result": {
"title": "Song One",
"album": "Album One",
"clean_title": "Song One",
"clean_album": "Album One",
"clean_artist": "Artist One",
},
}
)
album_info = build_import_album_info(context)
assert album_info["is_album"] is True
assert album_info["album_name"] == "Album One"
assert album_info["track_number"] == 4
assert album_info["disc_number"] == 2
assert album_info["clean_track_name"] == "Song One"
assert album_info["album_image_url"] == "https://img.example/album.jpg"
assert album_info["source"] == "deezer"
@pytest.mark.parametrize(
"album_name,track_name,artist_name",
[
("Song One", "Song One", "Artist One"),
("Artist One", "Different Song", "Artist One"),
],
)
def test_detect_album_info_web_returns_none_for_ambiguous_album_names(album_name, track_name, artist_name):
context = normalize_import_context(
{
"source": "deezer",
"artist": {"name": artist_name},
"album": {"name": album_name, "total_tracks": 12, "album_type": "album"},
"track_info": {"name": track_name, "track_number": 4, "disc_number": 1},
"original_search_result": {
"title": track_name,
"clean_title": track_name,
"clean_album": album_name,
"clean_artist": artist_name,
},
}
)
assert detect_album_info_web(context) is None
def test_detect_album_info_web_forces_album_when_track_and_artist_differ():
context = normalize_import_context(
{
"source": "deezer",
"artist": {"name": "Artist One"},
"album": {
"name": "Album One",
"image_url": "https://img.example/album.jpg",
"release_date": "2024-05-01",
"total_tracks": 1,
"album_type": "album",
},
"track_info": {
"name": "Song One",
"track_number": 4,
"disc_number": 2,
"duration_ms": 240000,
"artists": [{"name": "Artist One"}],
},
"original_search_result": {
"title": "Song One",
"album": "Album One",
"clean_title": "Song One",
"clean_album": "Album One",
"clean_artist": "Artist One",
},
}
)
album_info = detect_album_info_web(context)
assert album_info is not None
assert album_info["is_album"] is True
assert album_info["confidence"] == 0.5
assert album_info["album_name"] == "Album One"
assert album_info["track_number"] == 4
assert album_info["disc_number"] == 2

View file

@ -0,0 +1,126 @@
import sys
import types
from core.imports.file_ops import (
cleanup_empty_directories,
safe_move_file,
)
from core.imports.filename import extract_track_number_from_filename
from core.imports.staging import read_staging_file_metadata
def test_extract_track_number_from_filename_handles_common_patterns():
assert extract_track_number_from_filename("01 - Song.mp3") == 1
assert extract_track_number_from_filename("1-03 - Song.mp3") == 3
assert extract_track_number_from_filename("Artist - Song.mp3") == 1
def test_safe_move_file_replaces_existing_destination(tmp_path):
src = tmp_path / "source.flac"
dst_dir = tmp_path / "dest"
dst_dir.mkdir()
dst = dst_dir / "track.flac"
src.write_text("new")
dst.write_text("old")
safe_move_file(src, dst)
assert not src.exists()
assert dst.read_text() == "new"
def test_cleanup_empty_directories_removes_nested_empty_paths(tmp_path):
download_root = tmp_path / "downloads"
nested_dir = download_root / "Artist" / "Album"
nested_dir.mkdir(parents=True)
moved_file_path = nested_dir / "track.flac"
cleanup_empty_directories(str(download_root), str(moved_file_path))
assert not nested_dir.exists()
assert not (download_root / "Artist").exists()
assert download_root.exists()
def test_read_staging_file_metadata_reads_tags(monkeypatch, tmp_path):
file_path = tmp_path / "Song One.flac"
file_path.write_text("fake")
class DummyTags:
def __init__(self):
self.values = {
"title": ["Song One"],
"artist": ["Artist One"],
"albumartist": ["Album Artist"],
"album": ["Album One"],
"tracknumber": ["03/12"],
"discnumber": ["2/3"],
}
def get(self, key, default=None):
return self.values.get(key, default)
fake_mutagen = types.ModuleType("mutagen")
fake_mutagen.File = lambda path, easy=True: DummyTags()
monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen)
metadata = read_staging_file_metadata(str(file_path), file_path.name)
assert metadata == {
"title": "Song One",
"artist": "Artist One",
"albumartist": "Album Artist",
"album": "Album One",
"track_number": 3,
"disc_number": 2,
}
def test_read_staging_file_metadata_falls_back_to_filename_track_number(monkeypatch, tmp_path):
file_path = tmp_path / "07 - Song Two.flac"
file_path.write_text("fake")
fake_mutagen = types.ModuleType("mutagen")
fake_mutagen.File = lambda path, easy=True: None
monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen)
metadata = read_staging_file_metadata(str(file_path), file_path.name)
assert metadata["title"] == "07 - Song Two"
assert metadata["track_number"] == 7
assert metadata["disc_number"] == 1
def test_read_staging_file_metadata_uses_filename_fallbacks_when_tags_are_invalid(monkeypatch, tmp_path):
file_path = tmp_path / "02 - Song Three.flac"
file_path.write_text("fake")
class DummyTags:
def __init__(self):
self.values = {
"title": [""],
"artist": "Artist One",
"albumartist": "",
"album": ["Album One"],
"tracknumber": ["not-a-number"],
"discnumber": ["bad/disc"],
}
def get(self, key, default=None):
return self.values.get(key, default)
fake_mutagen = types.ModuleType("mutagen")
fake_mutagen.File = lambda path, easy=True: DummyTags()
monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen)
metadata = read_staging_file_metadata(str(file_path), file_path.name)
assert metadata == {
"title": "02 - Song Three",
"artist": "Artist One",
"albumartist": "Artist One",
"album": "Album One",
"track_number": 2,
"disc_number": 1,
}

View file

@ -0,0 +1,33 @@
import pytest
from core.imports.filename import parse_filename_metadata
@pytest.mark.parametrize(
"filename,expected",
[
(
"01 - Artist One - Title One.mp3",
{"artist": "Artist One", "title": "Title One", "album": "", "track_number": 1},
),
(
"Artist Two - Title Two.flac",
{"artist": "Artist Two", "title": "Title Two", "album": "", "track_number": None},
),
(
r"Artist Three\Album Three\03 - Title Three.ogg",
{"artist": "", "title": "Title Three", "album": "Album Three", "track_number": 3},
),
(
"Loose Song.wav",
{"artist": "", "title": "Loose Song", "album": "", "track_number": None},
),
],
)
def test_parse_filename_metadata_handles_common_patterns(filename, expected):
parsed = parse_filename_metadata(filename)
assert parsed["artist"] == expected["artist"]
assert parsed["title"] == expected["title"]
assert parsed["album"] == expected["album"]
assert parsed["track_number"] == expected["track_number"]

View file

@ -0,0 +1,47 @@
from types import SimpleNamespace
from core.imports import guards
class _FakeDB:
def __init__(self, quality_profile):
self._quality_profile = quality_profile
def get_quality_profile(self):
return self._quality_profile
def test_check_flac_bit_depth_rejects_strict_mismatch(monkeypatch):
monkeypatch.setattr(
guards,
"MusicDatabase",
lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": False}}}),
)
monkeypatch.setattr(
guards,
"_get_config_manager",
lambda: SimpleNamespace(get=lambda _key, default=None: False),
)
context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}}
assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) == (
"FLAC bit depth mismatch: file is 24-bit, preference is 16-bit"
)
def test_check_flac_bit_depth_allows_fallback_when_enabled(monkeypatch):
monkeypatch.setattr(
guards,
"MusicDatabase",
lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": True}}}),
)
monkeypatch.setattr(
guards,
"_get_config_manager",
lambda: SimpleNamespace(get=lambda _key, default=None: False),
)
context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}}
assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) is None

View file

@ -0,0 +1,232 @@
import os
import core.imports.album_naming as album_naming
import core.imports.paths as import_paths
class _Config:
def __init__(self, values):
self._values = values
def get(self, key, default=None):
return self._values.get(key, default)
def test_sanitize_filename_replaces_illegal_characters():
assert import_paths.sanitize_filename("AC/DC: Song?") == "AC_DC_ Song_"
assert import_paths.sanitize_filename("AUX.txt").startswith("_")
def test_build_simple_download_destination_uses_album_folder(monkeypatch, tmp_path):
config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")})
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config)
destination, album_name, filename = import_paths.build_simple_download_destination(
{
"search_result": {
"filename": "Album Folder/source.flac",
"album": "Album Folder",
}
},
str(tmp_path / "source.flac"),
)
assert destination == tmp_path / "Transfer" / "Album Folder" / "source.flac"
assert album_name == "Album Folder"
assert filename == "source.flac"
assert destination.parent.exists()
def test_build_simple_download_destination_falls_back_to_transfer_root(monkeypatch, tmp_path):
config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")})
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config)
destination, album_name, filename = import_paths.build_simple_download_destination(
{
"search_result": {
"filename": "source.flac",
"album": "Unknown Album",
}
},
str(tmp_path / "source.flac"),
)
assert destination == tmp_path / "Transfer" / "source.flac"
assert album_name == ""
assert filename == "source.flac"
assert destination.parent.exists()
def test_get_file_path_from_template_raw_handles_quality_and_disc_placeholders(monkeypatch):
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config({}))
folder_path, filename = import_paths.get_file_path_from_template_raw(
"$artist/$album/$discnum - $title [$quality]",
{
"artist": "Artist One",
"album": "Album One",
"title": "Song One",
"quality": "FLAC 16bit",
"disc_number": 3,
},
)
assert folder_path == os.path.join("Artist One", "Album One")
assert filename == "3 - Song One [FLAC 16bit]"
def test_resolve_album_group_upgrades_standard_to_deluxe():
album_naming.clear_album_grouping_cache()
artist_context = {"name": "Cache Artist"}
standard_album = {"album_name": "Cache Album"}
deluxe_album = {"album_name": "Cache Album (Deluxe Edition)"}
assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album"
assert import_paths.resolve_album_group(artist_context, deluxe_album) == "Cache Album (Deluxe Edition)"
assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album (Deluxe Edition)"
def test_build_final_path_for_track_uses_template_and_disc_folder(monkeypatch, tmp_path):
config = _Config(
{
"soulseek.transfer_path": str(tmp_path / "Transfer"),
"file_organization.enabled": True,
"file_organization.templates": {
"album_path": "$albumartist/$albumartist - $album/$track - $title",
"single_path": "$artist/$artist - $title",
},
"file_organization.collab_artist_mode": "first",
"file_organization.disc_label": "Disc",
}
)
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config)
calls = []
def fake_get_album_tracks_for_source(source, album_id):
calls.append((source, album_id))
return {
"items": [
{"disc_number": 1},
{"disc_number": 2},
]
}
monkeypatch.setattr(import_paths, "_get_album_tracks_for_source", fake_get_album_tracks_for_source)
context = {
"artist": {"name": "Artist One"},
"album": {
"name": "Album One",
"id": "album-1",
"release_date": "2026-01-01",
"total_tracks": 12,
"album_type": "album",
"artists": [{"name": "Artist One"}],
},
"track_info": {
"name": "Song One",
"id": "track-1",
"track_number": 4,
"disc_number": 1,
"artists": [{"name": "Artist One"}],
},
"original_search_result": {
"title": "Song One",
"clean_title": "Song One",
"clean_album": "Album One",
"clean_artist": "Artist One",
"artists": [{"name": "Artist One"}],
},
"source": "deezer",
"is_album_download": False,
}
final_path, created = import_paths.build_final_path_for_track(
context,
{"name": "Artist One"},
{
"is_album": True,
"album_name": "Album One",
"track_number": 4,
"disc_number": 1,
},
".flac",
)
assert created is True
assert calls == [("deezer", "album-1")]
assert final_path == str(
tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1" / "04 - Song One.flac"
)
assert (tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1").is_dir()
def test_build_final_path_for_track_uses_track_disc_number_without_provider_lookup(monkeypatch, tmp_path):
config = _Config(
{
"soulseek.transfer_path": str(tmp_path / "Transfer"),
"file_organization.enabled": True,
"file_organization.templates": {
"album_path": "$albumartist/$albumartist - $album/$track - $title",
"single_path": "$artist/$artist - $title",
},
"file_organization.collab_artist_mode": "first",
"file_organization.disc_label": "Disc",
}
)
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config)
calls = []
monkeypatch.setattr(
import_paths,
"_get_album_tracks_for_source",
lambda source, album_id: calls.append((source, album_id)) or None,
)
context = {
"artist": {"name": "Artist One"},
"album": {
"name": "Album One",
"id": "album-1",
"release_date": "2026-01-01",
"total_tracks": 12,
"album_type": "album",
"artists": [{"name": "Artist One"}],
},
"track_info": {
"name": "Song Two",
"id": "track-2",
"track_number": 4,
"disc_number": 2,
"artists": [{"name": "Artist One"}],
},
"original_search_result": {
"title": "Song Two",
"clean_title": "Song Two",
"clean_album": "Album One",
"clean_artist": "Artist One",
"artists": [{"name": "Artist One"}],
},
"source": "deezer",
"is_album_download": False,
}
final_path, created = import_paths.build_final_path_for_track(
context,
{"name": "Artist One"},
{
"is_album": True,
"album_name": "Album One",
"track_number": 4,
"disc_number": 2,
},
".flac",
)
assert created is True
assert calls == []
assert final_path == str(
tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 2" / "04 - Song Two.flac"
)

View file

@ -0,0 +1,212 @@
import logging
import sys
import types
import core.imports.pipeline as import_pipeline
import core.imports.paths as import_paths
import core.runtime_state as runtime_state
class _Config:
def __init__(self, transfer_path):
self.transfer_path = transfer_path
def get(self, key, default=None):
if key == "soulseek.transfer_path":
return self.transfer_path
if key in {"post_processing.replaygain_enabled", "lossy_copy.enabled", "lossy_copy.delete_original", "import.replace_lower_quality"}:
return False
return default
class _FakeAcoustidVerifier:
def quick_check_available(self):
return False, "disabled"
class _ImmediateThread:
def __init__(self, target=None, daemon=None):
self._target = target
def start(self):
if self._target:
self._target()
def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch):
transfer_root = tmp_path / "Transfer"
transfer_root.mkdir()
source_path = tmp_path / "source.flac"
source_path.write_bytes(b"audio")
context_key = "ctx-1"
task_id = "task-1"
batch_id = "batch-1"
context = {
"search_result": {
"is_simple_download": True,
"filename": "Album Folder/source.flac",
"album": "Album Folder",
},
"track_info": {},
"original_search_result": {},
"is_album_download": False,
"task_id": task_id,
"batch_id": batch_id,
}
mark_calls = []
completion_calls = []
scan_calls = []
activity_calls = []
wishlist_calls = []
original_matched_context = dict(runtime_state.matched_downloads_context)
original_download_tasks = dict(runtime_state.download_tasks)
original_download_batches = dict(runtime_state.download_batches)
original_processed_ids = set(runtime_state.processed_download_ids)
original_post_locks = dict(runtime_state.post_process_locks)
runtime_state.matched_downloads_context.clear()
runtime_state.download_tasks.clear()
runtime_state.download_batches.clear()
runtime_state.processed_download_ids.clear()
runtime_state.post_process_locks.clear()
runtime = types.SimpleNamespace(
automation_engine=None,
on_download_completed=lambda batch, task, success: completion_calls.append((batch, task, success)),
web_scan_manager=types.SimpleNamespace(request_scan=lambda reason: scan_calls.append(reason)),
repair_worker=None,
)
fake_acoustid = types.ModuleType("core.acoustid_verification")
fake_acoustid.AcoustIDVerification = _FakeAcoustidVerifier
fake_acoustid.VerificationResult = types.SimpleNamespace(FAIL="FAIL")
monkeypatch.setitem(sys.modules, "core.acoustid_verification", fake_acoustid)
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config(str(transfer_root)))
monkeypatch.setattr(import_pipeline, "add_activity_item", lambda *args, **kwargs: activity_calls.append((args, kwargs)))
monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_library_history_download", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_download_provenance", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "check_and_remove_from_wishlist", lambda context: wishlist_calls.append(dict(context)))
monkeypatch.setattr(import_pipeline, "_mark_task_completed", lambda task, track_info: mark_calls.append((task, track_info)))
monkeypatch.setattr(import_pipeline.threading, "Thread", _ImmediateThread)
runtime_state.matched_downloads_context[context_key] = context
runtime_state.download_tasks[task_id] = {"track_info": {}, "status": "running"}
try:
import_pipeline.post_process_matched_download_with_verification(
context_key,
context,
str(source_path),
task_id,
batch_id,
runtime,
)
expected_path = transfer_root / "Album Folder" / "source.flac"
assert expected_path.exists()
assert not source_path.exists()
assert context["_simple_download_completed"] is True
assert context["_final_path"] == str(expected_path)
assert mark_calls == [(task_id, {})]
assert completion_calls == [(batch_id, task_id, True)]
assert context_key not in runtime_state.matched_downloads_context
assert scan_calls == ["Simple download completed"]
assert wishlist_calls and wishlist_calls[0]["search_result"]["is_simple_download"] is True
assert activity_calls
finally:
runtime_state.matched_downloads_context.clear()
runtime_state.matched_downloads_context.update(original_matched_context)
runtime_state.download_tasks.clear()
runtime_state.download_tasks.update(original_download_tasks)
runtime_state.download_batches.clear()
runtime_state.download_batches.update(original_download_batches)
runtime_state.processed_download_ids.clear()
runtime_state.processed_download_ids.update(original_processed_ids)
runtime_state.post_process_locks.clear()
runtime_state.post_process_locks.update(original_post_locks)
def test_post_process_matched_download_forwards_separate_metadata_runtime(tmp_path, monkeypatch):
source_path = tmp_path / "source.flac"
source_path.write_bytes(b"audio")
target_path = tmp_path / "Album Folder" / "track.flac"
runtime = types.SimpleNamespace(
automation_engine=None,
on_download_completed=None,
web_scan_manager=None,
repair_worker=None,
)
metadata_runtime = types.SimpleNamespace(marker="metadata-runtime")
seen = {}
monkeypatch.setattr(import_pipeline, "config_manager", types.SimpleNamespace(
get=lambda key, default=None: {
"post_processing.replaygain_enabled": False,
"lossy_copy.enabled": False,
"lossy_copy.delete_original": False,
"import.replace_lower_quality": False,
"soulseek.download_path": str(tmp_path / "downloads"),
}.get(key, default)
))
monkeypatch.setattr(import_pipeline, "normalize_import_context", lambda context: context)
monkeypatch.setattr(import_pipeline, "get_import_track_info", lambda context: {"_playlist_folder_mode": True, "_playlist_name": "Playlist"})
monkeypatch.setattr(import_pipeline, "get_import_original_search", lambda context: {"title": "Track", "album": "Album"})
monkeypatch.setattr(import_pipeline, "get_import_context_artist", lambda context: {"name": "Artist"})
monkeypatch.setattr(import_pipeline, "get_import_has_clean_metadata", lambda context: True)
monkeypatch.setattr(
import_pipeline,
"build_import_album_info",
lambda context, force_album=False: {
"is_album": True,
"album_name": "Album",
"track_number": 1,
"disc_number": 1,
"clean_track_name": "Track",
"source": "spotify",
},
)
monkeypatch.setattr(import_pipeline, "resolve_album_group", lambda artist_context, album_info, original_album: album_info["album_name"])
monkeypatch.setattr(import_pipeline, "get_import_clean_title", lambda *args, **kwargs: "Track")
monkeypatch.setattr(import_pipeline, "get_audio_quality_string", lambda file_path: "")
monkeypatch.setattr(import_pipeline, "check_flac_bit_depth", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "build_final_path_for_track", lambda *args, **kwargs: (str(target_path), None))
def _capture_enhance(file_path, context, artist, album_info, runtime=None):
seen["runtime"] = runtime
return True
monkeypatch.setattr(import_pipeline, "enhance_file_metadata", _capture_enhance)
monkeypatch.setattr(import_pipeline, "safe_move_file", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "download_cover_art", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "generate_lrc_file", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "downsample_hires_flac", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "create_lossy_copy", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "cleanup_empty_directories", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_library_history_download", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_download_provenance", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_soulsync_library_entry", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "check_and_remove_from_wishlist", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_retag_download", lambda *args, **kwargs: None)
context = {
"track_info": {"_playlist_folder_mode": True, "_playlist_name": "Playlist"},
"original_search_result": {"title": "Track", "album": "Album"},
"is_album_download": False,
}
import_pipeline.post_process_matched_download(
"ctx-1",
context,
str(source_path),
runtime,
metadata_runtime=metadata_runtime,
)
assert seen["runtime"] is metadata_runtime

View file

@ -0,0 +1,217 @@
from types import SimpleNamespace
from core import metadata_service
from core.imports import resolution
class FakeClient:
def __init__(self, search_results=None, details=None, artist_details=None):
self.search_results = search_results or []
self.details = details or {}
self.artist_details = artist_details or {}
self.calls = []
def search_tracks(self, query, limit=1, allow_fallback=True):
self.calls.append(("search_tracks", query, limit, allow_fallback))
return self.search_results
def get_track_details(self, track_id):
self.calls.append(("get_track_details", track_id))
return self.details.get(str(track_id))
def get_artist(self, artist_id):
self.calls.append(("get_artist", artist_id))
return self.artist_details.get(str(artist_id))
def _track_result(track_id="track-1", name="Song One", artist="Artist One"):
return SimpleNamespace(
id=track_id,
name=name,
artists=[artist],
album="Album One",
duration_ms=123000,
track_number=1,
disc_number=1,
image_url="https://img.example/track.jpg",
)
def _track_details(source, track_id="track-1", name="Song One", artist_name="Artist One", artist_id="artist-1"):
return {
"id": track_id,
"name": name,
"track_number": 7,
"disc_number": 1,
"duration_ms": 210000,
"explicit": True,
"uri": f"{source}:track:{track_id}",
"artists": [{"name": artist_name, "id": artist_id}],
"album": {
"id": f"{source}-album-1",
"name": "Album One",
"release_date": "2024-01-01",
"album_type": "album",
"total_tracks": 10,
"images": [{"url": f"https://img.example/{source}-album.jpg"}],
"artists": [{"name": artist_name, "id": artist_id}],
},
}
def test_get_single_track_import_context_uses_primary_source_priority(monkeypatch):
deezer_client = FakeClient(
search_results=[_track_result(track_id="deezer-track-1")],
details={"deezer-track-1": _track_details("deezer", track_id="deezer-track-1", artist_name="Artist One", artist_id="deezer-artist-1")},
artist_details={"deezer-artist-1": {"id": "deezer-artist-1", "genres": ["electronic"]}},
)
spotify_client = FakeClient()
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
monkeypatch.setattr(
metadata_service,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source),
)
result = resolution.get_single_track_import_context("Song One", "Artist One")
assert result["success"] is True
assert result["source"] == "deezer"
assert result["source_priority"] == ["deezer", "spotify", "itunes"]
assert result["context"]["track_info"]["name"] == "Song One"
assert result["context"]["track_info"]["album_id"] == "deezer-album-1"
assert result["context"]["album"]["image_url"] == "https://img.example/deezer-album.jpg"
assert result["context"]["artist"]["genres"] == ["electronic"]
assert result["context"]["original_search_result"]["clean_title"] == "Song One"
assert deezer_client.calls == [
('search_tracks', 'artist:"Artist One" track:"Song One"', 5, True),
("get_track_details", "deezer-track-1"),
("get_artist", "deezer-artist-1"),
]
assert spotify_client.calls == []
def test_get_single_track_import_context_falls_back_to_next_source(monkeypatch):
deezer_client = FakeClient(search_results=[])
spotify_client = FakeClient(
search_results=[_track_result(track_id="spotify-track-1")],
details={"spotify-track-1": _track_details("spotify", track_id="spotify-track-1", artist_name="Artist Two", artist_id="spotify-artist-1")},
artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["indie"]}},
)
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
monkeypatch.setattr(
metadata_service,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source),
)
result = resolution.get_single_track_import_context("Song Two", "Artist Two")
assert result["success"] is True
assert result["source"] == "spotify"
assert result["context"]["track_info"]["id"] == "spotify-track-1"
assert result["context"]["album"]["name"] == "Album One"
assert deezer_client.calls == [
('search_tracks', 'artist:"Artist Two" track:"Song Two"', 5, True),
('search_tracks', 'Song Two', 5, True),
('search_tracks', 'Artist Two', 5, True),
]
assert spotify_client.calls == [
("search_tracks", "Song Two Artist Two", 5, False),
("get_track_details", "spotify-track-1"),
("get_artist", "spotify-artist-1"),
]
def test_get_single_track_import_context_uses_explicit_override_first(monkeypatch):
spotify_client = FakeClient(
details={"override-track-1": _track_details("spotify", track_id="override-track-1", artist_name="Override Artist", artist_id="spotify-artist-1")},
artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["pop"]}},
)
deezer_client = FakeClient()
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
monkeypatch.setattr(
metadata_service,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source),
)
result = resolution.get_single_track_import_context(
"Ignored Title",
"Ignored Artist",
override_id="override-track-1",
)
assert result["success"] is True
assert result["source"] == "spotify"
assert result["context"]["track_info"]["id"] == "override-track-1"
assert result["context"]["artist"]["genres"] == ["pop"]
assert spotify_client.calls == [
("get_track_details", "override-track-1"),
("get_artist", "spotify-artist-1"),
]
assert deezer_client.calls == []
def test_get_single_track_import_context_uses_explicit_override_source(monkeypatch):
itunes_client = FakeClient(
details={"override-track-2": _track_details("itunes", track_id="override-track-2", artist_name="Override Artist Two", artist_id="itunes-artist-1")},
artist_details={"itunes-artist-1": {"id": "itunes-artist-1", "genres": ["singer-songwriter"]}},
)
spotify_client = FakeClient()
deezer_client = FakeClient()
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
monkeypatch.setattr(
metadata_service,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": itunes_client}.get(source),
)
result = resolution.get_single_track_import_context(
"Ignored Title",
"Ignored Artist",
override_id="override-track-2",
override_source="itunes",
)
assert result["success"] is True
assert result["source"] == "itunes"
assert result["context"]["track_info"]["id"] == "override-track-2"
assert result["context"]["artist"]["genres"] == ["singer-songwriter"]
assert itunes_client.calls == [
("get_track_details", "override-track-2"),
("get_artist", "itunes-artist-1"),
]
assert deezer_client.calls == []
assert spotify_client.calls == []
def test_get_single_track_import_context_returns_fallback_payload_when_no_source_matches(monkeypatch):
deezer_client = FakeClient()
spotify_client = FakeClient()
itunes_client = FakeClient()
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
monkeypatch.setattr(
metadata_service,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": itunes_client}.get(source),
)
result = resolution.get_single_track_import_context("Missing Song", "Missing Artist")
assert result["success"] is False
assert result["source"] is None
assert result["context"]["artist"]["name"] == "Missing Artist"
assert result["context"]["track_info"]["name"] == "Missing Song"
assert result["context"]["original_search_result"]["clean_title"] == "Missing Song"
assert result["context"]["original_search_result"]["clean_artist"] == "Missing Artist"

View file

@ -0,0 +1,187 @@
import sqlite3
from types import SimpleNamespace
from core.imports import side_effects
class _FakeDB:
def __init__(self, conn):
self._conn = conn
def _get_connection(self):
return self._conn
class _FakeWishlistService:
def __init__(self, tracks):
self.tracks = tracks
self.removed = []
def get_wishlist_tracks_for_download(self, profile_id=1):
return list(self.tracks)
def mark_track_download_result(self, spotify_track_id, success, error_message=None, profile_id=1):
self.removed.append((spotify_track_id, success, error_message, profile_id))
return True
def _make_soulsync_db():
conn = sqlite3.connect(":memory:")
conn.row_factory = sqlite3.Row
conn.execute(
"""
CREATE TABLE artists (
id TEXT PRIMARY KEY,
name TEXT,
genres TEXT,
thumb_url TEXT,
server_source TEXT,
created_at TEXT,
updated_at TEXT,
spotify_artist_id TEXT
)
"""
)
conn.execute(
"""
CREATE TABLE albums (
id TEXT PRIMARY KEY,
artist_id TEXT,
title TEXT,
year INTEGER,
thumb_url TEXT,
genres TEXT,
track_count INTEGER,
duration INTEGER,
server_source TEXT,
created_at TEXT,
updated_at TEXT,
spotify_album_id TEXT
)
"""
)
conn.execute(
"""
CREATE TABLE tracks (
id TEXT PRIMARY KEY,
album_id TEXT,
artist_id TEXT,
title TEXT,
track_number INTEGER,
duration INTEGER,
file_path TEXT,
bitrate INTEGER,
track_artist TEXT,
server_source TEXT,
created_at TEXT,
updated_at TEXT,
spotify_track_id TEXT
)
"""
)
return conn
def test_record_soulsync_library_entry_writes_artist_album_and_track(tmp_path, monkeypatch):
conn = _make_soulsync_db()
fake_db = _FakeDB(conn)
final_path = tmp_path / "track.flac"
final_path.write_bytes(b"audio")
monkeypatch.setattr(side_effects, "get_database", lambda: fake_db)
monkeypatch.setattr(
side_effects,
"_get_config_manager",
lambda: SimpleNamespace(get_active_media_server=lambda: "soulsync"),
)
import core.genre_filter as genre_filter
monkeypatch.setattr(genre_filter, "filter_genres", lambda genres, _cfg: [genre.upper() for genre in genres])
context = {
"source": "spotify",
"artist": {"id": "sp-artist", "name": "Artist One"},
"album": {
"id": "sp-album",
"name": "Album One",
"release_date": "2024-02-03",
"total_tracks": 12,
"image_url": "https://img.example/album.jpg",
},
"track_info": {
"id": "sp-track",
"name": "Song One",
"track_number": 7,
"duration_ms": 210000,
"artists": [{"name": "Guest Artist"}],
"_source": "spotify",
},
"original_search_result": {
"title": "Song One",
"artists": [{"name": "Guest Artist"}],
"_source": "spotify",
},
"_final_processed_path": str(final_path),
}
artist_context = {"name": "Artist One", "genres": ["rock", "indie"]}
album_info = {"is_album": True, "album_name": "Album One", "track_number": 7}
side_effects.record_soulsync_library_entry(context, artist_context, album_info)
artist_row = conn.execute("SELECT * FROM artists").fetchone()
album_row = conn.execute("SELECT * FROM albums").fetchone()
track_row = conn.execute("SELECT * FROM tracks").fetchone()
assert artist_row["name"] == "Artist One"
assert artist_row["server_source"] == "soulsync"
assert artist_row["spotify_artist_id"] == "sp-artist"
assert artist_row["genres"] == '["ROCK", "INDIE"]'
assert album_row["title"] == "Album One"
assert album_row["server_source"] == "soulsync"
assert album_row["spotify_album_id"] == "sp-album"
assert album_row["year"] == 2024
assert album_row["track_count"] == 12
assert album_row["duration"] == 210000
assert album_row["artist_id"] == artist_row["id"]
assert track_row["title"] == "Song One"
assert track_row["server_source"] == "soulsync"
assert track_row["spotify_track_id"] == "sp-track"
assert track_row["track_number"] == 7
assert track_row["duration"] == 210000
assert track_row["track_artist"] == "Guest Artist"
assert track_row["album_id"] == album_row["id"]
assert track_row["file_path"] == str(final_path)
def test_check_and_remove_from_wishlist_uses_search_result_fallback(monkeypatch):
fake_db = SimpleNamespace(get_all_profiles=lambda: [{"id": 1}])
wishlist_service = _FakeWishlistService([
{
"wishlist_id": 11,
"spotify_track_id": "sp-track-1",
"id": "sp-track-1",
"name": "Song One",
"artists": [{"name": "Artist One"}],
}
])
monkeypatch.setattr(side_effects, "get_database", lambda: fake_db)
monkeypatch.setattr(side_effects, "get_wishlist_service", lambda: wishlist_service)
context = {
"search_result": {
"title": "Song One",
"artist": "Artist One",
"album": "Album One",
},
"track_info": {},
"original_search_result": {},
}
side_effects.check_and_remove_from_wishlist(context)
assert wishlist_service.removed == [("sp-track-1", True, None, 1)]

View file

@ -0,0 +1,265 @@
from types import SimpleNamespace
import core.imports.staging as import_staging
class FakeClient:
def __init__(self, results=None):
self.results = results or []
self.calls = []
def search_albums(self, query, **kwargs):
self.calls.append((query, kwargs))
return self.results
def search_tracks(self, query, **kwargs):
self.calls.append((query, kwargs))
return self.results
def _album_result(album_id, name, artist, release_date="2024-01-01", total_tracks=10, image_url="https://img.example/album.jpg", album_type="album"):
return SimpleNamespace(
id=album_id,
name=name,
artists=[artist],
release_date=release_date,
total_tracks=total_tracks,
image_url=image_url,
album_type=album_type,
)
def test_search_import_albums_prefers_primary_source(monkeypatch):
deezer_client = FakeClient([
_album_result("deezer-1", "Album One", "Artist One"),
])
spotify_client = FakeClient([
_album_result("spotify-1", "Album One", "Artist One"),
])
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
monkeypatch.setattr(
import_staging,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
)
monkeypatch.setattr(
import_staging,
"_search_albums_for_source",
lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit),
)
results = import_staging.search_import_albums("Album One", limit=2)
assert results == [
{
"id": "deezer-1",
"name": "Album One",
"artist": "Artist One",
"release_date": "2024-01-01",
"total_tracks": 10,
"image_url": "https://img.example/album.jpg",
"album_type": "album",
"source": "deezer",
}
]
assert deezer_client.calls == [("Album One", {"limit": 2})]
assert spotify_client.calls == []
def test_search_import_albums_falls_back_when_primary_has_no_results(monkeypatch):
deezer_client = FakeClient([])
spotify_client = FakeClient([
_album_result("spotify-1", "Album Two", "Artist Two"),
])
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
monkeypatch.setattr(
import_staging,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
)
monkeypatch.setattr(
import_staging,
"_search_albums_for_source",
lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit),
)
results = import_staging.search_import_albums("Album Two", limit=2)
assert results == [
{
"id": "spotify-1",
"name": "Album Two",
"artist": "Artist Two",
"release_date": "2024-01-01",
"total_tracks": 10,
"image_url": "https://img.example/album.jpg",
"album_type": "album",
"source": "spotify",
}
]
assert deezer_client.calls == [("Album Two", {"limit": 2})]
assert spotify_client.calls == [("Album Two", {"limit": 2, "allow_fallback": False})]
def test_search_import_tracks_prefers_primary_source(monkeypatch):
deezer_client = FakeClient([
SimpleNamespace(
id="deezer-track-1",
name="Song One",
artists=[{"name": "Artist One"}],
album={"id": "deezer-album-1", "name": "Album One"},
duration_ms=210000,
image_url="https://img.example/track.jpg",
track_number=7,
),
])
spotify_client = FakeClient([
SimpleNamespace(
id="spotify-track-1",
name="Song One",
artists=["Artist One"],
album="Album One",
duration_ms=210000,
image_url="https://img.example/track.jpg",
track_number=7,
),
])
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
monkeypatch.setattr(
import_staging,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
)
monkeypatch.setattr(
import_staging,
"_search_tracks_for_source",
lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit),
)
results = import_staging.search_import_tracks("Song One", limit=2)
assert results == [
{
"id": "deezer-track-1",
"name": "Song One",
"artist": "Artist One",
"album": "Album One",
"album_id": "deezer-album-1",
"duration_ms": 210000,
"image_url": "https://img.example/track.jpg",
"track_number": 7,
"source": "deezer",
}
]
assert deezer_client.calls == [("Song One", {"limit": 2})]
assert spotify_client.calls == []
def test_search_import_tracks_falls_back_when_primary_has_no_results(monkeypatch):
deezer_client = FakeClient([])
spotify_client = FakeClient([
SimpleNamespace(
id="spotify-track-1",
name="Song Two",
artists=["Artist Two"],
album="Album Two",
duration_ms=180000,
image_url="",
track_number=3,
),
])
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
monkeypatch.setattr(
import_staging,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
)
monkeypatch.setattr(
import_staging,
"_search_tracks_for_source",
lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit),
)
results = import_staging.search_import_tracks("Song Two", limit=2)
assert results == [
{
"id": "spotify-track-1",
"name": "Song Two",
"artist": "Artist Two",
"album": "Album Two",
"album_id": "",
"duration_ms": 180000,
"image_url": "",
"track_number": 3,
"source": "spotify",
}
]
assert deezer_client.calls == [("Song Two", {"limit": 2})]
assert spotify_client.calls == [("Song Two", {"limit": 2, "allow_fallback": False})]
def test_build_import_suggestions_background_uses_collected_queries(monkeypatch, tmp_path):
staging_root = tmp_path / "Staging"
staging_root.mkdir()
cache = import_staging.get_import_suggestions_cache()
original_cache = dict(cache)
cache["suggestions"] = []
cache["building"] = False
cache["built"] = False
monkeypatch.setattr(import_staging, "get_staging_path", lambda: str(staging_root))
monkeypatch.setattr(import_staging, "_collect_import_suggestion_queries", lambda staging_path: ["Album One", "Folder Hint"])
monkeypatch.setattr(
import_staging,
"search_import_albums",
lambda query, limit=2: [
{
"id": query.lower().replace(" ", "-"),
"name": query,
"artist": f"{query} Artist",
"release_date": "2024-02-01",
"total_tracks": 12,
"image_url": "",
"album_type": "album",
}
],
)
try:
import_staging._build_import_suggestions_background()
assert cache["built"] is True
assert cache["building"] is False
assert cache["suggestions"] == [
{
"id": "album-one",
"name": "Album One",
"artist": "Album One Artist",
"release_date": "2024-02-01",
"total_tracks": 12,
"image_url": "",
"album_type": "album",
},
{
"id": "folder-hint",
"name": "Folder Hint",
"artist": "Folder Hint Artist",
"release_date": "2024-02-01",
"total_tracks": 12,
"image_url": "",
"album_type": "album",
},
]
finally:
cache.clear()
cache.update(original_cache)

View file

@ -0,0 +1,595 @@
from collections import OrderedDict
import types
import sqlite3
import pytest
import requests
from core.metadata import enrichment as me
from core.metadata import artwork as ma
from core.metadata import source as ms
class _Config:
def __init__(self, values=None):
self.values = values or {}
def get(self, key, default=None):
return self.values.get(key, default)
class _FakeTag:
def __init__(self, kind, **kwargs):
self.kind = kind
self.kwargs = kwargs
class _FakeID3Tags:
def __init__(self):
self.added = []
def add(self, frame):
self.added.append(frame)
def clear(self):
self.added.clear()
def getall(self, _key):
return []
def keys(self):
return [frame.kind for frame in self.added]
def __len__(self):
return len(self.added)
class _FakeAudio:
def __init__(self):
self.tags = _FakeID3Tags()
self.save_calls = []
self.clear_pictures_calls = 0
def clear_pictures(self):
self.clear_pictures_calls += 1
def add_tags(self):
self.tags = _FakeID3Tags()
def save(self, **kwargs):
self.save_calls.append(kwargs)
class _FakeResponse:
def __init__(self, payload, content_type="image/jpeg"):
self._payload = payload
self._content_type = content_type
def read(self):
return self._payload
def info(self):
return types.SimpleNamespace(get_content_type=lambda: self._content_type)
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
class _FileDB:
def __init__(self, db_path):
self.db_path = db_path
def _get_connection(self):
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
return conn
def _fake_symbols(audio):
def _tag_factory(kind):
return lambda **kwargs: _FakeTag(kind, **kwargs)
return types.SimpleNamespace(
File=lambda _path: audio,
APEv2=type("FakeAPEv2", (), {}),
APENoHeaderError=Exception,
FLAC=type("FakeFLAC", (), {}),
Picture=type("FakePicture", (), {"__init__": lambda self: None}),
ID3=_FakeID3Tags,
APIC=_tag_factory("APIC"),
TBPM=_tag_factory("TBPM"),
TCOP=_tag_factory("TCOP"),
TDOR=_tag_factory("TDOR"),
TDRC=_tag_factory("TDRC"),
TCON=_tag_factory("TCON"),
TIT2=_tag_factory("TIT2"),
TALB=_tag_factory("TALB"),
TPE1=_tag_factory("TPE1"),
TPE2=_tag_factory("TPE2"),
TPOS=_tag_factory("TPOS"),
TPUB=_tag_factory("TPUB"),
TRCK=_tag_factory("TRCK"),
TSRC=_tag_factory("TSRC"),
TXXX=_tag_factory("TXXX"),
UFID=_tag_factory("UFID"),
TMED=_tag_factory("TMED"),
MP4=type("FakeMP4", (), {}),
MP4Cover=types.SimpleNamespace(FORMAT_JPEG=1, FORMAT_PNG=2, __call__=lambda *args, **kwargs: ("cover", args, kwargs)),
MP4FreeForm=lambda data: ("freeform", data),
OggVorbis=type("FakeOggVorbis", (), {}),
OggOpus=None,
)
def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_for_non_itunes_sources(monkeypatch):
monkeypatch.setattr(ms, "get_config_manager", lambda: _Config({"file_organization.collab_artist_mode": "first"}))
monkeypatch.setattr(ms, "get_itunes_client", lambda: (_ for _ in ()).throw(AssertionError("itunes fallback should not run for non-itunes sources")))
context = {
"source": "spotify",
"artist": {"name": "Artist One & Artist Two", "id": "123", "genres": ["rock", "indie"]},
"album": {
"name": "Album One",
"total_tracks": 12,
"release_date": "2024-01-02",
"images": [{"url": "https://img.example/album.jpg"}],
},
"track_info": {
"artists": [{"name": "Artist One"}],
"_source": "spotify",
"track_number": 3,
"disc_number": 2,
"total_tracks": 12,
},
"original_search_result": {
"title": "Song One",
"artists": [{"name": "Artist One"}],
"clean_title": "Song One",
"clean_album": "Album One",
"clean_artist": "Artist One",
"disc_number": 2,
"duration_ms": 180000,
},
}
metadata = me.extract_source_metadata(
context,
context["artist"],
{"is_album": True, "album_name": "Album One", "track_number": 3, "disc_number": 2, "album_image_url": "https://img.example/album.jpg"},
)
assert metadata["source"] == "spotify"
assert metadata["title"] == "Song One"
assert metadata["artist"] == "Artist One"
assert metadata["album_artist"] == "Artist One & Artist Two"
assert metadata["album"] == "Album One"
assert metadata["track_number"] == 3
assert metadata["total_tracks"] == 12
assert metadata["disc_number"] == 2
assert metadata["album_art_url"] == "https://img.example/album.jpg"
def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatch):
audio = _FakeAudio()
symbols = _fake_symbols(audio)
monkeypatch.setattr(ms, "get_config_manager", lambda: _Config())
monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols)
monkeypatch.setattr(ms, "get_database", lambda: None)
current_metadata = {
"source": "deezer",
"source_track_id": "dz-track",
"source_artist_id": "dz-artist",
"source_album_id": "dz-album",
"title": "Song One",
"artist": "Artist One",
"album_artist": "Artist One",
"album": "Album One",
}
me.embed_source_ids(audio, current_metadata, context={"track_info": {}, "original_search_result": {}})
current_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"]
assert "DEEZER_TRACK_ID" in current_descs
assert "DEEZER_ARTIST_ID" in current_descs
audio = _FakeAudio()
symbols = _fake_symbols(audio)
monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols)
legacy_metadata = {
"source": "",
"spotify_track_id": "sp-track",
"spotify_artist_id": "sp-artist",
"spotify_album_id": "sp-album",
"itunes_track_id": "it-track",
"itunes_artist_id": "it-artist",
"itunes_album_id": "it-album",
"title": "Song One",
"artist": "Artist One",
"album_artist": "Artist One",
"album": "Album One",
}
me.embed_source_ids(audio, legacy_metadata, context={"track_info": {}, "original_search_result": {}})
legacy_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"]
assert "SPOTIFY_TRACK_ID" in legacy_descs
assert "SPOTIFY_ARTIST_ID" in legacy_descs
assert "SPOTIFY_ALBUM_ID" in legacy_descs
assert "ITUNES_TRACK_ID" in legacy_descs
assert "ITUNES_ARTIST_ID" in legacy_descs
assert "ITUNES_ALBUM_ID" in legacy_descs
def test_embed_source_ids_skips_disabled_source_specific_tags(monkeypatch):
audio = _FakeAudio()
symbols = _fake_symbols(audio)
monkeypatch.setattr(
ms,
"get_config_manager",
lambda: _Config({"deezer.embed_tags": False}),
)
monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols)
monkeypatch.setattr(ms, "get_database", lambda: None)
metadata = {
"source": "deezer",
"source_track_id": "dz-track",
"source_artist_id": "dz-artist",
"source_album_id": "dz-album",
"title": "Song One",
"artist": "Artist One",
"album_artist": "Artist One",
"album": "Album One",
}
me.embed_source_ids(audio, metadata, context={"track_info": {}, "original_search_result": {}})
assert audio.tags.added == []
def test_embed_source_ids_writes_musicbrainz_release_year_and_updates_album_year(tmp_path, monkeypatch):
db_path = tmp_path / "music.db"
conn = sqlite3.connect(db_path)
conn.execute("CREATE TABLE artists (id TEXT PRIMARY KEY, name TEXT)")
conn.execute("CREATE TABLE albums (id TEXT PRIMARY KEY, artist_id TEXT, title TEXT, year INTEGER)")
conn.execute("INSERT INTO artists (id, name) VALUES (?, ?)", ("artist-1", "Artist One"))
conn.execute(
"INSERT INTO albums (id, artist_id, title, year) VALUES (?, ?, ?, ?)",
("album-1", "artist-1", "Album One", None),
)
conn.commit()
conn.close()
class _FakeMBClient:
def get_recording(self, mbid, includes=None):
return {
"isrcs": ["ISRC-123"],
"genres": [{"name": "Post Rock", "count": 10}],
}
def get_release(self, mbid, includes=None):
return {
"release-group": {
"id": "rg-1",
"primary-type": "album",
"first-release-date": "2021-09-17",
},
"artist-credit": [{"artist": {"id": "artist-mb-1"}}],
"status": "Official",
"country": "US",
"barcode": "1234567890",
"media": [{"format": "CD", "tracks": [{"position": 1, "id": "reltrack-1", "recording": {"id": "rec-1"}}]}],
"label-info": [{"catalog-number": "CAT-1"}],
"text-representation": {"script": "Latn"},
"asin": "ASIN1",
}
class _FakeMBService:
def __init__(self):
self.mb_client = _FakeMBClient()
def match_recording(self, title, artist):
return {"mbid": "rec-mbid"}
def match_artist(self, artist):
return {"mbid": "artist-mbid"}
def match_release(self, album, artist):
return {"mbid": "release-mbid"}
audio = _FakeAudio()
symbols = _fake_symbols(audio)
runtime = types.SimpleNamespace(mb_worker=types.SimpleNamespace(mb_service=_FakeMBService()))
monkeypatch.setattr(
ms,
"get_config_manager",
lambda: _Config(
{
"metadata_enhancement.enabled": True,
"metadata_enhancement.embed_album_art": False,
"metadata_enhancement.tags.write_multi_artist": False,
"musicbrainz.embed_tags": True,
}
),
)
monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols)
monkeypatch.setattr(ms, "get_database", lambda: _FileDB(str(db_path)))
metadata = {
"source": "musicbrainz",
"title": "Song One",
"artist": "Artist One",
"album_artist": "Artist One",
"album": "Album One",
"track_number": 1,
"total_tracks": 12,
"disc_number": 1,
}
me.embed_source_ids(audio, metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime)
assert metadata["musicbrainz_release_id"] == "release-mbid"
assert metadata["date"] == "2021"
assert any(frame.kind == "TDRC" for frame in audio.tags.added)
assert any(frame.kind == "TSRC" for frame in audio.tags.added)
check = sqlite3.connect(db_path)
check.row_factory = sqlite3.Row
row = check.execute(
"""
SELECT albums.year
FROM albums
JOIN artists ON artists.id = albums.artist_id
WHERE albums.title = ? AND artists.name = ?
""",
("Album One", "Artist One"),
).fetchone()
check.close()
assert row["year"] == 2021
def test_musicbrainz_release_lookup_failure_does_not_poison_cache(monkeypatch):
class _FakeMBClient:
def get_release(self, mbid, includes=None):
return {}
class _FakeMBService:
def __init__(self):
self.release_calls = 0
self.mb_client = _FakeMBClient()
def match_recording(self, title, artist):
return None
def match_artist(self, artist):
return {"mbid": "artist-mbid"}
def match_release(self, album, artist):
self.release_calls += 1
if self.release_calls == 1:
raise requests.RequestException("temporary MusicBrainz outage")
return {"mbid": "release-mbid"}
monkeypatch.setattr(ms, "get_config_manager", lambda: _Config({"musicbrainz.embed_tags": True}))
monkeypatch.setattr(ms, "mb_release_cache", {})
monkeypatch.setattr(ms, "mb_release_detail_cache", {})
service = _FakeMBService()
runtime = types.SimpleNamespace(mb_worker=types.SimpleNamespace(mb_service=service))
pp = {
"id_tags": {},
"track_title": "Song One",
"artist_name": "Artist One",
"batch_artist_name": "Artist One",
"metadata": {"album": "Album One"},
"recording_mbid": None,
"artist_mbid": None,
"release_mbid": "",
"mb_genres": [],
"isrc": None,
"deezer_bpm": None,
"deezer_isrc": None,
"audiodb_mood": None,
"audiodb_style": None,
"audiodb_genre": None,
"tidal_isrc": None,
"tidal_copyright": None,
"qobuz_isrc": None,
"qobuz_copyright": None,
"qobuz_label": None,
"lastfm_tags": [],
"lastfm_url": None,
"genius_url": None,
"release_year": None,
}
metadata = {"album": "Album One", "artist": "Artist One"}
ms._process_musicbrainz_source(pp, metadata, _Config({"musicbrainz.embed_tags": True}), runtime, "Song One", "Artist One")
assert service.release_calls == 1
assert ms.mb_release_cache == {}
poisoned_norm_key = (ms.normalize_album_cache_key("Album One"), "artist one")
poisoned_exact_key = ("album one", "artist one")
ms.mb_release_cache[poisoned_norm_key] = ""
ms.mb_release_cache[poisoned_exact_key] = ""
ms._process_musicbrainz_source(pp, metadata, _Config({"musicbrainz.embed_tags": True}), runtime, "Song One", "Artist One")
assert service.release_calls == 2
def test_source_processors_do_not_swallow_programmer_errors():
class _BoomClient:
def search_track(self, artist_name, track_title):
raise ValueError("boom")
runtime = types.SimpleNamespace(deezer_worker=types.SimpleNamespace(client=_BoomClient()))
pp = {
"id_tags": {},
"batch_artist_name": "Artist One",
"release_year": None,
}
metadata = {"album": "Album One"}
with pytest.raises(ValueError):
ms._process_deezer_source(pp, metadata, _Config({"deezer.embed_tags": True}), runtime, "Song One", "Artist One")
def test_musicbrainz_caches_evict_oldest_entries():
release_cache = OrderedDict()
detail_cache = OrderedDict()
ms._bounded_cache_set(release_cache, ("album-1", "artist"), "release-1", 2)
ms._bounded_cache_set(release_cache, ("album-2", "artist"), "release-2", 2)
assert list(release_cache.keys()) == [("album-1", "artist"), ("album-2", "artist")]
assert ms._bounded_cache_get(release_cache, ("album-1", "artist")) == "release-1"
ms._bounded_cache_set(release_cache, ("album-3", "artist"), "release-3", 2)
assert list(release_cache.keys()) == [("album-1", "artist"), ("album-3", "artist")]
ms._bounded_cache_set(detail_cache, "release-1", {"title": "One"}, 1)
ms._bounded_cache_set(detail_cache, "release-2", {"title": "Two"}, 1)
assert list(detail_cache.keys()) == ["release-2"]
def test_enhance_file_metadata_forwards_runtime_to_source_embedding(monkeypatch):
audio = _FakeAudio()
symbols = _fake_symbols(audio)
seen = {}
runtime = types.SimpleNamespace(marker="runtime")
monkeypatch.setattr(me, "get_config_manager", lambda: _Config(
{
"metadata_enhancement.enabled": True,
"metadata_enhancement.embed_album_art": False,
"metadata_enhancement.tags.write_multi_artist": False,
}
))
monkeypatch.setattr(me, "get_mutagen_symbols", lambda: symbols)
monkeypatch.setattr(me, "strip_all_non_audio_tags", lambda file_path: {"apev2_stripped": False, "apev2_tag_count": 0})
monkeypatch.setattr(
me,
"extract_source_metadata",
lambda context, artist, album_info: {
"source": "deezer",
"source_track_id": "dz-track",
"source_artist_id": "dz-artist",
"source_album_id": "dz-album",
"title": "Song One",
"artist": "Artist One",
"album_artist": "Artist One",
"album": "Album One",
"track_number": 3,
"total_tracks": 12,
"disc_number": 2,
"date": "2024",
"genre": "Rock",
},
)
monkeypatch.setattr(
me,
"embed_source_ids",
lambda audio_file, metadata, context, runtime=None: seen.setdefault("runtime", runtime),
)
monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None)
monkeypatch.setattr(me, "verify_metadata_written", lambda file_path: True)
result = me.enhance_file_metadata(
"song.flac",
{"_audio_quality": ""},
{"name": "Artist One"},
{},
runtime=runtime,
)
assert result is True
assert seen["runtime"] is runtime
def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch):
audio = _FakeAudio()
symbols = _fake_symbols(audio)
strip_calls = []
verify_calls = []
monkeypatch.setattr(me, "get_config_manager", lambda: _Config(
{
"metadata_enhancement.enabled": True,
"metadata_enhancement.embed_album_art": False,
"metadata_enhancement.tags.write_multi_artist": False,
}
))
monkeypatch.setattr(ms, "get_config_manager", lambda: _Config())
monkeypatch.setattr(me, "get_mutagen_symbols", lambda: symbols)
monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols)
monkeypatch.setattr(ms, "get_database", lambda: None)
monkeypatch.setattr(me, "strip_all_non_audio_tags", lambda file_path: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0})
monkeypatch.setattr(
me,
"extract_source_metadata",
lambda context, artist, album_info: {
"source": "deezer",
"source_track_id": "dz-track",
"source_artist_id": "dz-artist",
"source_album_id": "dz-album",
"title": "Song One",
"artist": "Artist One",
"album_artist": "Artist One",
"album": "Album One",
"track_number": 3,
"total_tracks": 12,
"disc_number": 2,
"date": "2024",
"genre": "Rock",
"musicbrainz_release_id": "mb-release-1",
},
)
monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None)
monkeypatch.setattr(me, "verify_metadata_written", lambda file_path: verify_calls.append(file_path) or True)
album_info = {}
result = me.enhance_file_metadata(
"song.flac",
{"_audio_quality": ""},
{"name": "Artist One"},
album_info,
)
assert result is True
assert strip_calls == ["song.flac"]
assert verify_calls == ["song.flac"]
assert audio.clear_pictures_calls == 1
assert len(audio.save_calls) == 2
assert album_info["musicbrainz_release_id"] == "mb-release-1"
assert any(frame.kind == "TIT2" for frame in audio.tags.added)
assert any(frame.kind == "TPE1" for frame in audio.tags.added)
assert any(frame.kind == "TXXX" and frame.kwargs.get("desc") == "DEEZER_TRACK_ID" for frame in audio.tags.added)
def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch):
monkeypatch.setattr(ma, "get_config_manager", lambda: _Config(
{
"metadata_enhancement.cover_art_download": True,
"metadata_enhancement.prefer_caa_art": False,
}
))
monkeypatch.setattr(ma.urllib.request, "urlopen", lambda *args, **kwargs: _FakeResponse(b"cover-bytes"))
target_dir = tmp_path / "Album One"
target_dir.mkdir()
ma.download_cover_art(
{},
str(target_dir),
{"album": {"image_url": "https://img.example/album.jpg"}},
)
cover_path = target_dir / "cover.jpg"
assert cover_path.exists()
assert cover_path.read_bytes() == b"cover-bytes"

View file

@ -0,0 +1,56 @@
import types
from core.imports.pipeline import build_import_pipeline_runtime
from core.metadata.enrichment import build_metadata_enrichment_runtime
def test_build_import_pipeline_runtime_exposes_expected_contract():
import_fields = {
"automation_engine": object(),
"on_download_completed": object(),
"web_scan_manager": object(),
"repair_worker": object(),
}
runtime = build_import_pipeline_runtime(**import_fields)
assert isinstance(runtime, types.SimpleNamespace)
for name, value in import_fields.items():
assert hasattr(runtime, name)
assert getattr(runtime, name) is value
for name in (
"mb_worker",
"deezer_worker",
"audiodb_worker",
"tidal_client",
"qobuz_enrichment_worker",
"lastfm_worker",
"genius_worker",
"spotify_enrichment_worker",
"itunes_enrichment_worker",
):
assert not hasattr(runtime, name)
def test_build_metadata_enrichment_runtime_exposes_expected_contract():
metadata_fields = {
"mb_worker": object(),
"deezer_worker": object(),
"audiodb_worker": object(),
"tidal_client": object(),
"qobuz_enrichment_worker": object(),
"lastfm_worker": object(),
"genius_worker": object(),
"spotify_enrichment_worker": object(),
"itunes_enrichment_worker": object(),
}
runtime = build_metadata_enrichment_runtime(**metadata_fields)
assert isinstance(runtime, types.SimpleNamespace)
for name, value in metadata_fields.items():
assert hasattr(runtime, name)
assert getattr(runtime, name) is value
for name in ("automation_engine", "on_download_completed", "web_scan_manager", "repair_worker"):
assert not hasattr(runtime, name)

View file

@ -2,6 +2,14 @@ import sys
import types
class _DummyConfigManager:
def get(self, key, default=None):
return default
def get_active_media_server(self):
return "plex"
if "spotipy" not in sys.modules:
spotipy = types.ModuleType("spotipy")
@ -26,13 +34,6 @@ if "config.settings" not in sys.modules:
config_pkg = types.ModuleType("config")
settings_mod = types.ModuleType("config.settings")
class _DummyConfigManager:
def get(self, key, default=None):
return default
def get_active_media_server(self):
return "plex"
settings_mod.config_manager = _DummyConfigManager()
config_pkg.settings = settings_mod
sys.modules["config"] = config_pkg

View file

@ -7,7 +7,6 @@ and `status` query params and includes a `total` count.
"""
import sys
import threading
import types
from unittest.mock import patch
@ -45,6 +44,7 @@ _install_flask_limiter_stub()
from flask import Flask, Blueprint # noqa: E402
from api import downloads as downloads_mod # noqa: E402
import core.runtime_state as runtime_state # noqa: E402
def _make_task(status="downloading", when=None):
@ -63,11 +63,10 @@ def _make_task(status="downloading", when=None):
def _make_app_with_tasks(tasks_dict):
"""Create a minimal Flask app with the downloads blueprint mounted and
a fake `web_server` module exposing the given download_tasks dict."""
fake_ws = types.ModuleType("web_server")
fake_ws.download_tasks = tasks_dict
fake_ws.tasks_lock = threading.RLock()
sys.modules["web_server"] = fake_ws
the shared import runtime state populated with the given download_tasks dict."""
original_tasks = dict(runtime_state.download_tasks)
runtime_state.download_tasks.clear()
runtime_state.download_tasks.update(tasks_dict)
# Bypass API key auth for tests.
def _passthrough(f):
@ -83,6 +82,7 @@ def _make_app_with_tasks(tasks_dict):
downloads_mod.register_routes(bp)
app.register_blueprint(bp)
app._original_download_tasks = original_tasks
return app
@ -96,8 +96,12 @@ def client():
for i in range(25)
}
app = _make_app_with_tasks(tasks)
with app.test_client() as c:
yield c
try:
with app.test_client() as c:
yield c
finally:
runtime_state.download_tasks.clear()
runtime_state.download_tasks.update(app._original_download_tasks)
def test_default_limit_applied(client):

View file

@ -0,0 +1,32 @@
import pytest
import core.runtime_state as runtime_state
def test_mark_task_completed_requires_tasks_lock():
original_tasks = dict(runtime_state.download_tasks)
runtime_state.download_tasks.clear()
runtime_state.download_tasks["task-1"] = {"status": "running", "stream_processed": False}
try:
with pytest.raises(RuntimeError, match="tasks_lock"):
runtime_state.mark_task_completed("task-1")
finally:
runtime_state.download_tasks.clear()
runtime_state.download_tasks.update(original_tasks)
def test_mark_task_completed_succeeds_when_lock_held():
original_tasks = dict(runtime_state.download_tasks)
runtime_state.download_tasks.clear()
runtime_state.download_tasks["task-1"] = {"status": "running", "stream_processed": False}
try:
with runtime_state.tasks_lock:
assert runtime_state.mark_task_completed("task-1", {"name": "Song One"}) is True
assert runtime_state.download_tasks["task-1"]["status"] == "completed"
assert runtime_state.download_tasks["task-1"]["stream_processed"] is True
assert runtime_state.download_tasks["task-1"]["track_info"] == {"name": "Song One"}
finally:
runtime_state.download_tasks.clear()
runtime_state.download_tasks.update(original_tasks)

View file

@ -2,10 +2,8 @@ import sys
import types
from datetime import datetime, timedelta
_RECENT_RELEASE_DATE = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
if "spotipy" not in sys.modules:
spotipy = types.ModuleType("spotipy")
@ -1045,6 +1043,10 @@ def test_update_discovery_pool_incremental_uses_source_priority(monkeypatch):
assert scanner.database.discovery_pool_calls
assert scanner.database.discovery_pool_calls[0][1] == "deezer"
assert scanner.database.discovery_pool_calls[0][0]["deezer_track_id"] == "dz-track-1"
assert scanner.database.discovery_pool_calls[0][0]["deezer_album_id"] == "dz-release-1"
assert scanner.database.discovery_pool_calls[0][0]["deezer_artist_id"] == "dz-artist"
assert scanner.database.discovery_pool_calls[0][0].get("spotify_track_id") is None
assert deezer_client.search_calls == [("Incremental Artist", 1, {})]
assert deezer_client.album_calls
assert spotify_client.search_calls == []

File diff suppressed because it is too large Load diff

View file

@ -1214,6 +1214,7 @@ function importPageRenderMatchList() {
// Build rows
let matchedCount = 0;
const rows = data.matches.map((m, idx) => {
const trackInfo = _importPageGetTrackDisplayInfo(m, idx);
let file = null;
let confidence = m.confidence;
let isOverride = false;
@ -1253,8 +1254,8 @@ function importPageRenderMatchList() {
<div class="import-page-match-row ${file ? 'matched' : ''}"
ondragover="importPageHandleDragOver(event)" ondragleave="this.classList.remove('drag-over')" ondrop="importPageHandleDrop(event, ${idx})"
onclick="importPageTapAssign(${idx})">
<span class="import-page-match-num">${m.spotify_track.track_number}</span>
<span class="import-page-match-track">${_esc(m.spotify_track.name)}</span>
<span class="import-page-match-num">${trackInfo.displayTrackNumber}</span>
<span class="import-page-match-track">${_esc(trackInfo.name)}</span>
<span class="import-page-match-file ${file ? 'has-file' : ''}">
${file
? `<span class="import-page-match-file-name">${_esc(file.filename)}</span>
@ -1304,6 +1305,21 @@ function importPageRenderMatchList() {
processBtn.textContent = `Process ${matchedCount} Track${matchedCount !== 1 ? 's' : ''}`;
}
function _importPageGetTrackDisplayInfo(item, index) {
const track = item?.track || item?.spotify_track || {};
const rawTrackNumber = track.track_number ?? track.trackNumber ?? null;
const trackNumber = rawTrackNumber === null || rawTrackNumber === undefined || rawTrackNumber === ''
? null
: String(rawTrackNumber).split('/')[0].trim();
return {
track,
name: track.name || track.title || `Track ${index + 1}`,
trackNumber,
displayTrackNumber: trackNumber || String(index + 1),
};
}
// --- Album Tab: Drag and Drop ---
function importPageStartDrag(event, stagingFileIndex) {
@ -1622,7 +1638,7 @@ function importPageProcessSingles() {
const f = importPageState.stagingFiles[i];
const manualMatch = importPageState.singlesManualMatches[i];
if (manualMatch) {
return { ...f, spotify_override: manualMatch };
return { ...f, manual_match: manualMatch };
}
return f;
});
@ -1669,7 +1685,7 @@ function _importQueueAdd(job) {
async function _importQueueRunJob(entry, job) {
for (let i = 0; i < job.items.length; i++) {
const itemName = job.type === 'album'
? (job.items[i].spotify_track?.name || `Track ${i + 1}`)
? _importPageGetTrackDisplayInfo(job.items[i], i).name
: (job.items[i].title || job.items[i].filename || `File ${i + 1}`);
// Update status with current track info
@ -7608,4 +7624,3 @@ window.updateEnhanceSelectedCount = updateEnhanceSelectedCount;
window.submitEnhanceQuality = submitEnhanceQuality;
// ===== END ENHANCE QUALITY MODAL =====