Move the import flows and related post-processing pipelines into separate modules

- Extract the import pipeline, album import, staging, path, file ops, guards, runtime state, side effects, and metadata enrichment out of .
- Canonicalize the refactored import path around  and remove legacy , , , and  request shapes from the import endpoints.
- Make album and track metadata lookups follow the configured provider priority instead of hard-coding Spotify, while still falling back when needed.
- Update the import routes and frontend payloads to use the new core helpers.
- Add coverage for the extracted helpers and the refactored import flows.

PS. apologies to anyone who might check this commit out - the intention was to start small, but things kinda snowballed out of control at some point since the logic just kept going on and on, and everything kinda had to be changed all at once for it all to make any sense
This commit is contained in:
Antti Kettunen 2026-04-24 16:22:33 +03:00
parent eb442da728
commit 0bbf44809f
No known key found for this signature in database
GPG key ID: C6B2A3D250359BD7
30 changed files with 8238 additions and 4913 deletions

View file

@ -5,6 +5,7 @@ Download management endpoints — list, cancel active downloads.
from flask import request, current_app
from .auth import require_api_key
from .helpers import api_success, api_error
from core.import_runtime_state import download_tasks, tasks_lock
def _serialize_download(task_id, task):
@ -53,8 +54,6 @@ def register_routes(bp):
descending so newest/in-flight tasks appear first.
"""
try:
from web_server import download_tasks, tasks_lock
# Parse pagination params
try:
limit = int(request.args.get("limit", 100))

View file

@ -55,7 +55,7 @@ def register_routes(bp):
def system_activity():
"""Recent activity feed."""
try:
from web_server import activity_feed
from core.import_runtime_state import activity_feed
items = list(activity_feed) if activity_feed else []
return api_success({"activities": items})
except Exception as e:
@ -74,7 +74,7 @@ def register_routes(bp):
# Active download count
download_count = 0
try:
from web_server import download_tasks, tasks_lock
from core.import_runtime_state import download_tasks, tasks_lock
with tasks_lock:
download_count = sum(
1 for t in download_tasks.values()

515
core/import_album.py Normal file
View file

@ -0,0 +1,515 @@
"""Album import helpers for staging matching and post-processing context."""
from __future__ import annotations
import os
from typing import Any, Dict, Iterable, List, Optional, Set
from core.import_context import normalize_import_context
from core.import_file_ops import read_staging_file_metadata
from core.import_staging import AUDIO_EXTENSIONS, get_staging_path
from utils.logging_config import get_logger
logger = get_logger("import_album")
def get_client_for_source(source: str):
from core.metadata_service import get_client_for_source as _get_client_for_source
return _get_client_for_source(source)
def get_artist_album_tracks(
album_id: str,
artist_name: str = "",
album_name: str = "",
source: Optional[str] = None,
):
from core.metadata_service import get_artist_album_tracks as _get_artist_album_tracks
return _get_artist_album_tracks(
album_id,
artist_name=artist_name,
album_name=album_name,
source_override=source,
)
try:
from core.matching_engine import MusicMatchingEngine
_MATCHING_ENGINE_IMPORT_ERROR = None
except Exception as exc: # pragma: no cover - only hits in stripped-down environments
MusicMatchingEngine = None # type: ignore[assignment]
_MATCHING_ENGINE_IMPORT_ERROR = exc
_MATCHING_ENGINE = None
def _get_matching_engine() -> Any:
global _MATCHING_ENGINE
if _MATCHING_ENGINE is None:
if MusicMatchingEngine is None:
raise RuntimeError("Music matching engine is unavailable") from _MATCHING_ENGINE_IMPORT_ERROR
_MATCHING_ENGINE = MusicMatchingEngine()
return _MATCHING_ENGINE
def _normalize_artist_entries(artists: Any) -> List[Dict[str, Any]]:
if not artists:
return []
if isinstance(artists, (str, bytes)):
artists = [artists]
elif isinstance(artists, dict):
artists = [artists]
else:
try:
artists = list(artists)
except TypeError:
artists = [artists]
normalized: List[Dict[str, Any]] = []
for artist in artists:
if isinstance(artist, dict):
entry: Dict[str, Any] = {}
name = artist.get("name") or artist.get("artist_name") or artist.get("title") or ""
artist_id = artist.get("id") or artist.get("artist_id") or ""
if name:
entry["name"] = str(name)
if artist_id:
entry["id"] = str(artist_id)
genres = artist.get("genres")
if genres is not None:
entry["genres"] = genres
if entry:
normalized.append(entry)
continue
name = str(artist).strip()
if name:
normalized.append({"name": name})
return normalized
def _normalize_album_source(album: Dict[str, Any], source: str = "") -> str:
album_source = source or album.get("source") or ""
return str(album_source).strip().lower()
def _strip_legacy_source_fields(payload: Any) -> Any:
if not isinstance(payload, dict):
return payload
cleaned = dict(payload)
cleaned.pop("_source", None)
cleaned.pop("provider", None)
return cleaned
def _extract_track_artist_name(track: Dict[str, Any]) -> str:
artists = track.get("artists") or []
if isinstance(artists, (str, bytes)):
artists = [artists]
elif isinstance(artists, dict):
artists = [artists]
else:
try:
artists = list(artists)
except TypeError:
artists = [artists]
if not artists:
return ""
first = artists[0]
if isinstance(first, dict):
return str(first.get("name") or first.get("artist_name") or first.get("title") or "").strip()
return str(first or "").strip()
def _coerce_track_int(value: Any, default: int = 1) -> int:
if value in (None, ""):
return default
try:
return int(str(value).split("/")[0].strip() or default)
except (TypeError, ValueError):
return default
def _collect_staging_files(file_paths: Optional[Iterable[str]] = None) -> List[Dict[str, Any]]:
staging_path = get_staging_path()
file_filter: Optional[Set[str]] = set(file_paths) if file_paths else None
staging_files: List[Dict[str, Any]] = []
if not os.path.isdir(staging_path):
return staging_files
for root, _dirs, filenames in os.walk(staging_path):
for filename in filenames:
ext = os.path.splitext(filename)[1].lower()
if ext not in AUDIO_EXTENSIONS:
continue
full_path = os.path.join(root, filename)
if file_filter is not None and full_path not in file_filter:
continue
meta = read_staging_file_metadata(full_path, filename)
staging_files.append(
{
"filename": filename,
"full_path": full_path,
"title": meta.get("title", ""),
"artist": meta.get("albumartist") or meta.get("artist") or "",
"album": meta.get("album", ""),
"albumartist": meta.get("albumartist") or meta.get("artist") or "",
"track_number": meta.get("track_number", 1),
"disc_number": meta.get("disc_number", 1),
}
)
return staging_files
def _normalize_match_track(track: Dict[str, Any], source: str, album: Dict[str, Any]) -> Dict[str, Any]:
track_album = track.get("album") if isinstance(track.get("album"), dict) else album
if isinstance(track_album, dict):
track_album = _strip_legacy_source_fields(track_album)
track_source = _normalize_album_source(track, source)
track_artists = _normalize_artist_entries(track.get("artists") or [])
if not track_artists and album.get("artists"):
track_artists = _normalize_artist_entries(album.get("artists"))
return {
"id": track.get("id", ""),
"name": track.get("name", "Unknown Track"),
"track_number": _coerce_track_int(track.get("track_number", 1), default=1),
"disc_number": _coerce_track_int(track.get("disc_number", 1), default=1),
"duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0),
"artists": track_artists,
"uri": track.get("uri", ""),
"album": track_album,
"source": track_source,
}
def _score_album_track_match(track: Dict[str, Any], staging_file: Dict[str, Any], album_name: str) -> float:
engine = _get_matching_engine()
track_name = track.get("name", "")
staging_title = staging_file.get("title", "")
score = 0.0
title_sim = engine.similarity_score(
engine.normalize_string(track_name),
engine.normalize_string(staging_title or ""),
)
score += title_sim * 0.45
track_artist_name = _extract_track_artist_name(track)
staging_artist = staging_file.get("artist") or ""
if track_artist_name and staging_artist:
artist_sim = engine.similarity_score(
engine.normalize_string(track_artist_name),
engine.normalize_string(staging_artist),
)
score += artist_sim * 0.15
else:
score += 0.075
track_number = _coerce_track_int(track.get("track_number", 1), default=1)
staging_track_number = _coerce_track_int(staging_file.get("track_number", 1), default=1)
if staging_track_number and track_number:
if staging_track_number == track_number:
score += 0.30
elif abs(staging_track_number - track_number) <= 1:
score += 0.12
staging_album = staging_file.get("album") or ""
if staging_album and album_name:
album_sim = engine.similarity_score(
engine.normalize_string(staging_album),
engine.normalize_string(album_name),
)
score += album_sim * 0.10
return score
def _fetch_artist_data_for_source(client: Any, artist_id: str, source: str) -> Any:
if source == "spotify":
try:
return client.get_artist(artist_id, allow_fallback=False)
except TypeError:
return client.get_artist(artist_id)
return client.get_artist(artist_id)
def resolve_album_artist_context(album: Dict[str, Any], source: str = "") -> Dict[str, Any]:
"""Build a neutral artist context for album import processing."""
album = dict(album or {})
source = _normalize_album_source(album, source)
artists = _normalize_artist_entries(album.get("artists") or [])
if not artists:
artist_name = album.get("artist") or album.get("artist_name") or ""
artist_id = album.get("artist_id") or ""
if artist_name or artist_id:
artist_entry: Dict[str, Any] = {}
if artist_name:
artist_entry["name"] = str(artist_name)
if artist_id:
artist_entry["id"] = str(artist_id)
artists = [artist_entry]
primary_artist = artists[0] if artists else {}
artist_name = str(
primary_artist.get("name")
or album.get("artist")
or album.get("artist_name")
or "Unknown Artist"
).strip()
artist_id = str(primary_artist.get("id") or album.get("artist_id") or "").strip()
genres: List[Any] = []
if artist_id and source:
client = get_client_for_source(source)
if client and hasattr(client, "get_artist"):
try:
artist_data = _fetch_artist_data_for_source(client, artist_id, source)
raw_genres = artist_data.get("genres") if isinstance(artist_data, dict) else getattr(artist_data, "genres", [])
if isinstance(raw_genres, str):
genres = [raw_genres]
elif raw_genres:
try:
genres = list(raw_genres)
except TypeError:
genres = [raw_genres]
except Exception as exc:
logger.debug("Could not resolve artist genres for %s on %s: %s", artist_id, source, exc)
return {
"id": artist_id,
"name": artist_name,
"genres": genres,
"source": source,
}
def build_album_import_context(
album: Dict[str, Any],
track: Dict[str, Any],
*,
artist_context: Optional[Dict[str, Any]] = None,
total_discs: int = 1,
source: str = "",
) -> Dict[str, Any]:
"""Build a neutral post-processing context for one album track."""
album = dict(album or {})
track = dict(track or {})
source = _normalize_album_source(album, source)
album_artists = _normalize_artist_entries(album.get("artists") or [])
if not album_artists and artist_context:
album_artists = _normalize_artist_entries([artist_context])
if artist_context:
artist_ctx = dict(artist_context)
else:
artist_ctx = resolve_album_artist_context(album, source)
artist_ctx = _strip_legacy_source_fields(artist_ctx)
artist_ctx.setdefault("genres", [])
artist_ctx.setdefault("source", source)
artist_ctx["genres"] = artist_ctx.get("genres") or []
track_artists = _normalize_artist_entries(track.get("artists") or [])
if not track_artists:
track_artists = album_artists or [artist_ctx]
track_album_value = track.get("album")
if isinstance(track_album_value, dict):
track_album_name = (
track_album_value.get("name")
or track_album_value.get("title")
or album.get("name")
or album.get("album_name")
or ""
)
track_album_id = str(track_album_value.get("id") or track_album_value.get("album_id") or "").strip()
track_album_type = track_album_value.get("album_type") or album.get("album_type") or "album"
track_album_release = track_album_value.get("release_date") or album.get("release_date") or ""
track_album_image = track_album_value.get("image_url") or album.get("image_url") or ""
else:
track_album_name = str(track_album_value or album.get("name") or album.get("album_name") or "").strip()
track_album_id = str(album.get("id") or album.get("album_id") or "").strip()
track_album_type = album.get("album_type") or "album"
track_album_release = album.get("release_date") or ""
track_album_image = album.get("image_url") or ""
album_name = str(album.get("name") or album.get("album_name") or track_album_name or "Unknown Album").strip()
artist_name = str(
artist_ctx.get("name")
or album.get("artist")
or album.get("artist_name")
or "Unknown Artist"
).strip()
track_number = _coerce_track_int(track.get("track_number", 1), default=1)
disc_number = _coerce_track_int(track.get("disc_number", 1), default=1)
normalized_track = {
"id": str(track.get("id") or track.get("track_id") or "").strip(),
"name": str(track.get("name") or "Unknown Track").strip(),
"track_number": track_number,
"disc_number": disc_number,
"duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0),
"artists": track_artists,
"uri": str(track.get("uri") or "").strip(),
"album": track_album_name,
"album_id": track_album_id,
"album_type": track_album_type,
"release_date": track_album_release,
"source": source,
}
normalized_album = {
"id": str(album.get("id") or album.get("album_id") or track_album_id or "").strip(),
"name": album_name,
"artist": artist_name,
"artist_name": artist_name,
"artist_id": str(artist_ctx.get("id") or album.get("artist_id") or "").strip(),
"artists": album_artists,
"release_date": str(album.get("release_date") or track_album_release or "").strip(),
"total_tracks": int(album.get("total_tracks") or track.get("total_tracks") or 0) or 1,
"total_discs": int(total_discs or 1) if str(total_discs or 1).isdigit() else total_discs or 1,
"album_type": str(album.get("album_type") or track_album_type or "album").strip() or "album",
"image_url": str(album.get("image_url") or track_album_image or "").strip(),
"images": album.get("images") or ([] if not track_album_image else [{"url": track_album_image}]),
"source": source,
}
original_search = {
"title": normalized_track["name"],
"artist": artist_name,
"album": album_name,
"track_number": track_number,
"disc_number": disc_number,
"clean_title": normalized_track["name"],
"clean_album": album_name,
"clean_artist": artist_name,
"artists": track_artists,
"duration_ms": normalized_track["duration_ms"],
"id": normalized_track["id"],
"source": source,
}
context = {
"artist": artist_ctx,
"album": normalized_album,
"track_info": normalized_track,
"original_search_result": original_search,
"is_album_download": True,
"has_clean_metadata": bool(normalized_track["id"]),
"has_full_metadata": bool(normalized_track["id"]),
"source": source,
}
normalized_context = normalize_import_context(context)
normalized_context["artist"] = _strip_legacy_source_fields(normalized_context.get("artist"))
normalized_context["album"] = _strip_legacy_source_fields(normalized_context.get("album"))
normalized_context["track_info"] = _strip_legacy_source_fields(normalized_context.get("track_info"))
normalized_context["original_search_result"] = _strip_legacy_source_fields(normalized_context.get("original_search_result"))
return normalized_context
def build_album_import_match_payload(
album_id: str,
*,
album_name: str = "",
album_artist: str = "",
file_paths: Optional[Iterable[str]] = None,
source: Optional[str] = None,
) -> Dict[str, Any]:
"""Build the album import match payload using provider-priority metadata lookup."""
album_response = get_artist_album_tracks(
album_id,
artist_name=album_artist,
album_name=album_name,
source=source,
)
album = _strip_legacy_source_fields(dict(album_response.get("album") or {}))
source = _normalize_album_source(album, album_response.get("source") or source or "")
tracks = list(album_response.get("tracks") or [])
if not album_response.get("success") or not tracks:
return {
"success": False,
"error": album_response.get("error", "Album not found"),
"status_code": album_response.get("status_code", 404),
"album": {
"id": album_id,
"name": album_name or album_id,
"artist": album_artist or "Unknown Artist",
"artist_name": album_artist or "Unknown Artist",
"artist_id": "",
"artists": [],
"release_date": "",
"total_tracks": 0,
"total_discs": 1,
"album_type": "album",
"image_url": "",
"images": [],
"source": source,
},
"matches": [],
"unmatched_files": [],
"source": source,
"source_priority": album_response.get("source_priority", []),
"resolved_album_id": album_response.get("resolved_album_id") or album_id,
}
staging_files = _collect_staging_files(file_paths)
album_name_for_match = album.get("name") or album_name or ""
matches: List[Dict[str, Any]] = []
used_files: Set[int] = set()
for track in tracks:
normalized_track = _normalize_match_track(track, source, album)
best_match = None
best_score = 0.0
for index, staging_file in enumerate(staging_files):
if index in used_files:
continue
score = _score_album_track_match(normalized_track, staging_file, album_name_for_match)
if score > best_score and score >= 0.4:
best_score = score
best_match = index
matches.append(
{
"track": normalized_track,
"staging_file": staging_files[best_match] if best_match is not None else None,
"confidence": round(best_score, 2) if best_match is not None else 0,
}
)
if best_match is not None:
used_files.add(best_match)
unmatched_files = [sf for index, sf in enumerate(staging_files) if index not in used_files]
return {
"success": True,
"album": album,
"matches": matches,
"unmatched_files": unmatched_files,
"source": source,
"source_priority": album_response.get("source_priority", []),
"resolved_album_id": album_response.get("resolved_album_id") or album_id,
}

326
core/import_context.py Normal file
View file

@ -0,0 +1,326 @@
"""Helpers for normalizing and reading import contexts.
These functions keep the single-import pipeline source-agnostic while still
accepting legacy `spotify_*` payloads from older callers.
"""
from __future__ import annotations
from typing import Any, Dict, Optional
def _as_dict(value: Any) -> Dict[str, Any]:
return value if isinstance(value, dict) else {}
def _first_value(mapping: Dict[str, Any], *keys: str, default: Any = "") -> Any:
for key in keys:
if key in mapping:
value = mapping.get(key)
if value not in (None, ""):
return value
return default
def _first_id_value(*values: Any) -> str:
for value in values:
if value in (None, ""):
continue
text = str(value).strip()
if text:
return text
return ""
def extract_artist_name(artist: Any) -> str:
if isinstance(artist, dict):
return str(artist.get("name", "") or "")
if hasattr(artist, "name"):
return str(getattr(artist, "name") or "")
return str(artist) if artist else ""
def normalize_import_context(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
"""Normalize an import context to neutral fields in place and drop legacy aliases."""
if not isinstance(context, dict):
return {}
artist = _as_dict(context.get("artist") or context.get("spotify_artist"))
album = _as_dict(context.get("album") or context.get("spotify_album"))
track_info = _as_dict(context.get("track_info"))
original_search = _as_dict(context.get("original_search_result"))
context["artist"] = artist
context["album"] = album
context["track_info"] = track_info
context["original_search_result"] = original_search
context.pop("spotify_artist", None)
context.pop("spotify_album", None)
for clean_key, legacy_key in (
("clean_title", "spotify_clean_title"),
("clean_album", "spotify_clean_album"),
("clean_artist", "spotify_clean_artist"),
):
if clean_key not in original_search or original_search.get(clean_key) in (None, ""):
legacy_value = original_search.get(legacy_key)
if legacy_value not in (None, ""):
original_search[clean_key] = legacy_value
original_search.pop(legacy_key, None)
has_clean = bool(context.get("has_clean_metadata", context.get("has_clean_spotify_data", False)))
has_full = bool(context.get("has_full_metadata", context.get("has_full_spotify_metadata", False)))
context["has_clean_metadata"] = has_clean
context["has_full_metadata"] = has_full
context.pop("has_clean_spotify_data", None)
context.pop("has_full_spotify_metadata", None)
return context
def get_import_context_artist(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if not isinstance(context, dict):
return {}
return _as_dict(context.get("artist") or context.get("spotify_artist"))
def get_import_context_album(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if not isinstance(context, dict):
return {}
return _as_dict(context.get("album") or context.get("spotify_album"))
def get_import_track_info(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if not isinstance(context, dict):
return {}
return _as_dict(context.get("track_info"))
def get_import_original_search(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if not isinstance(context, dict):
return {}
return _as_dict(context.get("original_search_result"))
def get_import_source(context: Optional[Dict[str, Any]]) -> str:
if not isinstance(context, dict):
return ""
source = context.get("source")
if source:
return str(source)
track_info = get_import_track_info(context)
source = _first_value(track_info, "source", default="")
if source:
return str(source)
original_search = get_import_original_search(context)
source = _first_value(original_search, "source", default="")
if source:
return str(source)
album = get_import_context_album(context)
source = _first_value(album, "source", default="")
if source:
return str(source)
artist = get_import_context_artist(context)
source = _first_value(artist, "source", default="")
return str(source) if source else ""
def get_import_clean_title(
context: Optional[Dict[str, Any]],
album_info: Optional[Dict[str, Any]] = None,
default: str = "Unknown Track",
) -> str:
original_search = get_import_original_search(context)
title = _first_value(
original_search,
"clean_title",
"title",
default="",
)
if not title and album_info:
title = _first_value(album_info, "clean_track_name", "track_name", default="")
if not title:
track_info = get_import_track_info(context)
title = _first_value(track_info, "name", "title", default="")
return str(title or default)
def get_import_clean_album(
context: Optional[Dict[str, Any]],
album_info: Optional[Dict[str, Any]] = None,
default: str = "Unknown Album",
) -> str:
original_search = get_import_original_search(context)
album = _first_value(
original_search,
"clean_album",
"album",
default="",
)
if not album and album_info:
album = _first_value(album_info, "album_name", "clean_album_name", default="")
if not album:
album_ctx = get_import_context_album(context)
album = _first_value(album_ctx, "name", default="")
return str(album or default)
def get_import_clean_artist(context: Optional[Dict[str, Any]], default: str = "Unknown Artist") -> str:
original_search = get_import_original_search(context)
artist = _first_value(
original_search,
"clean_artist",
"artist",
default="",
)
if not artist:
artist_ctx = get_import_context_artist(context)
artist = _first_value(artist_ctx, "name", default="")
return str(artist or default)
def get_import_has_clean_metadata(context: Optional[Dict[str, Any]]) -> bool:
if not isinstance(context, dict):
return False
return bool(context.get("has_clean_metadata", False))
def get_import_has_full_metadata(context: Optional[Dict[str, Any]]) -> bool:
if not isinstance(context, dict):
return False
return bool(context.get("has_full_metadata", False))
def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]:
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
artist = get_import_context_artist(context)
album = get_import_context_album(context)
return {
"track_id": _first_id_value(
_first_value(track_info, "id", "track_id", "trackId", "source_track_id", default=""),
_first_value(track_info, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""),
_first_value(original_search, "id", "track_id", "source_track_id", default=""),
_first_value(original_search, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""),
),
"artist_id": _first_id_value(
_first_value(artist, "id", "artist_id", "source_artist_id", default=""),
_first_value(artist, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""),
_first_value(original_search, "artist_id", "source_artist_id", default=""),
_first_value(original_search, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""),
),
"album_id": _first_id_value(
_first_value(album, "id", "album_id", "collectionId", "source_album_id", default=""),
_first_value(album, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""),
_first_value(original_search, "album_id", "source_album_id", default=""),
_first_value(original_search, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""),
_first_value(track_info, "album_id", "source_album_id", default=""),
_first_value(track_info, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""),
),
}
def get_source_tag_names(source: str) -> Dict[str, Optional[str]]:
source_name = (source or "").strip().lower()
if source_name == "spotify":
return {"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"}
if source_name == "itunes":
return {"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"}
if source_name == "deezer":
return {"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None}
if source_name == "hydrabase":
return {"track": None, "artist": None, "album": None}
if source_name == "discogs":
return {"track": None, "artist": None, "album": None}
return {"track": None, "artist": None, "album": None}
def get_library_source_id_columns(source: str) -> Dict[str, Optional[str]]:
source_name = (source or "").strip().lower()
if source_name == "spotify":
return {"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"}
if source_name == "itunes":
return {"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"}
if source_name == "deezer":
return {"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"}
if source_name == "hydrabase":
return {"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"}
if source_name == "discogs":
return {"artist": "discogs_id", "album": "discogs_id", "track": None}
return {}
def build_import_album_info(
context: Optional[Dict[str, Any]],
*,
album_info: Optional[Dict[str, Any]] = None,
force_album: bool = False,
) -> Dict[str, Any]:
"""Build the album-info payload used by post-processing."""
album_ctx = get_import_context_album(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
artist_ctx = get_import_context_artist(context)
track_number = (
(album_info or {}).get("track_number")
or track_info.get("track_number")
or original_search.get("track_number")
or 1
)
disc_number = (
(album_info or {}).get("disc_number")
or track_info.get("disc_number")
or original_search.get("disc_number")
or 1
)
clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track"))
album_name = get_import_clean_album(context, album_info=album_info, default=original_search.get("album", "Unknown Album"))
album_image_url = (
(album_info or {}).get("album_image_url")
or album_ctx.get("image_url")
or ""
)
total_tracks = (
album_ctx.get("total_tracks")
or track_info.get("total_tracks")
or (album_info or {}).get("total_tracks")
or 0
)
album_type = (album_ctx.get("album_type") or track_info.get("album_type") or "album")
source = get_import_source(context)
artist_name = artist_ctx.get("name") or original_search.get("artist") or get_import_clean_artist(context)
normalized_album = str(album_name or "").strip().lower()
normalized_title = str(clean_track_name or "").strip().lower()
normalized_artist = str(artist_name or "").strip().lower()
is_album = bool(
force_album
or (
normalized_album
and total_tracks
and int(total_tracks) > 1
and normalized_album != normalized_title
and normalized_album != normalized_artist
)
)
return {
"is_album": is_album,
"album_name": album_name,
"track_number": int(track_number) if str(track_number).isdigit() else track_number,
"disc_number": int(disc_number) if str(disc_number).isdigit() else disc_number,
"clean_track_name": clean_track_name,
"album_image_url": album_image_url,
"confidence": (album_info or {}).get("confidence", 1.0 if is_album or force_album else 0.0),
"source": source,
"album_type": album_type,
"total_tracks": int(total_tracks) if str(total_tracks).isdigit() else total_tracks,
}

475
core/import_file_ops.py Normal file
View file

@ -0,0 +1,475 @@
"""Shared file and path helpers for import processing."""
from __future__ import annotations
import logging
import os
import re
import shutil
import subprocess
import time
from pathlib import Path
from typing import Any, Dict, Optional
logger = logging.getLogger("import_file_ops")
def _get_config_manager():
from config.settings import config_manager
return config_manager
def safe_move_file(src, dst):
"""Move a file safely across filesystems."""
src = Path(src)
dst = Path(dst)
dst.parent.mkdir(parents=True, exist_ok=True)
if not src.exists():
if dst.exists():
logger.info(f"Source gone but destination exists, file already transferred: {dst.name}")
return
raise FileNotFoundError(f"Source file not found and destination does not exist: {src}")
if dst.exists():
for _attempt in range(3):
try:
dst.unlink()
break
except PermissionError:
if _attempt < 2:
time.sleep(1)
else:
logger.warning(f"Could not remove locked destination after 3 attempts: {dst.name}")
except Exception:
break
try:
shutil.move(str(src), str(dst))
return
except FileNotFoundError:
if dst.exists():
logger.info(f"Source moved by another thread, destination exists: {dst.name}")
return
raise
except (OSError, PermissionError) as e:
error_msg = str(e).lower()
if dst.exists() and dst.stat().st_size > 0:
logger.warning(f"Move raised {type(e).__name__} but destination exists, treating as success: {e}")
try:
src.unlink()
except Exception:
logger.info(f"Could not delete source file (may be owned by another process): {src}")
return
if "cross-device" in error_msg or "operation not permitted" in error_msg or "permission denied" in error_msg:
logger.warning(f"Cross-device move detected, using fallback copy method: {e}")
try:
with open(src, "rb") as f_src:
with open(dst, "wb") as f_dst:
shutil.copyfileobj(f_src, f_dst)
f_dst.flush()
os.fsync(f_dst.fileno())
try:
src.unlink()
except PermissionError:
logger.info(f"Could not delete source file (may be owned by another process): {src}")
logger.info(f"Successfully moved file using fallback method: {src} -> {dst}")
return
except Exception as fallback_error:
logger.error(f"Fallback copy also failed: {fallback_error}")
raise
raise
def extract_track_number_from_filename(filename: str, title: str = None) -> int:
"""Extract track number from a filename. Returns 1 if not found."""
basename = os.path.splitext(os.path.basename(filename))[0].strip()
match = re.match(r"^\d[\-\.](\d{1,2})\s*[\-\.]\s*", basename)
if match:
num = int(match.group(1))
if 1 <= num <= 99:
return num
match = re.match(r"^\(?(\d{1,3})\)?\s*[\-\.)\]]\s*", basename)
if match:
num = int(match.group(1))
if 1 <= num <= 999:
return num
return 1
def _coerce_tag_number(value: Any, default: int = 1) -> int:
if value in (None, ""):
return default
if isinstance(value, (list, tuple)):
value = value[0] if value else None
if value in (None, ""):
return default
text = str(value).strip()
if not text:
return default
match = re.match(r"^(\d+)", text)
if match:
try:
return int(match.group(1))
except ValueError:
return default
try:
return int(text)
except (TypeError, ValueError):
return default
def read_staging_file_metadata(file_path: str, filename: Optional[str] = None) -> Dict[str, Any]:
"""Read common audio tag metadata from a staging file."""
try:
from mutagen import File as MutagenFile
tags = MutagenFile(file_path, easy=True)
except Exception:
tags = None
filename = filename or os.path.basename(file_path)
stem = os.path.splitext(os.path.basename(filename))[0]
def _first_tag(*keys: str) -> str:
if not tags:
return ""
for key in keys:
try:
value = tags.get(key) # type: ignore[attr-defined]
except Exception:
value = None
if value:
if isinstance(value, (list, tuple)):
value = value[0] if value else ""
text = str(value).strip()
if text:
return text
return ""
title = _first_tag("title")
artist = _first_tag("artist")
albumartist = _first_tag("albumartist")
album = _first_tag("album")
if not title:
title = stem
if not albumartist:
albumartist = artist
track_number = _coerce_tag_number(_first_tag("tracknumber", "track_number"), default=0)
if not track_number:
track_number = extract_track_number_from_filename(filename or file_path)
disc_number = _coerce_tag_number(_first_tag("discnumber", "disc_number"), default=1)
return {
"title": title,
"artist": artist,
"albumartist": albumartist,
"album": album,
"track_number": track_number,
"disc_number": disc_number,
}
def cleanup_empty_directories(download_path, moved_file_path):
"""Remove empty directories after a move, ignoring hidden files."""
try:
current_dir = os.path.dirname(moved_file_path)
while current_dir != download_path and current_dir.startswith(download_path):
is_empty = not any(not f.startswith(".") for f in os.listdir(current_dir))
if is_empty:
logger.warning(f"Removing empty directory: {current_dir}")
os.rmdir(current_dir)
current_dir = os.path.dirname(current_dir)
else:
break
except Exception as e:
logger.error(f"An error occurred during directory cleanup: {e}")
def get_audio_quality_string(file_path):
"""Return a compact audio quality string for the given file."""
try:
ext = os.path.splitext(file_path)[1].lower()
if ext == ".flac":
from mutagen.flac import FLAC
audio = FLAC(file_path)
return f"FLAC {audio.info.bits_per_sample}bit"
if ext == ".mp3":
from mutagen.mp3 import MP3, BitrateMode
audio = MP3(file_path)
bitrate_kbps = audio.info.bitrate // 1000
if audio.info.bitrate_mode == BitrateMode.VBR:
return "MP3-VBR"
return f"MP3-{bitrate_kbps}"
if ext in (".m4a", ".aac", ".mp4"):
from mutagen.mp4 import MP4
audio = MP4(file_path)
return f"M4A-{audio.info.bitrate // 1000}"
if ext == ".ogg":
from mutagen.oggvorbis import OggVorbis
audio = OggVorbis(file_path)
return f"OGG-{audio.info.bitrate // 1000}"
if ext == ".opus":
from mutagen.oggopus import OggOpus
audio = OggOpus(file_path)
return f"OPUS-{audio.info.bitrate // 1000}"
return ""
except Exception as e:
logger.debug(f"Could not determine audio quality for {file_path}: {e}")
return ""
def get_quality_tier_from_extension(file_path):
"""Classify a file extension into a quality tier."""
if not file_path:
return ("unknown", 999)
ext = os.path.splitext(file_path)[1].lower()
quality_tiers = {
"lossless": {
"extensions": [".flac", ".ape", ".wav", ".alac", ".dsf", ".dff", ".aiff", ".aif"],
"tier": 1,
},
"high_lossy": {
"extensions": [".opus", ".ogg"],
"tier": 2,
},
"standard_lossy": {
"extensions": [".m4a", ".aac"],
"tier": 3,
},
"low_lossy": {
"extensions": [".mp3", ".wma"],
"tier": 4,
},
}
for tier_name, tier_data in quality_tiers.items():
if ext in tier_data["extensions"]:
return (tier_name, tier_data["tier"])
return ("unknown", 999)
def downsample_hires_flac(final_path, context):
"""Downsample a hi-res FLAC to 16-bit/44.1kHz if enabled."""
from mutagen.flac import FLAC
config_manager = _get_config_manager()
if not config_manager.get("lossy_copy.downsample_hires", False):
return None
if os.path.splitext(final_path)[1].lower() != ".flac":
return None
try:
audio = FLAC(final_path)
original_bits = audio.info.bits_per_sample
original_rate = audio.info.sample_rate
except Exception as e:
logger.error(f"[Downsample] Could not read FLAC info: {e}")
return None
if original_bits <= 16 and original_rate <= 44100:
return None
logger.info(f"[Downsample] Converting {original_bits}-bit/{original_rate}Hz -> 16-bit/44100Hz: {os.path.basename(final_path)}")
ffmpeg_bin = shutil.which("ffmpeg")
if not ffmpeg_bin:
local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg")
if os.path.isfile(local):
ffmpeg_bin = local
else:
logger.warning("[Downsample] ffmpeg not found - skipping hi-res conversion")
return None
temp_path = final_path + ".tmp.flac"
try:
result = subprocess.run(
[
ffmpeg_bin, "-i", final_path,
"-sample_fmt", "s16",
"-ar", "44100",
"-map_metadata", "0",
"-compression_level", "8",
"-y", temp_path,
],
capture_output=True,
text=True,
timeout=300,
)
if result.returncode != 0:
logger.error(f"[Downsample] ffmpeg failed: {result.stderr[:200]}")
if os.path.exists(temp_path):
os.remove(temp_path)
return None
if not os.path.isfile(temp_path) or os.path.getsize(temp_path) == 0:
logger.warning("[Downsample] Output file missing or empty")
if os.path.exists(temp_path):
os.remove(temp_path)
return None
verify_audio = FLAC(temp_path)
if verify_audio.info.bits_per_sample != 16:
logger.info(f"[Downsample] Output not 16-bit ({verify_audio.info.bits_per_sample}-bit), aborting")
os.remove(temp_path)
return None
os.replace(temp_path, final_path)
logger.info(f"[Downsample] Converted to 16-bit/44.1kHz: {os.path.basename(final_path)}")
new_quality = "FLAC 16bit"
try:
updated_audio = FLAC(final_path)
updated_audio["QUALITY"] = new_quality
updated_audio.save()
except Exception as tag_err:
logger.error(f"[Downsample] Could not update QUALITY tag: {tag_err}")
old_quality = context.get("_audio_quality", "")
context["_audio_quality"] = new_quality
if old_quality and old_quality != new_quality and old_quality in os.path.basename(final_path):
new_basename = os.path.basename(final_path).replace(old_quality, new_quality)
new_path = os.path.join(os.path.dirname(final_path), new_basename)
try:
os.rename(final_path, new_path)
logger.info(f"[Downsample] Renamed: {os.path.basename(final_path)} -> {new_basename}")
for lyrics_ext in (".lrc", ".txt"):
old_lyrics = os.path.splitext(final_path)[0] + lyrics_ext
if os.path.isfile(old_lyrics):
new_lyrics = os.path.splitext(new_path)[0] + lyrics_ext
os.rename(old_lyrics, new_lyrics)
return new_path
except Exception as rename_err:
logger.error(f"[Downsample] Could not rename file: {rename_err}")
return final_path
except subprocess.TimeoutExpired:
logger.info(f"[Downsample] Conversion timed out for: {os.path.basename(final_path)}")
if os.path.exists(temp_path):
os.remove(temp_path)
except Exception as e:
logger.error(f"[Downsample] Conversion error: {e}")
if os.path.exists(temp_path):
try:
os.remove(temp_path)
except Exception:
pass
return None
def create_lossy_copy(final_path):
"""Convert a FLAC file to a lossy copy using the configured codec."""
from mutagen.flac import FLAC
config_manager = _get_config_manager()
if not config_manager.get("lossy_copy.enabled", False):
return None
if os.path.splitext(final_path)[1].lower() != ".flac":
return None
codec = config_manager.get("lossy_copy.codec", "mp3").lower()
bitrate = config_manager.get("lossy_copy.bitrate", "320")
if codec == "opus" and int(bitrate) > 256:
bitrate = "256"
codec_map = {
"mp3": ("libmp3lame", ".mp3", f"MP3-{bitrate}", ["-vn", "-id3v2_version", "3"]),
"opus": ("libopus", ".opus", f"OPUS-{bitrate}", ["-vn", "-map", "0:a", "-vbr", "on"]),
"aac": ("aac", ".m4a", f"AAC-{bitrate}", ["-vn", "-movflags", "+faststart"]),
}
if codec not in codec_map:
logger.info(f"[Lossy Copy] Unknown codec '{codec}' - skipping conversion")
return None
ffmpeg_codec, out_ext, quality_label, extra_args = codec_map[codec]
out_path = os.path.splitext(final_path)[0] + out_ext
original_quality = get_audio_quality_string(final_path)
if original_quality:
out_basename = os.path.basename(out_path)
if original_quality in out_basename:
out_basename = out_basename.replace(original_quality, quality_label)
out_path = os.path.join(os.path.dirname(out_path), out_basename)
ffmpeg_bin = shutil.which("ffmpeg")
if not ffmpeg_bin:
local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg")
if os.path.isfile(local):
ffmpeg_bin = local
else:
logger.warning(f"[Lossy Copy] ffmpeg not found - skipping {codec.upper()} conversion")
return None
try:
logger.info(f"[Lossy Copy] Converting to {quality_label}: {os.path.basename(final_path)}")
cmd = [
ffmpeg_bin, "-i", final_path,
"-codec:a", ffmpeg_codec,
"-b:a", f"{bitrate}k",
"-map_metadata", "0",
] + extra_args + ["-y", out_path]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
if result.returncode == 0:
logger.info(f"[Lossy Copy] Created {quality_label} copy: {os.path.basename(out_path)}")
try:
from mutagen import File as MutagenFile
audio = MutagenFile(out_path)
if audio is not None:
if codec == "mp3":
from mutagen.id3 import TXXX
audio.tags.add(TXXX(encoding=3, desc="QUALITY", text=[quality_label]))
elif codec == "opus":
audio["QUALITY"] = [quality_label]
elif codec == "aac":
from mutagen.mp4 import MP4FreeForm
audio["----:com.apple.iTunes:QUALITY"] = [MP4FreeForm(quality_label.encode("utf-8"))]
audio.save()
except Exception as tag_err:
logger.error(f"[Lossy Copy] Could not update QUALITY tag: {tag_err}")
return out_path
logger.error(f"[Lossy Copy] ffmpeg failed: {result.stderr[:200]}")
if os.path.exists(out_path):
try:
os.remove(out_path)
except Exception:
pass
return None
except subprocess.TimeoutExpired:
logger.warning(f"[Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}")
except Exception as e:
logger.error(f"[Lossy Copy] Conversion error: {e}")
return None

73
core/import_filename.py Normal file
View file

@ -0,0 +1,73 @@
"""Filename parsing helpers used by import flows."""
from __future__ import annotations
import os
import re
from typing import Any, Dict
_TRACK_PATTERNS = (
r"^(\d+)\s*[-\.]\s*(.+?)\s*[-]\s*(.+)$",
r"^(.+?)\s*[-]\s*(.+)$",
r"^(\d+)\s*[-\.]\s*(.+)$",
)
def parse_filename_metadata(filename: str) -> Dict[str, Any]:
"""Extract artist/title/album hints from a loose filename."""
raw_path = str(filename or "")
normalized_path = raw_path.replace("\\", "/")
base_name = os.path.splitext(os.path.basename(normalized_path))[0]
result: Dict[str, Any] = {
"artist": "",
"title": "",
"album": "",
"track_number": None,
}
if not base_name:
return result
for pattern in _TRACK_PATTERNS:
match = re.match(pattern, base_name)
if not match:
continue
groups = match.groups()
if len(groups) == 3:
try:
result["track_number"] = int(groups[0])
result["artist"] = result["artist"] or groups[1].strip()
result["title"] = result["title"] or groups[2].strip()
except ValueError:
result["artist"] = result["artist"] or groups[0].strip()
result["title"] = result["title"] or f"{groups[1]} - {groups[2]}".strip()
elif len(groups) == 2:
if groups[0].isdigit():
try:
result["track_number"] = int(groups[0])
result["title"] = result["title"] or groups[1].strip()
except ValueError:
pass
else:
result["artist"] = result["artist"] or groups[0].strip()
result["title"] = result["title"] or groups[1].strip()
break
if not result["title"]:
result["title"] = base_name
if not result["album"] and "/" in normalized_path:
path_parts = normalized_path.split("/")
for part in reversed(path_parts[:-1]):
if not part or part.startswith("@"):
continue
cleaned = re.sub(r"^\d+\s*[-\.]\s*", "", part).strip()
if len(cleaned) > 3:
result["album"] = cleaned
break
return result

121
core/import_guards.py Normal file
View file

@ -0,0 +1,121 @@
"""Import post-processing guards and quarantine helpers."""
from __future__ import annotations
import json
import os
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Optional
from core.import_context import (
get_import_clean_artist,
get_import_clean_title,
get_import_context_artist,
get_import_original_search,
get_import_track_info,
normalize_import_context,
)
from core.import_file_ops import safe_move_file
from database.music_database import MusicDatabase
from utils.logging_config import get_logger
logger = get_logger("import_guards")
def _get_config_manager():
from config.settings import config_manager
return config_manager
def move_to_quarantine(file_path: str, context: dict, reason: str, automation_engine=None) -> str:
"""Move a file to the quarantine folder and write a metadata sidecar."""
config_manager = _get_config_manager()
download_dir = config_manager.get("soulseek.download_path", "./downloads")
quarantine_dir = Path(download_dir) / "ss_quarantine"
quarantine_dir.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
original_name = Path(file_path).stem
file_ext = Path(file_path).suffix
quarantine_filename = f"{timestamp}_{original_name}{file_ext}.quarantined"
quarantine_path = quarantine_dir / quarantine_filename
safe_move_file(file_path, str(quarantine_path))
metadata_path = quarantine_dir / f"{timestamp}_{original_name}.json"
context = normalize_import_context(context)
original_search = get_import_original_search(context)
artist_context = get_import_context_artist(context)
metadata = {
"original_filename": Path(file_path).name,
"quarantine_reason": reason,
"timestamp": datetime.now().isoformat(),
"expected_track": get_import_clean_title(context, default=original_search.get("title", "Unknown")),
"expected_artist": get_import_clean_artist(context, default=(artist_context.get("name", "") if isinstance(artist_context, dict) else "Unknown")),
"context_key": context.get("context_key", "unknown"),
}
try:
with open(metadata_path, "w", encoding="utf-8") as f:
json.dump(metadata, f, indent=2, ensure_ascii=False)
except Exception as exc:
logger.warning("Failed to write quarantine metadata: %s", exc)
logger.warning("File quarantined: %s - Reason: %s", quarantine_path, reason)
if automation_engine:
try:
ti = context.get("track_info", {})
artists = ti.get("artists", [])
artist_name = ""
if artists:
first = artists[0]
artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first)
automation_engine.emit(
"download_quarantined",
{
"artist": artist_name,
"title": ti.get("name", ""),
"reason": reason or "Unknown",
},
)
except Exception:
pass
return str(quarantine_path)
def check_flac_bit_depth(file_path: str, context: dict) -> Optional[str]:
"""Return a rejection message if a FLAC file violates the configured bit depth."""
if not context.get("_audio_quality", "").startswith("FLAC"):
return None
config_manager = _get_config_manager()
quality_profile = MusicDatabase().get_quality_profile()
flac_config = quality_profile.get("qualities", {}).get("flac", {})
flac_pref = flac_config.get("bit_depth", "any")
if flac_pref == "any":
return None
actual_bits = context["_audio_quality"].replace("FLAC ", "").replace("bit", "")
if actual_bits == flac_pref:
return None
flac_fallback = flac_config.get("bit_depth_fallback", True)
downsample_enabled = config_manager.get("lossy_copy.downsample_hires", False)
track_info = context.get("track_info", {})
track_name = track_info.get("name", os.path.basename(file_path))
if flac_fallback or downsample_enabled:
if downsample_enabled:
logger.info("[FLAC Downsample] Accepted %s-bit FLAC (will be downsampled to %s-bit): %s", actual_bits, flac_pref, track_name)
else:
logger.warning("[FLAC Fallback] Accepted %s-bit FLAC (preferred %s-bit): %s", actual_bits, flac_pref, track_name)
return None
return f"FLAC bit depth mismatch: file is {actual_bits}-bit, preference is {flac_pref}-bit"

757
core/import_paths.py Normal file
View file

@ -0,0 +1,757 @@
"""Shared path and naming helpers for import processing."""
from __future__ import annotations
import json
import logging
import os
import re
import threading
from pathlib import Path
from typing import Any
from core.import_context import (
get_import_clean_title,
get_import_context_album,
get_import_original_search,
get_import_source,
get_import_track_info,
normalize_import_context,
)
logger = logging.getLogger("import_paths")
_album_cache_lock = threading.Lock()
_album_editions: dict[str, str] = {}
_album_name_cache: dict[str, str] = {}
def _get_config_manager():
try:
from config.settings import config_manager
return config_manager
except Exception:
class _FallbackConfig:
@staticmethod
def get(key, default=None):
return default
return _FallbackConfig()
def _get_itunes_client():
try:
from core.metadata_service import get_itunes_client
return get_itunes_client()
except Exception:
return None
def _get_album_tracks_for_source(source: str, album_id: str):
try:
from core.metadata_service import get_album_tracks_for_source
return get_album_tracks_for_source(source, album_id)
except Exception:
return None
def _extract_artist_name(artist_context: Any) -> str:
if not artist_context:
return ""
if isinstance(artist_context, dict):
return str(artist_context.get("name", "") or "").strip()
return str(artist_context).strip()
def docker_resolve_path(path_str: str) -> str:
"""Resolve Docker-hosted Windows paths into container paths."""
if os.path.exists("/.dockerenv") and len(path_str) >= 3 and path_str[1] == ":" and path_str[0].isalpha():
drive_letter = path_str[0].lower()
rest_of_path = path_str[2:].replace("\\", "/")
return f"/host/mnt/{drive_letter}{rest_of_path}"
return path_str
def build_simple_download_destination(context, file_path: str):
"""Build the destination path for a simple download into Transfer."""
context = normalize_import_context(context)
search_result = context.get("search_result", {}) or {}
if not isinstance(search_result, dict):
search_result = {}
transfer_dir = Path(docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer")))
album_name = None
original_filename = search_result.get("filename", "")
if "/" in original_filename or "\\" in original_filename:
path_parts = original_filename.replace("\\", "/").split("/")
if len(path_parts) >= 2:
album_name = path_parts[-2]
if not album_name:
album_value = search_result.get("album")
if isinstance(album_value, dict):
album_name = album_value.get("name", "")
else:
album_name = album_value
filename = Path(file_path).name
if album_name and str(album_name).lower() not in {"unknown", "unknown album", ""}:
album_name = sanitize_filename(str(album_name))
destination_dir = transfer_dir / album_name
else:
album_name = ""
destination_dir = transfer_dir
destination_dir.mkdir(parents=True, exist_ok=True)
return destination_dir / filename, album_name, filename
def sanitize_filename(filename: str) -> str:
"""Sanitize filename for file system compatibility."""
sanitized = re.sub(r'[<>:"/\\|?*]', "_", filename)
sanitized = re.sub(r"\s+", " ", sanitized).strip()
sanitized = sanitized.rstrip(". ") or "_"
if re.match(r"^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)", sanitized, re.IGNORECASE):
sanitized = "_" + sanitized
return sanitized[:200]
def sanitize_context_values(context: dict) -> dict:
"""Sanitize all string values in a template context for path safety."""
sanitized = {}
for key, value in context.items():
if isinstance(value, str) and value:
sanitized[key] = sanitize_filename(value)
else:
sanitized[key] = value
return sanitized
def clean_track_title(track_title: str, artist_name: str) -> str:
"""Clean up track title by removing artist prefix and other noise."""
original = (track_title or "").strip()
cleaned = original
cleaned = re.sub(r"^\d{1,2}[\.\s\-]+", "", cleaned)
artist_pattern = re.escape(artist_name or "") + r"\s*-\s*"
cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"^[A-Za-z0-9\.]+\s*-\s*\d{1,2}\s*-\s*", "", cleaned)
quality_patterns = [
r"\s*[\[\(][0-9]+\s*kbps[\]\)]\s*",
r"\s*[\[\(]flac[\]\)]\s*",
r"\s*[\[\(]mp3[\]\)]\s*",
]
for pattern in quality_patterns:
cleaned = re.sub(pattern, "", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"^[-\s\.]+", "", cleaned)
cleaned = re.sub(r"[-\s\.]+$", "", cleaned)
cleaned = re.sub(r"\s+", " ", cleaned).strip()
return cleaned if cleaned else original
def get_base_album_name(album_name: str) -> str:
"""Extract the base album name without edition indicators."""
base_name = album_name or ""
base_name = re.sub(
r"\s*[\[\(][^)\]]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^)\]]*[\]\)]\s*$",
"",
base_name,
flags=re.IGNORECASE,
)
base_name = re.sub(r"\s*[\[\(][^)\]]*\bedition\b[^)\]]*[\]\)]\s*$", "", base_name, flags=re.IGNORECASE)
base_name = re.sub(
r"\s+(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$",
"",
base_name,
flags=re.IGNORECASE,
)
return base_name.strip()
def detect_deluxe_edition(album_name: str) -> bool:
"""Detect if an album name indicates a deluxe/special edition."""
if not album_name:
return False
album_lower = album_name.lower()
deluxe_indicators = [
"deluxe",
"deluxe edition",
"special edition",
"expanded edition",
"extended edition",
"bonus",
"remastered",
"anniversary",
"collectors edition",
"limited edition",
"silver edition",
"gold edition",
"platinum edition",
]
for indicator in deluxe_indicators:
if indicator in album_lower:
logger.info("Detected deluxe edition: %r contains %r", album_name, indicator)
return True
return False
def normalize_base_album_name(base_album: str, artist_name: str) -> str:
"""Normalize the base album name to handle case variations and known corrections."""
normalized_lower = (base_album or "").lower().strip()
known_corrections = {
# Add specific album name corrections here as needed.
}
for variant, correction in known_corrections.items():
if normalized_lower == variant.lower():
logger.info("Album correction applied: %r -> %r", base_album, correction)
return correction
normalized = base_album or ""
normalized = re.sub(r"\s*&\s*", " & ", normalized)
normalized = re.sub(r"\s+", " ", normalized)
normalized = normalized.strip()
logger.info("Album variant normalization: %r -> %r", base_album, normalized)
return normalized
def clean_album_title(album_title: str, artist_name: str) -> str:
"""Clean up album title by removing common prefixes, suffixes, and artist redundancy."""
original = (album_title or "").strip()
cleaned = original
logger.info("Album Title Cleaning: %r (artist: %r)", original, artist_name)
cleaned = re.sub(r"^Album\s*-\s*", "", cleaned, flags=re.IGNORECASE)
artist_pattern = re.escape(artist_name or "") + r"\s*-\s*"
cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"\s*[\[\(]\d{4}[\]\)]\s*", " ", cleaned)
quality_patterns = [
r"\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*",
r"\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*",
r"\s*[\[\(].*?flac.*?[\]\)]\s*",
r"\s*[\[\(].*?mp3.*?[\]\)]\s*",
r"\s*[\[\(].*?itunes.*?[\]\)]\s*",
r"\s*[\[\(].*?web.*?[\]\)]\s*",
r"\s*[\[\(].*?cd.*?[\]\)]\s*",
]
for pattern in quality_patterns:
cleaned = re.sub(pattern, " ", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"\s*[\[\(][^\]\)]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"\s*[\[\(][^\]\)]*\bedition\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"\s*(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", "", cleaned, flags=re.IGNORECASE)
cleaned = re.sub(r"^[-\s\.]+", "", cleaned)
cleaned = re.sub(r"[-\s\.]+$", "", cleaned)
cleaned = re.sub(r"\s+", " ", cleaned).strip()
return cleaned if cleaned else original
def resolve_album_group(artist_context: dict, album_info: dict, original_album: str = None) -> str:
"""Smart album grouping: upgrade to deluxe if any track is deluxe."""
try:
with _album_cache_lock:
artist_name = _extract_artist_name(artist_context)
detected_album = (album_info or {}).get("album_name", "")
if detected_album:
base_album = get_base_album_name(detected_album)
elif original_album:
cleaned_original = clean_album_title(original_album, artist_name)
base_album = get_base_album_name(cleaned_original)
else:
base_album = get_base_album_name(detected_album)
base_album = normalize_base_album_name(base_album, artist_name)
album_key = f"{artist_name}::{base_album}"
is_deluxe_track = False
if detected_album:
is_deluxe_track = detect_deluxe_edition(detected_album)
elif original_album:
is_deluxe_track = detect_deluxe_edition(original_album)
if album_key in _album_name_cache:
cached_name = _album_name_cache[album_key]
current_edition = _album_editions.get(album_key, "standard")
if is_deluxe_track and current_edition == "standard":
final_album_name = f"{base_album} (Deluxe Edition)"
_album_editions[album_key] = "deluxe"
_album_name_cache[album_key] = final_album_name
logger.info("Album cache upgrade: %r -> %r", album_key, final_album_name)
return final_album_name
logger.info("Using cached album name for %r: %r", album_key, cached_name)
return cached_name
logger.info("Album grouping - Key: %r, Detected: %r", album_key, detected_album)
current_edition = _album_editions.get(album_key, "standard")
if is_deluxe_track and current_edition == "standard":
logger.info("UPGRADE: Album %r upgraded from standard to deluxe!", base_album)
_album_editions[album_key] = "deluxe"
current_edition = "deluxe"
if current_edition == "deluxe":
final_album_name = f"{base_album} (Deluxe Edition)"
else:
final_album_name = base_album
_album_name_cache[album_key] = final_album_name
logger.info("Album resolution: %r -> %r (edition: %s)", detected_album, final_album_name, current_edition)
return final_album_name
except Exception as e:
logger.error("Error resolving album group: %s", e)
album_name = (album_info or {}).get("album_name", "Unknown Album")
return album_name
def get_album_type_display(raw_type, track_count) -> str:
"""Return the display form of an album's type for the $albumtype template variable."""
raw = (raw_type or "").strip().lower()
try:
tc = int(track_count or 0)
except (TypeError, ValueError):
tc = 0
if raw in ("compilation", "compile"):
return "Compilation"
if raw == "album":
return "Album"
if raw in ("single", "ep"):
if tc <= 3:
return "Single"
if tc <= 6:
return "EP"
return "Album"
if tc <= 0:
return "Album"
if tc <= 3:
return "Single"
if tc <= 6:
return "EP"
return "Album"
def _replace_template_variables(template: str, context: dict) -> str:
clean_context = sanitize_context_values(context)
result = template
album_artist_value = clean_context.get("albumartist", clean_context.get("artist", "Unknown Artist"))
collab_mode = _get_config_manager().get("file_organization.collab_artist_mode", "first")
if collab_mode == "first" and album_artist_value:
artists_list = context.get("_artists_list")
if artists_list and len(artists_list) > 1:
first = artists_list[0]
album_artist_value = first.get("name", first) if isinstance(first, dict) else str(first)
elif artists_list and len(artists_list) == 1:
itunes_artist_id = context.get("_itunes_artist_id")
if itunes_artist_id and ("," in album_artist_value or " & " in album_artist_value):
try:
resolved_client = _get_itunes_client()
if resolved_client and hasattr(resolved_client, "resolve_primary_artist"):
resolved = resolved_client.resolve_primary_artist(itunes_artist_id)
if resolved and resolved != album_artist_value:
album_artist_value = resolved
except Exception:
pass
bracket_map = {
"albumartist": album_artist_value,
"albumtype": clean_context.get("albumtype", "Album"),
"playlist": clean_context.get("playlist_name", ""),
"artistletter": (clean_context.get("artist", "U") or "U")[0].upper(),
"artist": clean_context.get("artist", "Unknown Artist"),
"album": clean_context.get("album", "Unknown Album"),
"title": clean_context.get("title", "Unknown Track"),
"track": f"{_coerce_int(clean_context.get('track_number', 1), 1):02d}",
"disc": str(_coerce_int(clean_context.get("disc_number", 1), 1)),
"discnum": str(_coerce_int(clean_context.get("disc_number", 1), 1)),
"year": str(clean_context.get("year", "")),
"quality": clean_context.get("quality", ""),
}
for var_name, val in bracket_map.items():
result = result.replace("${" + var_name + "}", val)
result = result.replace("$albumartist", album_artist_value)
result = result.replace("$albumtype", clean_context.get("albumtype", "Album"))
result = result.replace("$playlist", clean_context.get("playlist_name", ""))
result = result.replace("$artistletter", (clean_context.get("artist", "U") or "U")[0].upper())
result = result.replace("$artist", clean_context.get("artist", "Unknown Artist"))
result = result.replace("$album", clean_context.get("album", "Unknown Album"))
result = result.replace("$title", clean_context.get("title", "Unknown Track"))
result = result.replace("$track", f"{clean_context.get('track_number', 1):02d}")
result = result.replace("$year", str(clean_context.get("year", "")))
result = re.sub(r"\s+", " ", result)
result = re.sub(r"\s*-\s*-\s*", " - ", result)
result = result.strip()
return result
def apply_path_template(template: str, context: dict) -> str:
"""Apply a template to build a path string."""
return _replace_template_variables(template, context)
def get_file_path_from_template_raw(template: str, context: dict) -> tuple[str, str]:
"""Build file path using a user-provided template string directly."""
full_path = apply_path_template(template, context)
quality_value = context.get("quality", "")
disc_number = _coerce_int(context.get("disc_number", 1), 1)
disc_value = f"{disc_number:02d}"
disc_value_raw = str(disc_number)
path_parts = full_path.split("/")
if len(path_parts) > 1:
folder_parts = path_parts[:-1]
filename_base = path_parts[-1]
cleaned_folders = []
for part in folder_parts:
part = part.replace("$quality", "")
part = part.replace("$discnum", "")
part = part.replace("$disc", "")
part = re.sub(r"\s*\[\s*\]", "", part)
part = re.sub(r"\s*\(\s*\)", "", part)
part = re.sub(r"\s*\{\s*\}", "", part)
part = re.sub(r"\s*-\s*$", "", part)
part = re.sub(r"^\s*-\s*", "", part)
part = re.sub(r"\s+", " ", part).strip()
if part:
cleaned_folders.append(part)
filename_base = filename_base.replace("$quality", quality_value)
filename_base = filename_base.replace("$discnum", disc_value_raw)
filename_base = filename_base.replace("$disc", disc_value)
filename_base = re.sub(r"\s*\[\s*\]", "", filename_base)
filename_base = re.sub(r"\s*\(\s*\)", "", filename_base)
filename_base = re.sub(r"\s*\{\s*\}", "", filename_base)
filename_base = re.sub(r"\s*-\s*$", "", filename_base)
filename_base = re.sub(r"\s+", " ", filename_base).strip()
sanitized_folders = [sanitize_filename(part) for part in cleaned_folders]
folder_path = os.path.join(*sanitized_folders) if sanitized_folders else ""
return folder_path, sanitize_filename(filename_base)
full_path = full_path.replace("$quality", quality_value)
full_path = full_path.replace("$discnum", disc_value_raw)
full_path = full_path.replace("$disc", disc_value)
full_path = re.sub(r"\s*\[\s*\]", "", full_path)
full_path = re.sub(r"\s*\(\s*\)", "", full_path)
full_path = re.sub(r"\s*\{\s*\}", "", full_path)
full_path = re.sub(r"\s*-\s*$", "", full_path)
full_path = re.sub(r"\s+", " ", full_path).strip()
return "", sanitize_filename(full_path)
def get_file_path_from_template(context: dict, template_type: str = "album_path") -> tuple[str, str]:
"""Build complete file path using configured templates."""
if not _get_config_manager().get("file_organization.enabled", True):
return None, None
templates = _get_config_manager().get("file_organization.templates", {})
template = templates.get(template_type)
if not template:
default_templates = {
"album_path": "$albumartist/$albumartist - $album/$track - $title",
"single_path": "$artist/$artist - $title/$title",
"compilation_path": "Compilations/$album/$track - $artist - $title",
"playlist_path": "$playlist/$artist - $title",
}
template = default_templates.get(template_type, "$artist/$album/$track - $title")
full_path = apply_path_template(template, context)
path_parts = full_path.split("/")
quality_value = context.get("quality", "")
disc_number = _coerce_int(context.get("disc_number", 1), 1)
disc_value = f"{disc_number:02d}"
disc_value_raw = str(disc_number)
if len(path_parts) > 1:
folder_parts = path_parts[:-1]
filename_base = path_parts[-1]
cleaned_folders = []
for part in folder_parts:
part = part.replace("$quality", "")
part = part.replace("$discnum", "")
part = part.replace("$disc", "")
part = re.sub(r"\s*\[\s*\]", "", part)
part = re.sub(r"\s*\(\s*\)", "", part)
part = re.sub(r"\s*\{\s*\}", "", part)
part = re.sub(r"\s*-\s*$", "", part)
part = re.sub(r"^\s*-\s*", "", part)
part = re.sub(r"\s+", " ", part).strip()
if part:
cleaned_folders.append(part)
filename_base = filename_base.replace("$quality", quality_value)
filename_base = filename_base.replace("$discnum", disc_value_raw)
filename_base = filename_base.replace("$disc", disc_value)
filename_base = re.sub(r"\s*\[\s*\]", "", filename_base)
filename_base = re.sub(r"\s*\(\s*\)", "", filename_base)
filename_base = re.sub(r"\s*\{\s*\}", "", filename_base)
filename_base = re.sub(r"\s*-\s*$", "", filename_base)
filename_base = re.sub(r"\s+", " ", filename_base).strip()
sanitized_folders = [sanitize_filename(part) for part in cleaned_folders]
folder_path = os.path.join(*sanitized_folders) if sanitized_folders else ""
filename = sanitize_filename(filename_base)
return folder_path, filename
full_path = full_path.replace("$quality", quality_value)
full_path = full_path.replace("$discnum", disc_value_raw)
full_path = full_path.replace("$disc", disc_value)
full_path = re.sub(r"\s*\[\s*\]", "", full_path)
full_path = re.sub(r"\s*\(\s*\)", "", full_path)
full_path = re.sub(r"\s*\{\s*\}", "", full_path)
full_path = re.sub(r"\s*-\s*$", "", full_path)
full_path = re.sub(r"\s+", " ", full_path).strip()
return "", sanitize_filename(full_path)
def _max_disc_number(album_tracks: Any) -> int:
items = []
if isinstance(album_tracks, dict):
items = album_tracks.get("items") or album_tracks.get("tracks") or []
elif isinstance(album_tracks, list):
items = album_tracks
max_disc = 1
for track in items:
if not isinstance(track, dict):
continue
try:
disc_number = int(track.get("disc_number", 1) or 1)
except (TypeError, ValueError):
disc_number = 1
if disc_number > max_disc:
max_disc = disc_number
return max_disc
def _coerce_int(value: Any, default: int = 1) -> int:
try:
coerced = int(value)
except (TypeError, ValueError):
return default
return coerced if coerced > 0 else default
def build_final_path_for_track(context, artist_context, album_info, file_ext):
"""Shared path builder used by both post-processing and verification."""
transfer_dir = docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer"))
context = normalize_import_context(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
album_context = get_import_context_album(context)
source = get_import_source(context)
playlist_folder_mode = track_info.get("_playlist_folder_mode", False)
artist_name = _extract_artist_name(artist_context)
source_info = track_info.get("source_info") or {}
if isinstance(source_info, str):
try:
source_info = json.loads(source_info)
except (json.JSONDecodeError, TypeError):
source_info = {}
if source_info.get("enhance") and source_info.get("original_file_path"):
original_path = source_info["original_file_path"]
original_dir = os.path.dirname(original_path)
original_stem = os.path.splitext(os.path.basename(original_path))[0]
final_path = os.path.join(original_dir, original_stem + file_ext)
os.makedirs(original_dir, exist_ok=True)
logger.info("[Enhance] Using original file location: %s", final_path)
return final_path, True
year = ""
if album_context and album_context.get("release_date"):
release_date = album_context["release_date"]
if release_date and len(release_date) >= 4:
year = release_date[:4]
raw_album_type = ""
if album_context:
raw_album_type = album_context.get("album_type", "") or ""
total_tracks = (album_context.get("total_tracks", 0) or 0) if album_context else 0
album_type_display = get_album_type_display(raw_album_type, total_tracks)
if playlist_folder_mode:
playlist_name = track_info.get("_playlist_name", "Unknown Playlist")
track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track"))
_artists = original_search.get("artists") or track_info.get("artists") or []
template_context = {
"artist": artist_name,
"albumartist": artist_name,
"album": track_name,
"title": track_name,
"playlist_name": playlist_name,
"track_number": 1,
"disc_number": 1,
"year": year,
"quality": context.get("_audio_quality", ""),
"albumtype": album_type_display,
"_artists_list": _artists,
"_itunes_artist_id": str(artist_context.get("id", "")) if isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source == "itunes" else None,
}
folder_path, filename_base = get_file_path_from_template(template_context, "playlist_path")
if folder_path and filename_base:
final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
return final_path, True
playlist_name_sanitized = sanitize_filename(playlist_name)
playlist_dir = os.path.join(transfer_dir, playlist_name_sanitized)
os.makedirs(playlist_dir, exist_ok=True)
artist_name_sanitized = sanitize_filename(template_context["artist"])
track_name_sanitized = sanitize_filename(track_name)
new_filename = f"{artist_name_sanitized} - {track_name_sanitized}{file_ext}"
return os.path.join(playlist_dir, new_filename), True
if album_info and album_info.get("is_album"):
clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track"))
track_number = _coerce_int(album_info.get("track_number", 1), 1)
disc_number = _coerce_int(album_info.get("disc_number", 1), 1)
_artists = original_search.get("artists") or track_info.get("artists") or []
_album_ctx = album_context
_itunes_aid = None
_is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer")
if _is_itunes and isinstance(artist_context, dict):
_aid = artist_context.get("id", "")
if str(_aid).isdigit():
_itunes_aid = str(_aid)
if not _itunes_aid and _album_ctx:
_ext = _album_ctx.get("external_urls", {})
if isinstance(_ext, dict) and _ext.get("itunes_artist_id"):
_itunes_aid = _ext["itunes_artist_id"]
_artist_name = artist_name
_album_artist_name = _artist_name
_album_artists_for_collab = None
_explicit_artist_ctx = track_info.get("_explicit_artist_context") if isinstance(track_info, dict) else None
if isinstance(_explicit_artist_ctx, dict) and _explicit_artist_ctx.get("name"):
_album_artist_name = _explicit_artist_ctx["name"]
_album_artists_for_collab = [_explicit_artist_ctx]
elif isinstance(_explicit_artist_ctx, str) and _explicit_artist_ctx:
_album_artist_name = _explicit_artist_ctx
_album_artists_for_collab = [{"name": _explicit_artist_ctx}]
else:
_sa_artists = _album_ctx.get("artists", []) if _album_ctx else []
if _sa_artists:
_first_sa = _sa_artists[0]
if isinstance(_first_sa, dict) and _first_sa.get("name"):
_album_artist_name = _first_sa["name"]
elif isinstance(_first_sa, str) and _first_sa:
_album_artist_name = _first_sa
_album_artists_for_collab = _sa_artists
template_context = {
"artist": _artist_name,
"albumartist": _album_artist_name,
"album": album_info["album_name"],
"title": clean_track_name,
"track_number": track_number,
"disc_number": disc_number,
"year": year,
"quality": context.get("_audio_quality", ""),
"albumtype": album_type_display,
"_artists_list": _album_artists_for_collab if _album_artists_for_collab else _artists,
"_itunes_artist_id": _itunes_aid,
}
total_discs = _coerce_int(album_context.get("total_discs", 1) if album_context else 1, 1)
if total_discs <= 1 and album_context and album_context.get("id"):
if disc_number > 1:
total_discs = disc_number
else:
try:
_album_tracks = _get_album_tracks_for_source(source, str(album_context["id"]))
if _album_tracks:
total_discs = _max_disc_number(_album_tracks)
if total_discs > 1:
album_context["total_discs"] = total_discs
logger.info(
"[Multi-Disc] Resolved %s discs for single-track download of %r",
total_discs,
album_context.get("name"),
)
except Exception as _disc_err:
logger.warning("[Multi-Disc] Could not resolve total_discs: %s", _disc_err)
album_template = _get_config_manager().get("file_organization.templates.album_path", "")
user_controls_disc = "$disc" in album_template
disc_label = _get_config_manager().get("file_organization.disc_label", "Disc")
folder_path, filename_base = get_file_path_from_template(template_context, "album_path")
if folder_path and filename_base:
if total_discs > 1 and not user_controls_disc:
disc_folder = f"{disc_label} {disc_number}"
final_path = os.path.join(transfer_dir, folder_path, disc_folder, filename_base + file_ext)
os.makedirs(os.path.join(transfer_dir, folder_path, disc_folder), exist_ok=True)
else:
final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
return final_path, True
artist_name_sanitized = sanitize_filename(template_context["albumartist"])
album_name_sanitized = sanitize_filename(album_info["album_name"])
artist_dir = os.path.join(transfer_dir, artist_name_sanitized)
album_folder_name = f"{artist_name_sanitized} - {album_name_sanitized}"
album_dir = os.path.join(artist_dir, album_folder_name)
if total_discs > 1:
album_dir = os.path.join(album_dir, f"{disc_label} {disc_number}")
os.makedirs(album_dir, exist_ok=True)
final_track_name_sanitized = sanitize_filename(clean_track_name)
new_filename = f"{track_number:02d} - {final_track_name_sanitized}{file_ext}"
return os.path.join(album_dir, new_filename), True
clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track"))
_artists = original_search.get("artists") or track_info.get("artists") or []
_album_ctx = album_context
_itunes_aid = None
_is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer")
if _is_itunes and isinstance(artist_context, dict):
_aid = artist_context.get("id", "")
if str(_aid).isdigit():
_itunes_aid = str(_aid)
if not _itunes_aid and _album_ctx:
_ext = _album_ctx.get("external_urls", {})
if isinstance(_ext, dict) and _ext.get("itunes_artist_id"):
_itunes_aid = _ext["itunes_artist_id"]
template_context = {
"artist": artist_name,
"albumartist": artist_name,
"album": album_info.get("album_name", clean_track_name) if album_info else clean_track_name,
"title": clean_track_name,
"track_number": 1,
"disc_number": 1,
"year": year,
"quality": context.get("_audio_quality", ""),
"albumtype": album_type_display,
"_artists_list": _artists,
"_itunes_artist_id": _itunes_aid,
}
folder_path, filename_base = get_file_path_from_template(template_context, "single_path")
if filename_base:
if folder_path:
final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
else:
final_path = os.path.join(transfer_dir, filename_base + file_ext)
os.makedirs(transfer_dir, exist_ok=True)
return final_path, True
artist_name_sanitized = sanitize_filename(template_context["artist"])
final_track_name_sanitized = sanitize_filename(clean_track_name)
artist_dir = os.path.join(transfer_dir, artist_name_sanitized)
single_folder_name = f"{artist_name_sanitized} - {final_track_name_sanitized}"
single_dir = os.path.join(artist_dir, single_folder_name)
os.makedirs(single_dir, exist_ok=True)
new_filename = f"{final_track_name_sanitized}{file_ext}"
return os.path.join(single_dir, new_filename), True

925
core/import_pipeline.py Normal file
View file

@ -0,0 +1,925 @@
"""Import/post-processing pipeline for downloads and imported files."""
from __future__ import annotations
import json
import os
import threading
import time
from config.settings import config_manager
from core.import_file_ops import (
cleanup_empty_directories,
create_lossy_copy,
downsample_hires_flac,
extract_track_number_from_filename,
get_audio_quality_string,
get_quality_tier_from_extension,
safe_move_file,
)
from core.import_context import (
build_import_album_info,
extract_artist_name,
get_import_clean_artist,
get_import_clean_title,
get_import_context_artist,
get_import_has_clean_metadata,
get_import_original_search,
get_import_source,
get_import_track_info,
normalize_import_context,
)
from core.import_guards import check_flac_bit_depth, move_to_quarantine
from core.import_side_effects import (
check_and_remove_from_wishlist,
emit_track_downloaded,
record_download_provenance,
record_library_history_download,
record_retag_download,
record_soulsync_library_entry,
)
from core.import_runtime_state import (
add_activity_item,
detect_album_info_web,
download_batches,
download_tasks,
matched_context_lock,
matched_downloads_context,
mark_task_completed as _mark_task_completed,
_post_process_locks,
_post_process_locks_lock,
_processed_download_ids,
tasks_lock,
)
from core.metadata_enrichment import (
download_cover_art,
enhance_file_metadata,
generate_lrc_file,
wipe_source_tags,
)
from core.import_paths import (
build_final_path_for_track,
build_simple_download_destination,
docker_resolve_path,
resolve_album_group,
)
from database.music_database import get_database
from utils.logging_config import get_logger
logger = get_logger("import_pipeline")
pp_logger = get_logger("post_processing")
def post_process_matched_download(context_key, context, file_path, runtime):
on_download_completed = getattr(runtime, "on_download_completed", None)
automation_engine = getattr(runtime, "automation_engine", None)
web_scan_manager = getattr(runtime, "web_scan_manager", None)
repair_worker = getattr(runtime, "repair_worker", None)
def _notify_download_completed(batch_id, task_id, success=True):
if on_download_completed:
on_download_completed(batch_id, task_id, success=success)
with _post_process_locks_lock:
if context_key not in _post_process_locks:
_post_process_locks[context_key] = threading.Lock()
file_lock = _post_process_locks[context_key]
file_lock.acquire()
try:
if not os.path.exists(file_path):
existing_final = context.get('_final_processed_path')
if existing_final and os.path.exists(existing_final):
logger.info(
f"[Race Guard] Source gone but destination exists — already processed by another thread: "
f"{os.path.basename(existing_final)}"
)
return
logger.error(
f"[Race Guard] Source file gone and no known destination — marking as failed: "
f"{os.path.basename(file_path)}"
)
context['_race_guard_failed'] = True
return
_basename = os.path.basename(file_path)
_prev_size = -1
for _stability_check in range(5):
try:
_cur_size = os.path.getsize(file_path)
except OSError:
_cur_size = -1
if _cur_size == _prev_size and _cur_size > 0:
break
_prev_size = _cur_size
if _stability_check == 0:
logger.info(f"Waiting for file to stabilise: {_basename} ({_cur_size} bytes)")
time.sleep(1.5)
else:
logger.info(f"File may still be writing after stability checks: {_basename} ({_prev_size} bytes)")
_skip_acoustid = False
try:
from core.acoustid_verification import AcoustIDVerification, VerificationResult
verifier = AcoustIDVerification()
available, available_reason = verifier.quick_check_available()
if available and not _skip_acoustid:
context = normalize_import_context(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
artist_context = get_import_context_artist(context)
expected_track = get_import_clean_title(context, default=original_search.get('title', ''))
expected_artist = ''
track_artists = track_info.get('artists', [])
if track_artists:
first = track_artists[0]
if isinstance(first, dict):
expected_artist = first.get('name', '')
elif isinstance(first, str):
expected_artist = first
if not expected_artist:
expected_artist = extract_artist_name(artist_context) or get_import_clean_artist(context, default='')
if expected_track and expected_artist:
logger.info(f"Running AcoustID verification for: '{expected_track}' by '{expected_artist}'")
verification_result, verification_msg = verifier.verify_audio_file(
file_path,
expected_track,
expected_artist,
context,
)
logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}")
context['_acoustid_result'] = verification_result.value
if verification_result == VerificationResult.FAIL:
try:
quarantine_path = move_to_quarantine(
file_path,
context,
verification_msg,
automation_engine,
)
logger.error(f"File quarantined due to verification failure: {quarantine_path}")
except Exception as quarantine_error:
logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}")
logger.error(f"Quarantine failed, deleting wrong file: {file_path}")
try:
os.remove(file_path)
except Exception as del_error:
logger.error(f"Could not delete wrong file either: {del_error}")
context['_acoustid_quarantined'] = True
context['_acoustid_failure_msg'] = verification_msg
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
task_id = context.get('task_id')
batch_id = context.get('batch_id')
if task_id:
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = (
f"AcoustID verification failed: {verification_msg}"
)
if task_id and batch_id:
_notify_download_completed(batch_id, task_id, success=False)
return
else:
logger.warning("AcoustID verification skipped: missing track/artist info")
context['_acoustid_result'] = 'skip'
else:
logger.info(f" AcoustID verification not available: {available_reason}")
context['_acoustid_result'] = 'disabled'
except Exception as verify_error:
logger.error(f"AcoustID verification error (continuing normally): {verify_error}")
context['_acoustid_result'] = 'error'
search_result = context.get('search_result', {}) or {}
if not isinstance(search_result, dict):
search_result = {}
is_simple_download = search_result.get('is_simple_download', False)
if is_simple_download:
logger.info(f"Processing simple download (no metadata enhancement): {file_path}")
destination, album_name, filename = build_simple_download_destination(context, file_path)
if album_name:
logger.info(f"Moving to album folder: {album_name}")
else:
logger.info("Moving to Transfer root (single track)")
safe_move_file(file_path, destination)
logger.info(f"Moved simple download to: {destination}")
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
if web_scan_manager:
threading.Thread(
target=lambda: web_scan_manager.request_scan("Simple download completed"),
daemon=True,
).start()
activity_target = f"{album_name}/{filename}" if album_name else filename
add_activity_item("", "Download Complete", activity_target, "Now")
logger.info(f"Simple download post-processing complete: {activity_target}")
context['_simple_download_completed'] = True
context['_final_path'] = str(destination)
emit_track_downloaded(context, automation_engine)
record_library_history_download(context)
record_download_provenance(context)
return
logger.info(f"Starting robust post-processing for: {context_key}")
context = normalize_import_context(context)
artist_context = get_import_context_artist(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
has_clean_metadata = get_import_has_clean_metadata(context)
if not artist_context:
logger.error("Post-processing failed: Missing artist context.")
return
_junk_artist_names = {'', 'unknown', 'unknown artist', 'various artists', 'none', 'null'}
_artist_name = (artist_context.get('name', '') if isinstance(artist_context, dict) else '').strip()
if _artist_name.lower() in _junk_artist_names:
logger.info(f"[Unknown Artist Guard] Artist name is '{_artist_name}' — attempting to resolve")
_resolved = False
track_info_guard = track_info or {}
original_search_guard = original_search or {}
_ti_artists = track_info_guard.get('artists', [])
if isinstance(_ti_artists, list) and _ti_artists:
_first = _ti_artists[0]
_name = _first.get('name', '') if isinstance(_first, dict) else str(_first)
if _name and _name.strip().lower() not in _junk_artist_names:
artist_context['name'] = _name.strip()
logger.info(f"[Unknown Artist Guard] Resolved from track_info.artists: '{_name}'")
_resolved = True
if not _resolved:
_os_artist = original_search_guard.get('artist') or original_search_guard.get('artist_name') or ''
if isinstance(_os_artist, str) and _os_artist.strip().lower() not in _junk_artist_names:
artist_context['name'] = _os_artist.strip()
logger.info(f"[Unknown Artist Guard] Resolved from original_search_result: '{_os_artist}'")
_resolved = True
if not _resolved:
_track_id = track_info_guard.get('id') or track_info_guard.get('track_id') or ''
if _track_id:
try:
from core.metadata_service import get_client_for_source, get_primary_source
_guard_source = get_import_source(context) or get_primary_source()
_fb_client = get_client_for_source(_guard_source) or get_client_for_source(get_primary_source())
if hasattr(_fb_client, 'get_track_details'):
_details = _fb_client.get_track_details(str(_track_id))
if _details and isinstance(_details, dict):
_d_artists = _details.get('artists', [])
if isinstance(_d_artists, list) and _d_artists:
_d_first = _d_artists[0]
_d_name = _d_first.get('name', '') if isinstance(_d_first, dict) else str(_d_first)
if _d_name and _d_name.strip().lower() not in _junk_artist_names:
artist_context['name'] = _d_name.strip()
logger.info(f"[Unknown Artist Guard] Resolved from metadata API: '{_d_name}'")
_resolved = True
except Exception as _guard_err:
logger.error(f"[Unknown Artist Guard] Metadata re-fetch failed: {_guard_err}")
if not _resolved:
logger.error(f"[Unknown Artist Guard] Could not resolve artist — proceeding with '{_artist_name}'")
context['artist'] = artist_context
playlist_folder_mode = track_info.get("_playlist_folder_mode", False)
logger.debug(f"[Debug] Post-processing - track_info type: {type(track_info)}, is None: {track_info is None}, is empty: {not track_info}")
logger.debug(f"[Debug] Post-processing - playlist_folder_mode: {playlist_folder_mode}")
if track_info:
logger.debug(f"[Debug] Post-processing - track_info keys: {list(track_info.keys())}")
if playlist_folder_mode:
playlist_name = track_info.get("_playlist_name", "Unknown Playlist")
logger.info(f"[Playlist Folder Mode] Organizing in playlist folder: {playlist_name}")
file_ext = os.path.splitext(file_path)[1]
final_path, _ = build_final_path_for_track(context, artist_context, None, file_ext)
logger.info(f"Playlist mode final path: '{final_path}'")
if not os.path.exists(file_path):
if os.path.exists(final_path):
logger.info(
f"[Playlist Folder Mode] Source gone but destination exists — already processed by another thread: "
f"{os.path.basename(final_path)}"
)
context['_final_processed_path'] = final_path
return
pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: Source file not found and destination does not exist: {file_path}")
raise FileNotFoundError(f"Source file not found and destination does not exist: {file_path}")
context['_audio_quality'] = get_audio_quality_string(file_path)
if context['_audio_quality']:
logger.info(f"Audio quality detected: {context['_audio_quality']}")
rejection_reason = check_flac_bit_depth(file_path, context)
if rejection_reason:
try:
quarantine_path = move_to_quarantine(
file_path,
context,
rejection_reason,
automation_engine,
)
logger.info(f"File quarantined due to bit depth filter: {quarantine_path}")
except Exception as quarantine_error:
logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}")
try:
os.remove(file_path)
except Exception:
pass
context['_bitdepth_rejected'] = True
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
task_id = context.get('task_id')
batch_id = context.get('batch_id')
if task_id:
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}"
if task_id and batch_id:
_notify_download_completed(batch_id, task_id, success=False)
return
try:
logger.warning(
f"[Metadata Input] Playlist mode - artist: '{artist_context.get('name', 'MISSING')}' "
f"(id: {artist_context.get('id', 'MISSING')})"
)
enhance_file_metadata(file_path, context, artist_context, None)
except Exception as meta_err:
import traceback
pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")
wipe_source_tags(file_path)
logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'")
safe_move_file(file_path, final_path)
context['_final_processed_path'] = final_path
if config_manager.get('post_processing.replaygain_enabled', False):
try:
from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF
if _rg_ffmpeg_ok():
lufs, peak_dbfs = _rg_analyze(final_path)
gain_db = _RG_REF - lufs
_rg_write(final_path, gain_db, peak_dbfs)
pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB — {os.path.basename(final_path)}")
except Exception as rg_err:
pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}")
downsampled_path = downsample_hires_flac(final_path, context)
if downsampled_path:
final_path = downsampled_path
context['_final_processed_path'] = final_path
blasphemy_path = create_lossy_copy(final_path)
if blasphemy_path:
context['_final_processed_path'] = blasphemy_path
downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
cleanup_empty_directories(downloads_path, file_path)
logger.info(f"[Playlist Folder Mode] Post-processing complete: {final_path}")
try:
check_and_remove_from_wishlist(context)
except Exception as wishlist_error:
logger.error(f"[Playlist Folder] Error checking wishlist removal: {wishlist_error}")
emit_track_downloaded(context, automation_engine)
record_library_history_download(context)
record_download_provenance(context)
task_id = context.get('task_id')
batch_id = context.get('batch_id')
if task_id and batch_id:
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['stream_processed'] = True
download_tasks[task_id]['status'] = 'completed'
logger.info(f"[Playlist Folder Mode] Marked task {task_id} as completed")
_notify_download_completed(batch_id, task_id, success=True)
return
is_album_download = bool(context.get("is_album_download", False))
album_info = build_import_album_info(context, force_album=is_album_download)
if is_album_download:
if has_clean_metadata:
logger.info("Album context with clean metadata found - using normalized album info")
else:
logger.warning("Album context found without clean metadata - using normalized album info")
elif not album_info.get('is_album'):
logger.info("Single track download - attempting album detection")
detected_album_info = detect_album_info_web(context, artist_context)
if detected_album_info:
album_info = detected_album_info
if album_info and album_info['is_album'] and not is_album_download:
logger.info(
"SMART ALBUM GROUPING for track=%r original_album=%r",
album_info.get('clean_track_name', 'Unknown'),
album_info.get('album_name', 'None'),
)
original_album = original_search.get("album") if original_search.get("album") else None
consistent_album_name = resolve_album_group(artist_context, album_info, original_album)
album_info['album_name'] = consistent_album_name
logger.info("Album grouping complete: final_album=%r", consistent_album_name)
elif album_info and album_info['is_album'] and is_album_download:
logger.info(
"EXPLICIT ALBUM DOWNLOAD - preserving album name=%r; skipping smart grouping",
album_info.get('album_name', 'None'),
)
context['_audio_quality'] = get_audio_quality_string(file_path)
if context['_audio_quality']:
logger.info(f"Audio quality detected: {context['_audio_quality']}")
rejection_reason = check_flac_bit_depth(file_path, context)
if rejection_reason:
try:
quarantine_path = move_to_quarantine(
file_path,
context,
rejection_reason,
automation_engine,
)
logger.info(f"File quarantined due to bit depth filter: {quarantine_path}")
except Exception as quarantine_error:
logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}")
try:
os.remove(file_path)
except Exception:
pass
context['_bitdepth_rejected'] = True
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
task_id = context.get('task_id')
batch_id = context.get('batch_id')
if task_id:
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}"
if task_id and batch_id:
_notify_download_completed(batch_id, task_id, success=False)
return
file_ext = os.path.splitext(file_path)[1]
clean_track_name = get_import_clean_title(
context,
album_info=album_info,
default=original_search.get('title', 'Unknown Track'),
)
track_number = album_info.get('track_number', 1)
logger.debug(
"Final track_number processing: source=%s album_info_track_number=%s track_number=%s",
album_info.get('source', 'unknown'),
album_info.get('track_number', 'NOT_FOUND'),
track_number,
)
if track_number is None:
track_number = extract_track_number_from_filename(file_path)
logger.info(
"Track number was None; extracted from filename=%r -> %s",
os.path.basename(file_path),
track_number,
)
if not isinstance(track_number, int) or track_number < 1:
logger.error(f"Invalid track number ({track_number}), defaulting to 1")
track_number = 1
logger.debug(f"FINAL track_number used for filename: {track_number}")
album_info['track_number'] = track_number
album_info['clean_track_name'] = clean_track_name
logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata")
final_path, _ = build_final_path_for_track(context, artist_context, album_info, file_ext)
logger.info(f"Resolved path: '{final_path}'")
context['_final_processed_path'] = final_path
try:
logger.warning(f"[Metadata Input] artist: '{artist_context.get('name', 'MISSING')}' (id: {artist_context.get('id', 'MISSING')})")
if album_info:
logger.warning(
f"[Metadata Input] album: '{album_info.get('album_name', 'MISSING')}', "
f"track#: {album_info.get('track_number', 'MISSING')}, disc#: {album_info.get('disc_number', 'MISSING')}, "
f"source: {album_info.get('source', 'unknown')}"
)
else:
logger.info("[Metadata Input] album_info: None (single track)")
enhance_file_metadata(file_path, context, artist_context, album_info)
except Exception as meta_err:
import traceback
pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")
wipe_source_tags(file_path)
_enhance_source_info = context.get('track_info', {}).get('source_info') or {}
if isinstance(_enhance_source_info, str):
try:
_enhance_source_info = json.loads(_enhance_source_info)
except (json.JSONDecodeError, TypeError):
_enhance_source_info = {}
is_enhance_download = _enhance_source_info.get('enhance', False)
logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'")
if os.path.exists(final_path):
if not os.path.exists(file_path):
logger.info(f"[Protection] Destination exists and source already gone - file already transferred: {os.path.basename(final_path)}")
return
try:
from mutagen import File as MutagenFile
existing_file = MutagenFile(final_path)
has_metadata = existing_file is not None and len(existing_file.tags or {}) > 2
if has_metadata and not is_enhance_download:
_replace_lower = config_manager.get('import.replace_lower_quality', False)
if _replace_lower:
_existing_tier = get_quality_tier_from_extension(final_path)
_incoming_tier = get_quality_tier_from_extension(file_path)
if _incoming_tier[1] < _existing_tier[1]:
logger.info(f"[Quality Replace] Replacing {_existing_tier[0]} with {_incoming_tier[0]}: {os.path.basename(final_path)}")
try:
os.remove(final_path)
except Exception as e:
logger.error(f"[Quality Replace] Could not remove existing file: {e}")
else:
logger.info(
f"[Protection] Existing file is same or better quality ({_existing_tier[0]} vs {_incoming_tier[0]}) - skipping: "
f"{os.path.basename(final_path)}"
)
try:
os.remove(file_path)
except FileNotFoundError:
pass
except Exception as e:
logger.error(f"[Protection] Error removing redundant file: {e}")
return
else:
logger.info(f"[Protection] Existing file already has metadata enhancement - skipping overwrite: {os.path.basename(final_path)}")
logger.info(f"[Protection] Removing redundant download file: {os.path.basename(file_path)}")
try:
os.remove(file_path)
except FileNotFoundError:
logger.error(f"[Protection] Could not remove redundant file (already gone): {file_path}")
except Exception as e:
logger.error(f"[Protection] Error removing redundant file: {e}")
return
elif is_enhance_download:
logger.info(f"[Enhance] Quality enhance mode — replacing existing file: {os.path.basename(final_path)}")
try:
os.remove(final_path)
except Exception as e:
logger.error(f"[Enhance] Could not remove existing file for replacement: {e}")
else:
logger.info(f"[Protection] Existing file lacks metadata - safe to overwrite: {os.path.basename(final_path)}")
try:
os.remove(final_path)
except FileNotFoundError:
pass
except Exception as check_error:
logger.error(f"[Protection] Error checking existing file metadata, proceeding with overwrite: {check_error}")
try:
if os.path.exists(final_path):
os.remove(final_path)
except Exception as e:
logger.error(f"[Protection] Failed to remove existing file for overwrite: {e}")
if not os.path.exists(file_path):
if os.path.exists(final_path):
logger.info(f"[Pre-Move] Source already gone and destination exists - another thread completed transfer: {os.path.basename(final_path)}")
download_cover_art(album_info, os.path.dirname(final_path), context)
generate_lrc_file(final_path, context, artist_context, album_info)
return
expected_dir = os.path.dirname(final_path)
expected_stem = os.path.splitext(os.path.basename(final_path))[0]
expected_ext = os.path.splitext(final_path)[1]
found_variant = None
check_exts = {expected_ext}
if expected_ext == '.flac' and config_manager.get('lossy_copy.enabled', False) and config_manager.get('lossy_copy.delete_original', False):
_lossy_ext_map = {'mp3': '.mp3', 'opus': '.opus', 'aac': '.m4a'}
_lossy_codec = config_manager.get('lossy_copy.codec', 'mp3')
check_exts.add(_lossy_ext_map.get(_lossy_codec, '.mp3'))
if os.path.exists(expected_dir):
for f in os.listdir(expected_dir):
f_ext = os.path.splitext(f)[1].lower()
if f_ext in check_exts and os.path.splitext(f)[0].startswith(expected_stem):
found_variant = os.path.join(expected_dir, f)
break
if found_variant:
logger.debug(f"[Pre-Move] Source gone but found variant in destination (stream processor handled it): {os.path.basename(found_variant)}")
context['_final_processed_path'] = found_variant
download_cover_art(album_info, expected_dir, context)
generate_lrc_file(found_variant, context, artist_context, album_info)
return
logger.warning(f"[Pre-Move] Source file gone and no matching file in destination: {os.path.basename(file_path)}")
raise FileNotFoundError(f"Source file vanished before move and destination does not exist: {file_path}")
safe_move_file(file_path, final_path)
if is_enhance_download and _enhance_source_info.get('original_file_path'):
original_enhance_path = _enhance_source_info['original_file_path']
if os.path.normpath(original_enhance_path) != os.path.normpath(final_path) and os.path.exists(original_enhance_path):
try:
os.remove(original_enhance_path)
old_fmt = os.path.splitext(original_enhance_path)[1]
new_fmt = os.path.splitext(final_path)[1]
logger.info(f"[Enhance] Upgraded {old_fmt}{new_fmt}: {os.path.basename(final_path)}")
except Exception as e:
logger.error(f"[Enhance] Could not remove old-format file: {e}")
elif is_enhance_download:
old_fmt = _enhance_source_info.get('original_format', 'unknown')
new_fmt = os.path.splitext(final_path)[1]
logger.info(f"[Enhance] Replaced in-place ({old_fmt}{new_fmt}): {os.path.basename(final_path)}")
download_cover_art(album_info, os.path.dirname(final_path), context)
generate_lrc_file(final_path, context, artist_context, album_info)
if config_manager.get('post_processing.replaygain_enabled', False):
try:
from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF
if _rg_ffmpeg_ok():
lufs, peak_dbfs = _rg_analyze(final_path)
gain_db = _RG_REF - lufs
_rg_write(final_path, gain_db, peak_dbfs)
pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB, peak {peak_dbfs:.2f} dBFS — {os.path.basename(final_path)}")
except Exception as rg_err:
pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}")
downsampled_path = downsample_hires_flac(final_path, context)
if downsampled_path:
final_path = downsampled_path
context['_final_processed_path'] = final_path
blasphemy_path = create_lossy_copy(final_path)
if blasphemy_path:
context['_final_processed_path'] = blasphemy_path
downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
cleanup_empty_directories(downloads_path, file_path)
logger.info(f"Post-processing complete for: {context.get('_final_processed_path', final_path)}")
emit_track_downloaded(context, automation_engine)
record_library_history_download(context)
record_download_provenance(context)
record_soulsync_library_entry(context, artist_context, album_info)
try:
if not playlist_folder_mode:
completed_path = context.get('_final_processed_path', final_path)
record_retag_download(context, artist_context, album_info, completed_path)
except Exception as retag_err:
logger.error(f"[Post-Process] Retag data capture failed (non-fatal): {retag_err}")
try:
completed_path = context.get('_final_processed_path', final_path)
batch_id_for_repair = context.get('batch_id')
if completed_path and batch_id_for_repair and repair_worker:
album_folder = os.path.dirname(str(completed_path))
if album_folder:
repair_worker.register_folder(batch_id_for_repair, album_folder)
except Exception as repair_err:
logger.error(f"[Post-Process] Repair folder registration failed: {repair_err}")
try:
completed_path = context.get('_final_processed_path', final_path)
batch_id_for_consistency = context.get('batch_id')
if completed_path and batch_id_for_consistency and album_info and album_info.get('is_album'):
_file_info = {
'path': str(completed_path),
'track_number': album_info.get('track_number', 1),
'disc_number': album_info.get('disc_number', 1),
'title': get_import_clean_title(
context,
album_info=album_info,
default=album_info.get('clean_track_name', ''),
),
}
with tasks_lock:
if batch_id_for_consistency in download_batches:
download_batches[batch_id_for_consistency].setdefault('_consistency_files', []).append(_file_info)
except Exception as cons_err:
logger.error(f"[Post-Process] Album consistency registration failed: {cons_err}")
try:
check_and_remove_from_wishlist(context)
except Exception as wishlist_error:
logger.error(f"[Post-Process] Error checking wishlist removal: {wishlist_error}")
task_id = context.get('task_id')
batch_id = context.get('batch_id')
if task_id and batch_id:
logger.info(f"[Post-Process] Calling completion callback for task {task_id} in batch {batch_id}")
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['stream_processed'] = True
download_tasks[task_id]['status'] = 'completed'
logger.info(f"[Post-Process] Marked task {task_id} as completed")
_notify_download_completed(batch_id, task_id, success=True)
except Exception as e:
import traceback
pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: {e}")
pp_logger.info(traceback.format_exc())
logger.error(f"\nCRITICAL ERROR in post-processing for {context_key}: {e}")
traceback.print_exc()
source_exists = os.path.exists(file_path) if file_path else False
if source_exists:
if context_key in _processed_download_ids:
_processed_download_ids.remove(context_key)
logger.warning(f"Removed {context_key} from processed set - will retry on next check")
with matched_context_lock:
if context_key not in matched_downloads_context:
matched_downloads_context[context_key] = context
logger.warning(f"Re-added {context_key} to context for retry")
else:
logger.warning(f"Source file gone, not retrying: {context_key}")
finally:
file_lock.release()
with _post_process_locks_lock:
_post_process_locks.pop(context_key, None)
def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime):
on_download_completed = getattr(runtime, "on_download_completed", None)
def _notify_download_completed(batch_id, task_id, success=True):
if on_download_completed:
on_download_completed(batch_id, task_id, success=success)
logger = pp_logger
try:
original_task_id = context.pop('task_id', None)
original_batch_id = context.pop('batch_id', None)
post_process_matched_download(context_key, context, file_path, runtime)
if original_task_id:
context['task_id'] = original_task_id
if original_batch_id:
context['batch_id'] = original_batch_id
if context.get('_race_guard_failed'):
logger.info(f"Race guard: source file gone for task {task_id} — marking as failed")
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = 'Source file was already processed or removed by another task'
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=False)
return
if context.get('_acoustid_quarantined'):
failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed')
logger.info(f"File was quarantined by AcoustID verification (task={task_id}): {failure_msg}")
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {failure_msg}"
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=False)
return
if context.get('_simple_download_completed'):
expected_final_path = context.get('_final_path')
if expected_final_path and os.path.exists(expected_final_path):
with tasks_lock:
if task_id in download_tasks:
_mark_task_completed(task_id, context.get('track_info'))
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=True)
return
logger.info(
f"FAILED simple download file not found at: {expected_final_path} "
f"(task={task_id}, context={context_key})"
)
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = (
f"Downloaded file not found at expected location: {os.path.basename(expected_final_path)}"
)
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=False)
return
expected_final_path = context.get('_final_processed_path')
if not expected_final_path:
logger.info(f"No _final_processed_path in context for task {task_id} — cannot verify, assuming success")
with tasks_lock:
if task_id in download_tasks:
_mark_task_completed(task_id, context.get('track_info'))
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=True)
return
if os.path.exists(expected_final_path):
redownload_ctx = None
with tasks_lock:
if task_id in download_tasks:
_mark_task_completed(task_id, context.get('track_info'))
download_tasks[task_id]['metadata_enhanced'] = True
redownload_ctx = download_tasks[task_id].get('_redownload_context')
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
if redownload_ctx:
try:
old_path = redownload_ctx.get('old_file_path')
lib_track_id = redownload_ctx.get('library_track_id')
if redownload_ctx.get('delete_old_file') and old_path and os.path.exists(old_path):
if os.path.normpath(old_path) != os.path.normpath(expected_final_path):
os.remove(old_path)
logger.info(f"[Redownload] Deleted old file: {old_path}")
if lib_track_id and expected_final_path:
_rd_db = get_database()
_rd_conn = _rd_db._get_connection()
_rd_cursor = _rd_conn.cursor()
_rd_cursor.execute(
"""
UPDATE tracks SET file_path = ?, updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(expected_final_path, lib_track_id),
)
_rd_conn.commit()
_rd_conn.close()
logger.info(f"[Redownload] Updated DB path for track {lib_track_id}")
except Exception as e:
logger.error(f"[Redownload] Post-processing hook error: {e}")
_notify_download_completed(batch_id, task_id, success=True)
else:
track_name = get_import_clean_title(context, default=context_key)
logger.info(f"FAILED verification for '{track_name}' (task={task_id})")
logger.info(f" expected_final_path: {expected_final_path}")
logger.info(f" file_path (source): {file_path}, exists={os.path.exists(file_path)}")
logger.info(
f" is_album={context.get('is_album_download', False)}, "
f"has_clean_data={get_import_has_clean_metadata(context)}"
)
expected_dir = os.path.dirname(expected_final_path)
if os.path.exists(expected_dir):
dir_contents = os.listdir(expected_dir)
logger.info(f" directory contains {len(dir_contents)} files: {dir_contents[:20]}")
else:
logger.info(f" directory does not exist: {expected_dir}")
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = (
f'File verification failed: expected file at {os.path.basename(expected_final_path)} but it was not found after processing'
)
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=False)
except Exception as e:
import traceback
logger.info(f"EXCEPTION in post-processing for '{context_key}' (task={task_id}): {e}")
logger.info(traceback.format_exc())
with tasks_lock:
if task_id in download_tasks:
download_tasks[task_id]['status'] = 'failed'
download_tasks[task_id]['error_message'] = f"Post-processing verification failed: {str(e)}"
with matched_context_lock:
if context_key in matched_downloads_context:
del matched_downloads_context[context_key]
_notify_download_completed(batch_id, task_id, success=False)

View file

@ -0,0 +1,121 @@
"""Shared runtime state and tiny helpers for import/post-processing code."""
from __future__ import annotations
import threading
import time
from typing import Any, Dict, Optional
from core.import_context import (
build_import_album_info,
extract_artist_name,
get_import_clean_artist,
get_import_context_album,
get_import_original_search,
get_import_track_info,
normalize_import_context,
)
matched_context_lock = threading.Lock()
matched_downloads_context: Dict[str, Dict[str, Any]] = {}
tasks_lock = threading.Lock()
download_tasks: Dict[str, Dict[str, Any]] = {}
download_batches: Dict[str, Dict[str, Any]] = {}
_processed_download_ids = set()
_post_process_locks: Dict[str, threading.Lock] = {}
_post_process_locks_lock = threading.Lock()
activity_feed = []
activity_feed_lock = threading.Lock()
_activity_toast_emitter = None
def set_activity_toast_emitter(emitter) -> None:
"""Set the WebSocket-style emitter used by add_activity_item."""
global _activity_toast_emitter
_activity_toast_emitter = emitter
def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True):
"""Append an activity item and emit a toast if an emitter is configured."""
activity_item = {
"icon": icon,
"title": title,
"subtitle": subtitle,
"time": time_ago,
"timestamp": time.time(),
"show_toast": show_toast,
}
with activity_feed_lock:
activity_feed.append(activity_item)
if len(activity_feed) > 20:
activity_feed.pop(0)
if show_toast and _activity_toast_emitter is not None:
try:
_activity_toast_emitter("dashboard:toast", activity_item)
except Exception:
pass
return activity_item
def mark_task_completed(task_id: str, track_info: Optional[Dict[str, Any]] = None) -> bool:
"""Mark a download task as completed in the shared task registry."""
with tasks_lock:
task = download_tasks.get(task_id)
if not task:
return False
task["status"] = "completed"
task["stream_processed"] = True
task["status_change_time"] = time.time()
if track_info is not None:
task["track_info"] = track_info
return True
def detect_album_info_web(context, artist_context=None):
"""Best-effort album detection for single-track downloads."""
context = normalize_import_context(context)
if artist_context is None:
artist_context = context.get("artist") or {}
album_info = build_import_album_info(context)
if album_info.get("is_album"):
return album_info
album_ctx = get_import_context_album(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
album_name = (
album_ctx.get("name")
or track_info.get("album")
or original_search.get("album")
or ""
)
track_name = (
track_info.get("name")
or original_search.get("title")
or ""
)
artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="")
if album_name and track_name and album_name.strip().lower() not in {
track_name.strip().lower(),
artist_name.strip().lower(),
}:
return build_import_album_info(
context,
album_info={
"album_name": album_name,
"track_number": track_info.get("track_number", 1),
"disc_number": track_info.get("disc_number", 1),
"album_image_url": album_ctx.get("image_url", ""),
"confidence": 0.5,
},
force_album=True,
)
return None

567
core/import_side_effects.py Normal file
View file

@ -0,0 +1,567 @@
"""Import post-processing side effects that do not need web runtime state."""
from __future__ import annotations
import hashlib
import json
import os
from typing import Any, Dict, List, Optional
from core.import_context import (
extract_artist_name,
get_import_clean_album,
get_import_clean_artist,
get_import_clean_title,
get_import_context_album,
get_import_context_artist,
get_import_original_search,
get_import_source,
get_import_source_ids,
get_import_track_info,
normalize_import_context,
get_library_source_id_columns,
)
from core.wishlist_service import get_wishlist_service
from database.music_database import get_database
from utils.logging_config import get_logger
logger = get_logger("import_side_effects")
def _get_config_manager():
from config.settings import config_manager
return config_manager
def _primary_track_artist_name(track_info: Dict[str, Any]) -> str:
artists = (track_info or {}).get("artists", [])
if isinstance(artists, list) and artists:
first = artists[0]
if isinstance(first, dict):
return str(first.get("name", "") or "")
return str(first or "")
if isinstance(artists, str):
return artists
return str((track_info or {}).get("artist", "") or "")
def _all_profile_wishlist_tracks(wishlist_service) -> List[Dict[str, Any]]:
database = get_database()
all_profiles = database.get_all_profiles()
wishlist_tracks: List[Dict[str, Any]] = []
for profile in all_profiles:
wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=profile["id"]))
return wishlist_tracks
def _stable_soulsync_id(text: str) -> str:
return str(abs(int(hashlib.md5(text.encode("utf-8", errors="replace")).hexdigest(), 16)) % (10 ** 9))
def emit_track_downloaded(context: Dict[str, Any], automation_engine=None) -> None:
"""Emit the track_downloaded automation event."""
try:
if not automation_engine:
return
ti = context.get("track_info") or context.get("search_result") or {}
artist_name = ""
artists = ti.get("artists", [])
if artists:
first = artists[0]
artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first)
automation_engine.emit(
"track_downloaded",
{
"artist": artist_name,
"title": ti.get("name", ti.get("title", "")),
"album": ti.get("album", ""),
"quality": context.get("_audio_quality", "Unknown"),
},
)
except Exception:
pass
def record_library_history_download(context: Dict[str, Any]) -> None:
"""Record a completed download to the library_history table."""
try:
search_result = context.get("original_search_result") or context.get("search_result") or {}
username = search_result.get("username", context.get("_download_username", ""))
source_map = {
"youtube": "YouTube",
"tidal": "Tidal",
"qobuz": "Qobuz",
"hifi": "HiFi",
"deezer_dl": "Deezer",
"lidarr": "Lidarr",
}
download_source = source_map.get(username, "Soulseek")
ti = context.get("track_info") or context.get("search_result") or {}
artist_name = _primary_track_artist_name(ti)
if not artist_name:
artist_name = ti.get("artist", "")
album_raw = ti.get("album", "")
album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "")
title = ti.get("name", ti.get("title", ""))
quality = context.get("_audio_quality", "")
file_path = context.get("_final_processed_path", context.get("_final_path", ""))
thumb_url = ""
album_context = get_import_context_album(context)
if album_context:
thumb_url = album_context.get("image_url", "")
if not thumb_url:
images = album_context.get("images", [])
if images:
thumb_url = images[0].get("url", "")
if not thumb_url:
album_info = context.get("album_info", {})
if isinstance(album_info, dict):
thumb_url = album_info.get("album_image_url", "")
source_filename = search_result.get("filename", "")
source_track_id = search_result.get("track_id", "") or search_result.get("id", "") or ti.get("id", "")
source_track_title = search_result.get("title", "") or search_result.get("name", "")
source_artist = search_result.get("artist", "")
if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr"):
stream_id = source_filename.split("||")[0]
if stream_id and not source_track_id:
source_track_id = stream_id
acoustid_result = context.get("_acoustid_result", "")
db = get_database()
db.add_library_history_entry(
event_type="download",
title=title,
artist_name=artist_name,
album_name=album_name,
quality=quality,
file_path=file_path,
thumb_url=thumb_url,
download_source=download_source,
source_track_id=source_track_id,
source_track_title=source_track_title,
source_filename=source_filename,
acoustid_result=acoustid_result,
source_artist=source_artist,
)
except Exception:
pass
def record_download_provenance(context: Dict[str, Any]) -> None:
"""Record source provenance for a completed download."""
try:
search_result = context.get("original_search_result") or context.get("search_result") or {}
username = search_result.get("username", context.get("_download_username", ""))
filename = search_result.get("filename", "")
source_service = {
"youtube": "youtube",
"tidal": "tidal",
"qobuz": "qobuz",
"hifi": "hifi",
"deezer_dl": "deezer",
"lidarr": "lidarr",
}.get(username, "soulseek")
ti = context.get("track_info") or context.get("search_result") or {}
artist_name = _primary_track_artist_name(ti)
if not artist_name:
artist_name = ti.get("artist", "")
album_raw = ti.get("album", "")
album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "")
title = ti.get("name", ti.get("title", ""))
file_path = context.get("_final_processed_path", context.get("_final_path", ""))
quality = context.get("_audio_quality", "")
size = search_result.get("size", 0)
bit_depth = None
sample_rate = None
bitrate = None
try:
if file_path and os.path.isfile(file_path):
from mutagen import File as MutagenFile
audio = MutagenFile(file_path)
if audio and audio.info:
sample_rate = getattr(audio.info, "sample_rate", None)
bitrate = getattr(audio.info, "bitrate", None)
bit_depth = getattr(audio.info, "bits_per_sample", None)
except Exception:
pass
db = get_database()
db.record_track_download(
file_path=file_path,
source_service=source_service,
source_username=username,
source_filename=filename,
source_size=size or 0,
audio_quality=quality,
track_title=title,
track_artist=artist_name,
track_album=album_name,
bit_depth=bit_depth,
sample_rate=sample_rate,
bitrate=bitrate,
)
except Exception:
pass
def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any]) -> None:
"""Write imported media to the SoulSync library tables when the active server is SoulSync."""
try:
config_manager = _get_config_manager()
if config_manager.get_active_media_server() != "soulsync":
return
context = normalize_import_context(context)
final_path = context.get("_final_processed_path")
if not final_path:
return
album_ctx = get_import_context_album(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
source = get_import_source(context)
source_ids = get_import_source_ids(context)
source_columns = get_library_source_id_columns(source)
artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="")
if not artist_name or artist_name in ("Unknown", "Unknown Artist"):
return
album_name = ""
if album_info and isinstance(album_info, dict):
album_name = album_info.get("album_name", "")
if not album_name:
album_name = album_ctx.get("name", "") or original_search.get("album", "")
if not album_name:
album_name = track_info.get("name", "Unknown")
track_name = get_import_clean_title(
context,
album_info=album_info,
default=track_info.get("name", "") or original_search.get("title", ""),
)
track_number = (track_info.get("track_number") or (album_info.get("track_number") if isinstance(album_info, dict) else None)) or 1
duration_ms = track_info.get("duration_ms", 0) or 0
year = None
release_date = album_ctx.get("release_date", "")
if release_date and len(release_date) >= 4:
try:
year = int(release_date[:4])
except ValueError:
pass
image_url = album_ctx.get("image_url", "")
if not image_url:
images = album_ctx.get("images", [])
if images and isinstance(images, list) and len(images) > 0:
img = images[0]
image_url = img.get("url", "") if isinstance(img, dict) else str(img)
artist_source_id = source_ids.get("artist_id", "")
album_source_id = source_ids.get("album_id", "")
track_source_id = source_ids.get("track_id", "")
for key in ("auto_import", "from_sync_modal", "explicit_artist", "explicit_album", ""):
if artist_source_id == key:
artist_source_id = ""
if album_source_id == key:
album_source_id = ""
if track_source_id == key:
track_source_id = ""
genres = (artist_context or {}).get("genres", []) if isinstance(artist_context, dict) else []
if genres:
from core.genre_filter import filter_genres as _filter_genres
genres = _filter_genres(genres, config_manager)
genres_json = json.dumps(genres) if genres else ""
bitrate = 0
try:
from mutagen import File as MutagenFile
audio = MutagenFile(final_path)
if audio and hasattr(audio, "info") and audio.info and hasattr(audio.info, "bitrate"):
bitrate = int(audio.info.bitrate / 1000) if audio.info.bitrate else 0
except Exception:
pass
artist_id = _stable_soulsync_id(artist_name.lower().strip())
album_id = _stable_soulsync_id(f"{artist_name}::{album_name}".lower().strip())
track_id = _stable_soulsync_id(final_path)
total_tracks = album_ctx.get("total_tracks", 0) or 0
db = get_database()
with db._get_connection() as conn:
cursor = conn.cursor()
cursor.execute("SELECT id FROM artists WHERE id = ? AND server_source = 'soulsync'", (artist_id,))
if not cursor.fetchone():
cursor.execute(
"SELECT id FROM artists WHERE name COLLATE NOCASE = ? AND server_source = 'soulsync' LIMIT 1",
(artist_name,),
)
existing_by_name = cursor.fetchone()
if existing_by_name:
artist_id = existing_by_name[0]
else:
cursor.execute("SELECT id FROM artists WHERE id = ?", (artist_id,))
if cursor.fetchone():
artist_id = _stable_soulsync_id(artist_name.lower().strip() + "::soulsync")
cursor.execute(
"""
INSERT INTO artists (id, name, genres, thumb_url, server_source, created_at, updated_at)
VALUES (?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
""",
(artist_id, artist_name, genres_json, image_url),
)
artist_source_col = source_columns.get("artist")
if artist_source_col and artist_source_id:
try:
cursor.execute(
f"UPDATE artists SET {artist_source_col} = ? WHERE id = ?",
(artist_source_id, artist_id),
)
except Exception:
pass
cursor.execute("SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'", (album_id,))
if not cursor.fetchone():
cursor.execute(
"SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? AND server_source = 'soulsync' LIMIT 1",
(album_name, artist_id),
)
existing_album_by_name = cursor.fetchone()
if existing_album_by_name:
album_id = existing_album_by_name[0]
else:
cursor.execute("SELECT id FROM albums WHERE id = ?", (album_id,))
if cursor.fetchone():
album_id = _stable_soulsync_id(f"{artist_name}::{album_name}::soulsync".lower().strip())
cursor.execute(
"""
INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, track_count,
duration, server_source, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
""",
(album_id, artist_id, album_name, year, image_url, genres_json, total_tracks, duration_ms),
)
album_source_col = source_columns.get("album")
if album_source_col and album_source_id:
try:
cursor.execute(
f"UPDATE albums SET {album_source_col} = ? WHERE id = ?",
(album_source_id, album_id),
)
except Exception:
pass
track_artist = None
track_artists_list = track_info.get("artists", []) or original_search.get("artists", [])
if track_artists_list:
first_track_artist = track_artists_list[0]
if isinstance(first_track_artist, dict):
ta_name = first_track_artist.get("name", "")
else:
ta_name = str(first_track_artist)
if ta_name and ta_name.lower() != artist_name.lower():
track_artist = ta_name
cursor.execute("SELECT id FROM tracks WHERE file_path = ?", (final_path,))
if not cursor.fetchone():
cursor.execute(
"""
INSERT INTO tracks (id, album_id, artist_id, title, track_number,
duration, file_path, bitrate, track_artist, server_source,
created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
""",
(
track_id,
album_id,
artist_id,
track_name,
track_number,
duration_ms,
final_path,
bitrate,
track_artist,
),
)
track_source_col = source_columns.get("track")
if track_source_col and track_source_id:
try:
cursor.execute(
f"UPDATE tracks SET {track_source_col} = ? WHERE id = ?",
(track_source_id, track_id),
)
track_album_col = source_columns.get("track_album")
if track_album_col and album_source_id:
cursor.execute(
f"UPDATE tracks SET {track_album_col} = ? WHERE id = ?",
(album_source_id, track_id),
)
except Exception:
pass
conn.commit()
logger.info("[SoulSync Library] Added: %s / %s / %s", artist_name, album_name, track_name)
except Exception as exc:
logger.error("[SoulSync Library] Could not record library entry: %s", exc)
def record_retag_download(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any], final_path: str) -> None:
"""Record a completed download for later re-tagging."""
try:
db = get_database()
context = normalize_import_context(context)
artist_context = get_import_context_artist(context) or (artist_context if isinstance(artist_context, dict) else {})
album_context = get_import_context_album(context)
track_info = get_import_track_info(context)
original_search = get_import_original_search(context)
source = get_import_source(context)
source_ids = get_import_source_ids(context)
artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="Unknown Artist")
is_album = album_info and album_info.get("is_album", False)
group_type = "album" if is_album else "single"
album_name = album_info.get("album_name", "") if album_info else get_import_clean_album(context, default=original_search.get("album", "Unknown"))
image_url = album_info.get("album_image_url") if album_info else None
if not image_url:
image_url = album_context.get("image_url", "")
if not image_url and album_context.get("images"):
images = album_context.get("images", [])
if images and isinstance(images[0], dict):
image_url = images[0].get("url", "")
total_tracks = album_context.get("total_tracks", 1) if album_context else 1
release_date = album_context.get("release_date", "") if album_context else ""
spotify_album_id = None
itunes_album_id = None
if source == "spotify":
spotify_album_id = source_ids.get("album_id", "") or None
elif source == "itunes":
itunes_album_id = source_ids.get("album_id", "") or None
group_id = db.find_retag_group(artist_name, album_name)
if group_id is None:
group_id = db.add_retag_group(
group_type=group_type,
artist_name=artist_name,
album_name=album_name,
image_url=image_url,
spotify_album_id=spotify_album_id,
itunes_album_id=itunes_album_id,
total_tracks=total_tracks,
release_date=release_date,
)
if group_id is None:
return
track_number = album_info.get("track_number", 1) if album_info else (track_info.get("track_number", 1) or 1)
disc_number = original_search.get("disc_number") or (album_info.get("disc_number", 1) if album_info else track_info.get("disc_number", 1) or 1)
title = get_import_clean_title(
context,
album_info=album_info,
default=album_info.get("clean_track_name", "Unknown Track") if album_info else "Unknown Track",
)
file_format = os.path.splitext(str(final_path))[1].lstrip(".").lower()
spotify_track_id = None
itunes_track_id = None
if source == "spotify":
spotify_track_id = source_ids.get("track_id", "") or None
elif source == "itunes":
itunes_track_id = source_ids.get("track_id", "") or None
if not db.retag_track_exists(group_id, str(final_path)):
db.add_retag_track(
group_id=group_id,
track_number=track_number,
disc_number=disc_number,
title=title,
file_path=str(final_path),
file_format=file_format,
spotify_track_id=spotify_track_id,
itunes_track_id=itunes_track_id,
)
logger.info("[Retag] Recorded track for retag: '%s' in '%s'", title, album_name)
db.trim_retag_groups(100)
except Exception as exc:
logger.error("[Retag] Could not record track for retag: %s", exc)
def check_and_remove_from_wishlist(context: Dict[str, Any]) -> None:
"""Check whether a successful download should be removed from the wishlist."""
try:
wishlist_service = get_wishlist_service()
spotify_track_id = None
track_info = context.get("track_info", {})
if track_info.get("id"):
spotify_track_id = track_info["id"]
logger.info("[Wishlist] Found Spotify ID from track_info: %s", spotify_track_id)
elif context.get("original_search_result", {}).get("id"):
spotify_track_id = context["original_search_result"]["id"]
logger.info("[Wishlist] Found Spotify ID from original_search_result: %s", spotify_track_id)
elif "wishlist_id" in track_info:
wishlist_id = track_info["wishlist_id"]
logger.info("[Wishlist] Found wishlist_id in context: %s", wishlist_id)
wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service)
for wishlist_track in wishlist_tracks:
if wishlist_track.get("wishlist_id") == wishlist_id:
spotify_track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id")
logger.info("[Wishlist] Found Spotify ID from wishlist entry: %s", spotify_track_id)
break
if not spotify_track_id:
track_name = track_info.get("name") or context.get("original_search_result", {}).get("title", "")
artist_name = _primary_track_artist_name(track_info) or _primary_track_artist_name(context.get("original_search_result", {}))
if track_name and artist_name:
logger.warning("[Wishlist] No Spotify ID found, checking for fuzzy match: '%s' by '%s'", track_name, artist_name)
wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service)
for wishlist_track in wishlist_tracks:
wl_name = wishlist_track.get("name", "").lower()
wl_artists = wishlist_track.get("artists", [])
wl_artist_name = ""
if wl_artists:
if isinstance(wl_artists[0], dict):
wl_artist_name = wl_artists[0].get("name", "").lower()
else:
wl_artist_name = str(wl_artists[0]).lower()
if wl_name == track_name.lower() and wl_artist_name == artist_name.lower():
spotify_track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id")
logger.info("[Wishlist] Found fuzzy match - Spotify ID: %s", spotify_track_id)
break
if spotify_track_id:
logger.info("[Wishlist] Attempting to remove track from wishlist: %s", spotify_track_id)
removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True)
if removed:
logger.info("[Wishlist] Successfully removed track from wishlist: %s", spotify_track_id)
else:
logger.warning(" [Wishlist] Track not found in wishlist or already removed: %s", spotify_track_id)
else:
logger.warning(" [Wishlist] No Spotify track ID found for wishlist removal check")
except Exception as exc:
logger.error("[Wishlist] Error in wishlist removal check: %s", exc)

454
core/import_staging.py Normal file
View file

@ -0,0 +1,454 @@
"""Shared staging folder and import suggestion helpers."""
from __future__ import annotations
import os
import threading
from typing import Any, Dict, List, Optional, Tuple
from core.import_paths import docker_resolve_path
from utils.logging_config import get_logger
logger = get_logger("import_staging")
AUDIO_EXTENSIONS = {".mp3", ".flac", ".ogg", ".opus", ".m4a", ".aac", ".wav", ".wma", ".aiff", ".aif", ".ape"}
_import_suggestions_cache_lock = threading.Lock()
_import_suggestions_cache: Dict[str, Any] = {
"suggestions": [],
"building": False,
"built": False,
}
def _get_config_manager():
try:
from config.settings import config_manager
return config_manager
except Exception:
class _FallbackConfig:
@staticmethod
def get(key, default=None):
return default
return _FallbackConfig()
def get_staging_path() -> str:
"""Resolve the configured staging folder path."""
raw = _get_config_manager().get("import.staging_path", "./Staging")
return docker_resolve_path(raw)
def get_import_suggestions_cache() -> Dict[str, Any]:
"""Expose the shared import suggestions cache."""
return _import_suggestions_cache
def get_primary_source() -> str:
from core.metadata_service import get_primary_source as _get_primary_source
return _get_primary_source()
def get_source_priority(preferred_source: str):
from core.metadata_service import get_source_priority as _get_source_priority
return _get_source_priority(preferred_source)
def get_client_for_source(source: str):
from core.metadata_service import get_client_for_source as _get_client_for_source
return _get_client_for_source(source)
def _search_albums_for_source(source: str, client: Any, query: str, limit: int = 5):
from core.metadata_service import _search_albums_for_source as _metadata_search_albums_for_source
return _metadata_search_albums_for_source(source, client, query, limit=limit)
def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 5):
from core.metadata_service import _search_tracks_for_source as _metadata_search_tracks_for_source
return _metadata_search_tracks_for_source(source, client, query, limit=limit)
def _extract_value(value: Any, *names: str, default: Any = None) -> Any:
if value is None:
return default
if isinstance(value, (str, bytes)):
return default
for name in names:
if isinstance(value, dict):
if name in value and value[name] is not None:
return value[name]
else:
candidate = getattr(value, name, None)
if candidate is not None:
return candidate
return default
def _extract_artist_names(artists: Any) -> List[str]:
if not artists:
return []
if isinstance(artists, (str, bytes)):
artist = str(artists).strip()
return [artist] if artist else []
try:
items = list(artists)
except TypeError:
items = [artists]
names: List[str] = []
for artist in items:
if isinstance(artist, dict):
name = str(_extract_value(artist, "name", "artist_name", "title", default="") or "").strip()
else:
candidate = getattr(artist, "name", None)
if candidate is None:
candidate = artist
name = str(candidate or "").strip()
if name:
names.append(name)
return names
def _normalize_album_result(album: Any, source: str) -> Dict[str, Any]:
album_id = str(_extract_value(album, "id", "album_id", "release_id", default="") or "").strip()
album_name = str(_extract_value(album, "name", "title", default="") or "").strip()
artists = _extract_artist_names(_extract_value(album, "artists", default=[]))
artist_name = ", ".join(artists) if artists else str(
_extract_value(album, "artist_name", "artist", default="Unknown Artist") or "Unknown Artist"
).strip()
release_date = str(_extract_value(album, "release_date", "releaseDate", default="") or "").strip()
album_type = str(_extract_value(album, "album_type", "type", default="album") or "album").strip() or "album"
total_tracks = _extract_value(album, "total_tracks", "track_count", default=0)
if isinstance(total_tracks, (list, tuple, set)):
total_tracks = len(total_tracks)
try:
total_tracks = int(total_tracks or 0)
except (TypeError, ValueError):
total_tracks = 0
image_url = _extract_value(album, "image_url", "thumb_url", "cover_image", "cover_url", default="")
if not image_url:
images = _extract_value(album, "images", default=[]) or []
if isinstance(images, dict):
images = [images]
elif isinstance(images, (str, bytes)):
images = [images]
try:
images = list(images)
except TypeError:
images = [images]
if images:
first_image = images[0]
if isinstance(first_image, (str, bytes)):
image_url = str(first_image).strip()
else:
image_url = _extract_value(first_image, "url", "image_url", "src", default="")
return {
"id": album_id or album_name or "unknown-album",
"name": album_name or album_id or "Unknown Album",
"artist": artist_name or "Unknown Artist",
"release_date": release_date,
"total_tracks": total_tracks,
"image_url": str(image_url or ""),
"album_type": album_type,
"source": source,
}
def _album_fingerprint(album: Dict[str, Any]) -> Tuple[str, str, str, str]:
return (
str(album.get("name", "") or "").strip().casefold(),
str(album.get("artist", "") or "").strip().casefold(),
str(album.get("release_date", "") or "").strip()[:10].casefold(),
str(album.get("album_type", "") or "").strip().casefold(),
)
def _normalize_track_result(track: Any, source: str) -> Dict[str, Any]:
track_id = str(_extract_value(track, "id", "track_id", "trackId", default="") or "").strip()
track_name = str(_extract_value(track, "name", "title", "track_name", default="") or "").strip()
artists = _extract_artist_names(_extract_value(track, "artists", default=[]))
artist_name = ", ".join(artists) if artists else str(
_extract_value(track, "artist", "artist_name", default="Unknown Artist") or "Unknown Artist"
).strip()
album_value = _extract_value(track, "album", default=None)
album_name = ""
album_id = str(_extract_value(track, "album_id", "collectionId", "albumId", default="") or "").strip()
if isinstance(album_value, dict):
album_name = str(_extract_value(album_value, "name", "title", default="") or "").strip()
album_id = album_id or str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip()
if not album_name:
album_name = album_id
elif isinstance(album_value, (str, bytes)):
album_name = str(album_value).strip()
elif album_value is not None:
album_name = str(_extract_value(album_value, "name", "title", default=album_value) or "").strip()
if not album_id:
album_id = str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip()
image_url = _extract_value(track, "image_url", "thumb_url", "cover_image", default="")
if not image_url:
images = _extract_value(track, "images", default=[]) or []
if isinstance(images, dict):
images = [images]
elif isinstance(images, (str, bytes)):
images = [images]
try:
images = list(images)
except TypeError:
images = [images]
if images:
first_image = images[0]
if isinstance(first_image, (str, bytes)):
image_url = str(first_image).strip()
else:
image_url = _extract_value(first_image, "url", "image_url", "src", default="")
if not image_url and album_value is not None:
album_images = _extract_value(album_value, "images", default=[]) or []
if isinstance(album_images, dict):
album_images = [album_images]
elif isinstance(album_images, (str, bytes)):
album_images = [album_images]
try:
album_images = list(album_images)
except TypeError:
album_images = [album_images]
if album_images:
first_album_image = album_images[0]
if isinstance(first_album_image, (str, bytes)):
image_url = str(first_album_image).strip()
else:
image_url = _extract_value(first_album_image, "url", "image_url", "src", default="")
duration_ms = _extract_value(track, "duration_ms", "duration", "trackTimeMillis", default=0)
try:
duration_ms = int(duration_ms or 0)
except (TypeError, ValueError):
duration_ms = 0
track_number = _extract_value(track, "track_number", "trackNumber", default=1)
try:
track_number = int(track_number or 1)
except (TypeError, ValueError):
track_number = 1
return {
"id": track_id or track_name or "unknown-track",
"name": track_name or track_id or "Unknown Track",
"artist": artist_name or "Unknown Artist",
"album": album_name or "",
"album_id": album_id or "",
"duration_ms": duration_ms,
"image_url": str(image_url or ""),
"track_number": track_number,
"source": source,
}
def _read_staging_audio_tags(file_path: str) -> Tuple[Optional[str], Optional[str]]:
try:
from mutagen import File as MutagenFile
tags = MutagenFile(file_path, easy=True)
if not tags:
return None, None
album = (tags.get("album") or [None])[0]
artist = (tags.get("artist") or (tags.get("albumartist") or [None]))[0]
album_text = str(album).strip() if album else ""
artist_text = str(artist).strip() if artist else ""
return (album_text or None, artist_text or None)
except Exception:
return None, None
def _collect_import_suggestion_queries(staging_path: str) -> List[str]:
tag_albums: Dict[Tuple[str, str], int] = {}
folder_hints: Dict[str, int] = {}
for root, _dirs, filenames in os.walk(staging_path):
audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS]
if not audio_files:
continue
rel_dir = os.path.relpath(root, staging_path)
if rel_dir != ".":
top_folder = rel_dir.split(os.sep)[0]
folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files)
for fname in audio_files:
full_path = os.path.join(root, fname)
album, artist = _read_staging_audio_tags(full_path)
if album:
key = (album.strip(), (artist or "").strip())
tag_albums[key] = tag_albums.get(key, 0) + 1
queries: List[str] = []
seen_lower = set()
for (album, artist), _count in sorted(tag_albums.items(), key=lambda item: -item[1]):
q = f"{album} {artist}".strip() if artist else album
if q and q.lower() not in seen_lower:
seen_lower.add(q.lower())
queries.append(q)
for folder, _count in sorted(folder_hints.items(), key=lambda item: -item[1]):
q = folder.replace("_", " ")
if q and q.lower() not in seen_lower:
seen_lower.add(q.lower())
queries.append(q)
return queries[:5]
def search_import_albums(query: str, limit: int = 12) -> List[Dict[str, Any]]:
"""Search albums using the configured metadata provider first."""
query = (query or "").strip()
if not query:
return []
results: List[Dict[str, Any]] = []
seen = set()
source_chain = get_source_priority(get_primary_source())
for source in source_chain:
client = get_client_for_source(source)
if not client:
continue
source_results = _search_albums_for_source(source, client, query, limit=limit)
if not source_results:
continue
added_for_source = False
for album in source_results:
suggestion = _normalize_album_result(album, source)
fingerprint = _album_fingerprint(suggestion)
if fingerprint in seen:
continue
seen.add(fingerprint)
results.append(suggestion)
added_for_source = True
if len(results) >= limit:
return results[:limit]
if added_for_source:
break
return results[:limit]
def search_import_tracks(query: str, limit: int = 30) -> List[Dict[str, Any]]:
"""Search tracks using the configured metadata provider priority order."""
query = (query or "").strip()
if not query:
return []
results: List[Dict[str, Any]] = []
source_chain = get_source_priority(get_primary_source())
for source in source_chain:
client = get_client_for_source(source)
if not client:
continue
source_results = _search_tracks_for_source(source, client, query, limit=limit)
if not source_results:
continue
for track in source_results:
results.append(_normalize_track_result(track, source))
if len(results) >= limit:
return results[:limit]
break
return results[:limit]
def _build_import_suggestions_background():
cache = _import_suggestions_cache
with _import_suggestions_cache_lock:
if cache["building"]:
return
cache["building"] = True
try:
staging_path = get_staging_path()
if not os.path.isdir(staging_path):
with _import_suggestions_cache_lock:
cache["suggestions"] = []
cache["built"] = True
return
queries = _collect_import_suggestion_queries(staging_path)
if not queries:
with _import_suggestions_cache_lock:
cache["suggestions"] = []
cache["built"] = True
return
suggestions: List[Dict[str, Any]] = []
seen = set()
for query in queries:
try:
albums = search_import_albums(query, limit=2)
for album in albums:
fingerprint = _album_fingerprint(album)
if fingerprint in seen:
continue
seen.add(fingerprint)
suggestions.append(album)
except Exception as exc:
logger.warning("Import suggestion search failed for %r: %s", query, exc)
with _import_suggestions_cache_lock:
cache["suggestions"] = suggestions[:8]
cache["built"] = True
logger.info(
"Import suggestions cache built: %s suggestions from %s hints",
len(cache["suggestions"]),
len(queries),
)
except Exception as exc:
logger.error("Error building import suggestions cache: %s", exc)
with _import_suggestions_cache_lock:
cache["suggestions"] = []
cache["built"] = True
finally:
with _import_suggestions_cache_lock:
cache["building"] = False
def start_import_suggestions_cache():
"""Start building the import suggestions cache in a background thread."""
threading.Thread(
target=_build_import_suggestions_background,
daemon=True,
name="import-suggestions-cache",
).start()
def refresh_import_suggestions_cache():
"""Invalidate and rebuild the suggestions cache."""
with _import_suggestions_cache_lock:
_import_suggestions_cache["built"] = False
start_import_suggestions_cache()

1486
core/metadata_enrichment.py Normal file

File diff suppressed because it is too large Load diff

View file

@ -719,6 +719,23 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti
if not isinstance(images, list):
images = list(images) if images else []
artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[]))
if not artists and artist_name:
artists = [{'name': artist_name}]
primary_artist = artists[0] if artists else {}
resolved_artist_name = (
_extract_lookup_value(primary_artist, 'name', default='')
or artist_name
or _extract_lookup_value(album_data, 'artist_name', 'artist', default='')
or ''
)
resolved_artist_id = str(
_extract_lookup_value(primary_artist, 'id', default='')
or _extract_lookup_value(album_data, 'artist_id', default='')
or ''
).strip()
image_url = None
if images:
image_url = _extract_lookup_value(images[0], 'url')
@ -728,12 +745,15 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti
return {
'id': _extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', 'release_id', default=album_id) or album_id,
'name': _extract_lookup_value(album_data, 'name', 'title', default=album_name or album_id) or album_name or album_id,
'artist': resolved_artist_name or '',
'artist_name': resolved_artist_name or '',
'artist_id': resolved_artist_id,
'artists': artists,
'image_url': image_url,
'images': images,
'release_date': _extract_lookup_value(album_data, 'release_date', default='') or '',
'album_type': _extract_lookup_value(album_data, 'album_type', default='album') or 'album',
'total_tracks': _extract_lookup_value(album_data, 'total_tracks', 'track_count', default=0) or 0,
'artist_name': artist_name or _extract_lookup_value(album_data, 'artist_name', default='') or '',
}
@ -754,6 +774,8 @@ def _build_album_track_entry(track_item: Any, album_info: Dict[str, Any], source
'external_urls': _extract_lookup_value(track_item, 'external_urls', default={}) or {},
'uri': _extract_lookup_value(track_item, 'uri', default='') or '',
'album': album_info,
'source': source,
'provider': source,
'_source': source,
}
@ -767,6 +789,9 @@ def _build_album_tracks_payload(
artist_name: str = '',
) -> Dict[str, Any]:
album_info = _build_album_info(album_data, album_id, album_name=album_name, artist_name=artist_name)
album_info['source'] = source
album_info['_source'] = source
album_info['provider'] = source
track_items = _extract_album_track_items(album_data, tracks_data)
tracks = [_build_album_track_entry(track, album_info, source) for track in track_items]
@ -778,6 +803,369 @@ def _build_album_tracks_payload(
}
def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]:
if not artists:
return []
if isinstance(artists, (str, bytes)):
artists = [artists]
elif isinstance(artists, dict):
artists = [artists]
else:
try:
artists = list(artists)
except TypeError:
artists = [artists]
normalized: List[Dict[str, Any]] = []
for artist in artists:
if isinstance(artist, dict):
name = _extract_lookup_value(artist, 'name', 'artist_name', 'title', default='') or ''
artist_id = _extract_lookup_value(artist, 'id', 'artist_id', default='') or ''
entry: Dict[str, Any] = {}
if name:
entry['name'] = str(name)
if artist_id:
entry['id'] = str(artist_id)
genres = _extract_lookup_value(artist, 'genres', default=None)
if genres is not None:
entry['genres'] = genres
if entry:
normalized.append(entry)
continue
name = str(artist).strip()
if name:
normalized.append({'name': name})
return normalized
def _build_single_import_context_payload(
track_data: Any,
source: Optional[str],
source_priority: List[str],
requested_title: str = '',
requested_artist: str = '',
) -> Dict[str, Any]:
album_data = _extract_lookup_value(track_data, 'album', default=None)
track_id = str(_extract_lookup_value(track_data, 'id', 'track_id', 'trackId', default='') or '')
track_name = _extract_lookup_value(track_data, 'name', 'title', 'trackName', default='') or requested_title or 'Unknown Track'
track_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'artists', default=[]))
if not track_artists and requested_artist:
track_artists = [{'name': requested_artist}]
primary_track_artist = track_artists[0] if track_artists else {}
primary_artist_name = primary_track_artist.get('name') or requested_artist or 'Unknown Artist'
primary_artist_id = str(primary_track_artist.get('id', '') or _extract_lookup_value(track_data, 'artist_id', 'artistId', default='') or '')
album_name = _extract_lookup_value(track_data, 'album_name', 'collectionName', default='') or ''
album_id = str(_extract_lookup_value(track_data, 'album_id', 'collectionId', 'albumId', default='') or '')
release_date = str(_extract_lookup_value(track_data, 'release_date', default='') or '')
album_type = str(_extract_lookup_value(track_data, 'album_type', default='album') or 'album')
total_tracks = int(_extract_lookup_value(track_data, 'total_tracks', 'track_count', default=0) or 0)
album_images: List[Dict[str, Any]] = []
album_image_url = str(_extract_lookup_value(track_data, 'image_url', 'thumb_url', default='') or '')
album_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'album_artists', 'artists', default=[]))
if isinstance(album_data, dict):
album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name
album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id)
release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date)
album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type)
total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks)
album_images = _extract_lookup_value(album_data, 'images', default=[]) or []
if not album_image_url:
album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '')
if not album_image_url and album_images:
album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '')
album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[]))
elif album_data:
album_name = album_name or str(album_data)
if not album_artists and primary_artist_name:
album_artists = [{'name': primary_artist_name}]
if not album_image_url and album_images:
album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '')
track_info = {
'id': track_id,
'name': track_name,
'track_number': int(_extract_lookup_value(track_data, 'track_number', 'trackNumber', default=1) or 1),
'disc_number': int(_extract_lookup_value(track_data, 'disc_number', 'discNumber', default=1) or 1),
'duration_ms': int(_extract_lookup_value(track_data, 'duration_ms', 'duration', 'trackTimeMillis', default=0) or 0),
'artists': track_artists or [{'name': primary_artist_name}],
'uri': str(_extract_lookup_value(track_data, 'uri', default='') or ''),
'album': album_name,
'album_id': album_id,
'album_type': album_type,
'release_date': release_date,
'_source': source or '',
}
album_payload = {
'id': album_id,
'name': album_name,
'release_date': release_date,
'total_tracks': total_tracks or 1,
'album_type': album_type,
'image_url': album_image_url,
'images': album_images,
'artists': album_artists,
'_source': source or '',
}
artist_payload = {
'id': primary_artist_id,
'name': primary_artist_name,
'genres': [],
'_source': source or '',
}
original_search = {
'title': track_name,
'artist': primary_artist_name,
'album': album_name,
'track_number': track_info['track_number'],
'disc_number': track_info['disc_number'],
'clean_title': track_name,
'clean_album': album_name,
'clean_artist': primary_artist_name,
'artists': track_info['artists'],
'duration_ms': track_info['duration_ms'],
'id': track_id,
'_source': source or '',
}
return {
'success': bool(track_id or track_name != requested_title or album_name),
'source': source,
'source_priority': source_priority,
'context': {
'artist': artist_payload,
'album': album_payload,
'track_info': track_info,
'original_search_result': original_search,
'is_album_download': False,
'has_clean_metadata': bool(track_id),
'has_full_metadata': bool(track_id),
'source': source,
'source_priority': source_priority,
},
}
def _build_single_import_fallback_context(
requested_title: str,
requested_artist: str,
source_priority: List[str],
) -> Dict[str, Any]:
artist_name = requested_artist or 'Unknown Artist'
title = requested_title or 'Unknown Track'
return {
'success': False,
'source': None,
'source_priority': source_priority,
'context': {
'artist': {
'id': '',
'name': artist_name,
'genres': [],
'_source': '',
},
'album': {
'id': '',
'name': '',
'release_date': '',
'total_tracks': 1,
'album_type': 'album',
'image_url': '',
'images': [],
'artists': [],
'_source': '',
},
'track_info': {
'id': '',
'name': title,
'track_number': 1,
'disc_number': 1,
'duration_ms': 0,
'artists': [{'name': artist_name}],
'uri': '',
'album': '',
'album_id': '',
'album_type': 'album',
'release_date': '',
'_source': '',
},
'original_search_result': {
'title': title,
'artist': artist_name,
'album': '',
'track_number': 1,
'disc_number': 1,
'clean_title': title,
'clean_album': '',
'clean_artist': artist_name,
'artists': [{'name': artist_name}],
'duration_ms': 0,
'id': '',
'_source': '',
},
'is_album_download': False,
'has_clean_metadata': False,
'has_full_metadata': False,
'source': None,
'source_priority': source_priority,
},
}
def _build_track_search_query(source: str, title: str, artist: str) -> str:
base_query = " ".join(part for part in (title, artist) if part).strip()
if source == 'deezer' and title:
if artist:
return f'artist:"{artist}" track:"{title}"'
return f'track:"{title}"'
return base_query or title or artist
def _pick_best_track_match(search_results: List[Any], title: str, artist: str = '') -> Optional[Any]:
if not search_results:
return None
target_title = str(title or '').strip().lower()
target_artist = str(artist or '').strip().lower()
for candidate in search_results:
candidate_title = str(_extract_lookup_value(candidate, 'name', 'title', 'track_name', default='') or '').strip().lower()
if candidate_title != target_title:
continue
if not target_artist:
return candidate
candidate_artists = _normalize_context_artists(_extract_lookup_value(candidate, 'artists', default=[]))
candidate_artist_name = candidate_artists[0]['name'].strip().lower() if candidate_artists else ''
if candidate_artist_name == target_artist:
return candidate
return search_results[0]
def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]:
if not client or not hasattr(client, 'search_tracks'):
return []
try:
kwargs = {'limit': limit}
if source == 'spotify':
kwargs['allow_fallback'] = False
return client.search_tracks(query, **kwargs) or []
except Exception as exc:
logger.debug("Could not search %s for %s: %s", source, query, exc)
return []
def get_single_track_import_context(
title: str,
artist: str = '',
override_id: Optional[str] = None,
override_source: str = 'spotify',
source_override: Optional[str] = None,
) -> Dict[str, Any]:
"""Build an import context for singles using source-priority metadata lookup."""
options = MetadataLookupOptions(source_override=source_override, allow_fallback=True)
source_priority = _get_source_chain_for_lookup(options)
title = (title or '').strip()
artist = (artist or '').strip()
if override_id:
chosen_source = (override_source or 'spotify').strip().lower() or 'spotify'
client = get_client_for_source(chosen_source)
if client and hasattr(client, 'get_track_details'):
try:
track_data = client.get_track_details(str(override_id))
if track_data:
payload = _build_single_import_context_payload(
track_data,
chosen_source,
source_priority,
requested_title=title,
requested_artist=artist,
)
if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'):
try:
artist_details = client.get_artist(payload['context']['artist']['id'])
if artist_details:
payload['context']['artist']['genres'] = _extract_lookup_value(
artist_details,
'genres',
default=[],
) or []
except Exception:
pass
return payload
except Exception as exc:
logger.debug("Override track lookup failed on %s for %s: %s", chosen_source, override_id, exc)
for source in source_priority:
client = get_client_for_source(source)
if not client:
continue
search_query = _build_track_search_query(source, title, artist)
if not search_query:
continue
search_results = _search_tracks_for_source(source, client, search_query, limit=5)
if not search_results and search_query != title:
search_results = _search_tracks_for_source(source, client, title, limit=5)
if not search_results and artist and search_query != artist:
search_results = _search_tracks_for_source(source, client, artist, limit=5)
if not search_results:
continue
best_match = _pick_best_track_match(search_results, title or search_query, artist)
if not best_match:
continue
resolved_track_id = str(_extract_lookup_value(best_match, 'id', 'track_id', 'trackId', default='') or '')
resolved_data = best_match
if resolved_track_id and hasattr(client, 'get_track_details'):
try:
detailed = client.get_track_details(resolved_track_id)
if detailed:
resolved_data = detailed
except Exception as exc:
logger.debug("Track detail lookup failed on %s for %s: %s", source, resolved_track_id, exc)
payload = _build_single_import_context_payload(
resolved_data,
source,
source_priority,
requested_title=title,
requested_artist=artist,
)
if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'):
try:
artist_details = client.get_artist(payload['context']['artist']['id'])
if artist_details:
payload['context']['artist']['genres'] = _extract_lookup_value(
artist_details,
'genres',
default=[],
) or []
except Exception:
pass
return payload
return _build_single_import_fallback_context(title, artist, source_priority)
def resolve_album_reference(
album_id: str,
preferred_source: Optional[str] = None,

View file

@ -8,6 +8,7 @@ import time
from pathlib import Path
from utils.logging_config import get_logger
from config.settings import config_manager
from core.import_filename import parse_filename_metadata
logger = get_logger("soulseek_client")
@ -87,60 +88,17 @@ class TrackResult(SearchResult):
def _parse_filename_metadata(self):
"""Extract artist, title, album from filename patterns"""
import re
import os
# Get just the filename without extension and path
base_name = os.path.splitext(os.path.basename(self.filename))[0]
# Common patterns for track naming
patterns = [
r'^(\d+)\s*[-\.]\s*(.+?)\s*[-]\s*(.+)$', # "01 - Artist - Title" or "01. Artist - Title"
r'^(.+?)\s*[-]\s*(.+)$', # "Artist - Title"
r'^(\d+)\s*[-\.]\s*(.+)$', # "01 - Title" or "01. Title"
]
for pattern in patterns:
match = re.match(pattern, base_name)
if match:
groups = match.groups()
if len(groups) == 3: # Track number, artist, title
try:
self.track_number = int(groups[0])
self.artist = self.artist or groups[1].strip()
self.title = self.title or groups[2].strip()
except ValueError:
# First group might not be a number
self.artist = self.artist or groups[0].strip()
self.title = self.title or f"{groups[1]} - {groups[2]}".strip()
elif len(groups) == 2:
if groups[0].isdigit(): # Track number and title
try:
self.track_number = int(groups[0])
self.title = self.title or groups[1].strip()
except ValueError:
pass
else: # Artist and title
self.artist = self.artist or groups[0].strip()
self.title = self.title or groups[1].strip()
break
# Fallback: use filename as title if nothing was extracted
if not self.title:
self.title = base_name
# Try to extract album from directory path
if not self.album and '/' in self.filename:
path_parts = self.filename.split('/')
if len(path_parts) >= 2:
# Look for album-like directory names
for part in reversed(path_parts[:-1]): # Exclude filename
if part and not part.startswith('@'): # Skip system directories
# Clean up common patterns
cleaned = re.sub(r'^\d+\s*[-\.]\s*', '', part) # Remove leading numbers
if len(cleaned) > 3: # Must be substantial
self.album = cleaned
break
parsed = parse_filename_metadata(self.filename)
if not self.artist and parsed.get("artist"):
self.artist = parsed["artist"]
if not self.title and parsed.get("title"):
self.title = parsed["title"]
if not self.album and parsed.get("album"):
self.album = parsed["album"]
if self.track_number is None:
track_number = parsed.get("track_number")
if track_number is not None:
self.track_number = track_number
@dataclass
class AlbumResult:
@ -1829,4 +1787,4 @@ class SoulseekClient:
def __del__(self):
# No persistent session to clean up
pass
pass

View file

@ -2955,9 +2955,6 @@ class WatchlistScanner:
}
track_data = {
'spotify_track_id': track['id'],
'spotify_album_id': album_data['id'],
'spotify_artist_id': artist.spotify_artist_id,
'track_name': track['name'],
'artist_name': artist.artist_name,
'album_name': album_data.get('name', 'Unknown Album'),

View file

@ -7,7 +7,6 @@ and `status` query params and includes a `total` count.
"""
import sys
import threading
import types
from unittest.mock import patch
@ -45,6 +44,7 @@ _install_flask_limiter_stub()
from flask import Flask, Blueprint # noqa: E402
from api import downloads as downloads_mod # noqa: E402
import core.import_runtime_state as runtime_state # noqa: E402
def _make_task(status="downloading", when=None):
@ -63,11 +63,10 @@ def _make_task(status="downloading", when=None):
def _make_app_with_tasks(tasks_dict):
"""Create a minimal Flask app with the downloads blueprint mounted and
a fake `web_server` module exposing the given download_tasks dict."""
fake_ws = types.ModuleType("web_server")
fake_ws.download_tasks = tasks_dict
fake_ws.tasks_lock = threading.RLock()
sys.modules["web_server"] = fake_ws
the shared import runtime state populated with the given download_tasks dict."""
original_tasks = dict(runtime_state.download_tasks)
runtime_state.download_tasks.clear()
runtime_state.download_tasks.update(tasks_dict)
# Bypass API key auth for tests.
def _passthrough(f):
@ -83,6 +82,7 @@ def _make_app_with_tasks(tasks_dict):
downloads_mod.register_routes(bp)
app.register_blueprint(bp)
app._original_download_tasks = original_tasks
return app
@ -96,8 +96,12 @@ def client():
for i in range(25)
}
app = _make_app_with_tasks(tasks)
with app.test_client() as c:
yield c
try:
with app.test_client() as c:
yield c
finally:
runtime_state.download_tasks.clear()
runtime_state.download_tasks.update(app._original_download_tasks)
def test_default_limit_applied(client):

186
tests/test_import_album.py Normal file
View file

@ -0,0 +1,186 @@
import core.import_album as import_album
class _FakeEngine:
def normalize_string(self, text):
return str(text or "").strip().lower()
def similarity_score(self, left, right):
return 1.0 if left == right else 0.0
def test_resolve_album_artist_context_uses_provider_genres(monkeypatch):
class _FakeClient:
def get_artist(self, artist_id):
assert artist_id == "artist-1"
return {"genres": ["rock", "indie"]}
monkeypatch.setattr(import_album, "get_client_for_source", lambda source: _FakeClient() if source == "spotify" else None)
context = import_album.resolve_album_artist_context(
{
"id": "album-1",
"name": "Album One",
"artist_id": "artist-1",
"artists": [{"name": "Artist One", "id": "artist-1"}],
},
source="spotify",
)
assert context == {
"id": "artist-1",
"name": "Artist One",
"genres": ["rock", "indie"],
"source": "spotify",
}
def test_build_album_import_context_is_neutral():
context = import_album.build_album_import_context(
{
"id": "album-1",
"name": "Album One",
"artist": "Artist One",
"artist_id": "artist-1",
"artists": [{"name": "Artist One", "id": "artist-1"}],
"release_date": "2024-01-01",
"total_tracks": 12,
"album_type": "album",
"image_url": "https://img.example/album.jpg",
"source": "deezer",
},
{
"id": "track-1",
"name": "Song One",
"track_number": 3,
"disc_number": 2,
"duration_ms": 180000,
"artists": [{"name": "Artist One"}],
"uri": "deezer:track:track-1",
"album": "Album One",
"source": "deezer",
},
artist_context={
"id": "artist-1",
"name": "Artist One",
"genres": ["rock"],
"source": "deezer",
},
total_discs=2,
source="deezer",
)
assert context["artist"]["name"] == "Artist One"
assert context["artist"]["genres"] == ["rock"]
assert context["album"]["name"] == "Album One"
assert context["album"]["total_discs"] == 2
assert context["track_info"]["name"] == "Song One"
assert context["track_info"]["track_number"] == 3
assert context["original_search_result"]["clean_title"] == "Song One"
assert context["source"] == "deezer"
assert "spotify_artist" not in context
assert "spotify_album" not in context
def test_build_album_import_match_payload_uses_generic_track_keys(monkeypatch, tmp_path):
staging_root = tmp_path / "Staging"
staging_root.mkdir()
(staging_root / "Song One.flac").write_text("fake")
monkeypatch.setattr(import_album, "get_staging_path", lambda: str(staging_root))
monkeypatch.setattr(import_album, "_get_matching_engine", lambda: _FakeEngine())
monkeypatch.setattr(
import_album,
"read_staging_file_metadata",
lambda file_path, filename=None: {
"title": "Song One",
"artist": "Artist One",
"albumartist": "Artist One",
"album": "Album One",
"track_number": 1,
"disc_number": 1,
},
)
monkeypatch.setattr(
import_album,
"get_artist_album_tracks",
lambda album_id, artist_name="", album_name="", source=None: {
"success": True,
"album": {
"id": album_id,
"name": "Album One",
"artist": "Artist One",
"artist_name": "Artist One",
"artist_id": "artist-1",
"artists": [{"name": "Artist One", "id": "artist-1"}],
"release_date": "2024-01-01",
"total_tracks": 1,
"total_discs": 1,
"album_type": "album",
"image_url": "https://img.example/album.jpg",
"images": [{"url": "https://img.example/album.jpg"}],
"source": "spotify",
},
"tracks": [
{
"id": "track-1",
"name": "Song One",
"track_number": 1,
"disc_number": 1,
"duration_ms": 180000,
"artists": [{"name": "Artist One"}],
"uri": "spotify:track:track-1",
"album": {
"id": album_id,
"name": "Album One",
"artist": "Artist One",
},
"source": "spotify",
}
],
"source": "spotify",
"source_priority": ["spotify"],
"resolved_album_id": album_id,
},
)
result = import_album.build_album_import_match_payload(
"album-1",
album_name="Album One",
album_artist="Artist One",
source="spotify",
)
assert result["success"] is True
assert result["album"]["artist"] == "Artist One"
assert result["source"] == "spotify"
assert result["matches"] == [
{
"track": {
"id": "track-1",
"name": "Song One",
"track_number": 1,
"disc_number": 1,
"duration_ms": 180000,
"artists": [{"name": "Artist One"}],
"uri": "spotify:track:track-1",
"album": {
"id": "album-1",
"name": "Album One",
"artist": "Artist One",
},
"source": "spotify",
},
"staging_file": {
"filename": "Song One.flac",
"full_path": str(staging_root / "Song One.flac"),
"title": "Song One",
"artist": "Artist One",
"album": "Album One",
"albumartist": "Artist One",
"track_number": 1,
"disc_number": 1,
},
"confidence": 1.0,
}
]

View file

@ -0,0 +1,202 @@
import pytest
from core.import_context import (
build_import_album_info,
get_import_clean_album,
get_import_clean_artist,
get_import_clean_title,
get_import_has_clean_metadata,
get_import_has_full_metadata,
get_import_source,
get_import_source_ids,
get_library_source_id_columns,
get_source_tag_names,
normalize_import_context,
)
def test_normalize_import_context_promotes_neutral_fields_without_legacy_aliases():
context = {
"source": "deezer",
"spotify_artist": {"name": "Artist One", "id": "artist-1"},
"spotify_album": {
"name": "Album One",
"id": "album-1",
"release_date": "2024-01-01",
"total_tracks": 12,
"album_type": "album",
"image_url": "https://img.example/album.jpg",
},
"track_info": {
"name": "Song One",
"id": "track-1",
"track_number": 3,
"disc_number": 2,
"artists": [{"name": "Artist One"}],
},
"original_search_result": {
"spotify_clean_title": "Song One",
"spotify_clean_album": "Album One",
"spotify_clean_artist": "Artist One",
},
"has_clean_spotify_data": True,
"has_full_spotify_metadata": False,
}
normalized = normalize_import_context(context)
assert normalized["artist"]["name"] == "Artist One"
assert normalized["album"]["name"] == "Album One"
assert "spotify_artist" not in normalized
assert "spotify_album" not in normalized
assert normalized["original_search_result"]["clean_title"] == "Song One"
assert normalized["original_search_result"]["clean_album"] == "Album One"
assert normalized["original_search_result"]["clean_artist"] == "Artist One"
assert "spotify_clean_title" not in normalized["original_search_result"]
assert "spotify_clean_album" not in normalized["original_search_result"]
assert "spotify_clean_artist" not in normalized["original_search_result"]
assert get_import_clean_title(normalized) == "Song One"
assert get_import_clean_album(normalized) == "Album One"
assert get_import_clean_artist(normalized) == "Artist One"
assert get_import_source(normalized) == "deezer"
assert get_import_has_clean_metadata(normalized) is True
assert get_import_has_full_metadata(normalized) is False
def test_neutral_import_context_helpers_work_without_legacy_aliases():
context = {
"source": "deezer",
"artist": {"name": "Artist One", "id": "artist-1"},
"album": {
"name": "Album One",
"id": "album-1",
"release_date": "2024-01-01",
"total_tracks": 12,
"album_type": "album",
"image_url": "https://img.example/album.jpg",
},
"track_info": {
"name": "Song One",
"id": "track-1",
"track_number": 3,
"disc_number": 2,
"artists": [{"name": "Artist One"}],
},
"original_search_result": {
"title": "Song One",
"artist": "Artist One",
"album": "Album One",
"clean_title": "Song One",
"clean_album": "Album One",
"clean_artist": "Artist One",
},
"has_clean_metadata": True,
"has_full_metadata": True,
}
assert get_import_clean_title(context) == "Song One"
assert get_import_clean_album(context) == "Album One"
assert get_import_clean_artist(context) == "Artist One"
assert get_import_source(context) == "deezer"
normalized = normalize_import_context(context)
assert normalized["artist"]["name"] == "Artist One"
assert normalized["album"]["name"] == "Album One"
assert normalized["original_search_result"]["clean_title"] == "Song One"
assert "spotify_artist" not in normalized
assert "spotify_album" not in normalized
assert get_import_has_clean_metadata(normalized) is True
assert get_import_has_full_metadata(normalized) is True
@pytest.mark.parametrize(
"source,expected_tags,expected_columns",
[
(
"spotify",
{"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"},
{"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"},
),
(
"itunes",
{"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"},
{"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"},
),
(
"deezer",
{"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None},
{"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"},
),
(
"hydrabase",
{"track": None, "artist": None, "album": None},
{"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"},
),
(
"discogs",
{"track": None, "artist": None, "album": None},
{"artist": "discogs_id", "album": "discogs_id", "track": None},
),
],
)
def test_source_tag_and_library_column_mappings(source, expected_tags, expected_columns):
assert get_source_tag_names(source) == expected_tags
assert get_library_source_id_columns(source) == expected_columns
def test_get_import_source_ids_prefers_nested_source_specific_ids():
context = normalize_import_context(
{
"source": "deezer",
"artist": {"deezer_id": "deezer-artist-1"},
"album": {"deezer_id": "deezer-album-1"},
"track_info": {"deezer_id": "deezer-track-1"},
"original_search_result": {"source_artist_id": "deezer-artist-1"},
}
)
assert get_import_source_ids(context) == {
"track_id": "deezer-track-1",
"artist_id": "deezer-artist-1",
"album_id": "deezer-album-1",
}
def test_build_import_album_info_uses_normalized_album_context():
context = normalize_import_context(
{
"source": "deezer",
"artist": {"name": "Artist One"},
"album": {
"name": "Album One",
"image_url": "https://img.example/album.jpg",
"release_date": "2024-05-01",
"total_tracks": 8,
"album_type": "album",
},
"track_info": {
"name": "Song One",
"track_number": 4,
"disc_number": 2,
"duration_ms": 240000,
"artists": [{"name": "Artist One"}],
},
"original_search_result": {
"title": "Song One",
"album": "Album One",
"clean_title": "Song One",
"clean_album": "Album One",
"clean_artist": "Artist One",
},
}
)
album_info = build_import_album_info(context)
assert album_info["is_album"] is True
assert album_info["album_name"] == "Album One"
assert album_info["track_number"] == 4
assert album_info["disc_number"] == 2
assert album_info["clean_track_name"] == "Song One"
assert album_info["album_image_url"] == "https://img.example/album.jpg"
assert album_info["source"] == "deezer"

View file

@ -0,0 +1,92 @@
import sys
import types
from core.import_file_ops import (
cleanup_empty_directories,
extract_track_number_from_filename,
safe_move_file,
read_staging_file_metadata,
)
def test_extract_track_number_from_filename_handles_common_patterns():
assert extract_track_number_from_filename("01 - Song.mp3") == 1
assert extract_track_number_from_filename("1-03 - Song.mp3") == 3
assert extract_track_number_from_filename("Artist - Song.mp3") == 1
def test_safe_move_file_replaces_existing_destination(tmp_path):
src = tmp_path / "source.flac"
dst_dir = tmp_path / "dest"
dst_dir.mkdir()
dst = dst_dir / "track.flac"
src.write_text("new")
dst.write_text("old")
safe_move_file(src, dst)
assert not src.exists()
assert dst.read_text() == "new"
def test_cleanup_empty_directories_removes_nested_empty_paths(tmp_path):
download_root = tmp_path / "downloads"
nested_dir = download_root / "Artist" / "Album"
nested_dir.mkdir(parents=True)
moved_file_path = nested_dir / "track.flac"
cleanup_empty_directories(str(download_root), str(moved_file_path))
assert not nested_dir.exists()
assert not (download_root / "Artist").exists()
assert download_root.exists()
def test_read_staging_file_metadata_reads_tags(monkeypatch, tmp_path):
file_path = tmp_path / "Song One.flac"
file_path.write_text("fake")
class DummyTags:
def __init__(self):
self.values = {
"title": ["Song One"],
"artist": ["Artist One"],
"albumartist": ["Album Artist"],
"album": ["Album One"],
"tracknumber": ["03/12"],
"discnumber": ["2/3"],
}
def get(self, key, default=None):
return self.values.get(key, default)
fake_mutagen = types.ModuleType("mutagen")
fake_mutagen.File = lambda path, easy=True: DummyTags()
monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen)
metadata = read_staging_file_metadata(str(file_path), file_path.name)
assert metadata == {
"title": "Song One",
"artist": "Artist One",
"albumartist": "Album Artist",
"album": "Album One",
"track_number": 3,
"disc_number": 2,
}
def test_read_staging_file_metadata_falls_back_to_filename_track_number(monkeypatch, tmp_path):
file_path = tmp_path / "07 - Song Two.flac"
file_path.write_text("fake")
fake_mutagen = types.ModuleType("mutagen")
fake_mutagen.File = lambda path, easy=True: None
monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen)
metadata = read_staging_file_metadata(str(file_path), file_path.name)
assert metadata["title"] == "07 - Song Two"
assert metadata["track_number"] == 7
assert metadata["disc_number"] == 1

View file

@ -0,0 +1,33 @@
import pytest
from core.import_filename import parse_filename_metadata
@pytest.mark.parametrize(
"filename,expected",
[
(
"01 - Artist One - Title One.mp3",
{"artist": "Artist One", "title": "Title One", "album": "", "track_number": 1},
),
(
"Artist Two - Title Two.flac",
{"artist": "Artist Two", "title": "Title Two", "album": "", "track_number": None},
),
(
r"Artist Three\Album Three\03 - Title Three.ogg",
{"artist": "", "title": "Title Three", "album": "Album Three", "track_number": 3},
),
(
"Loose Song.wav",
{"artist": "", "title": "Loose Song", "album": "", "track_number": None},
),
],
)
def test_parse_filename_metadata_handles_common_patterns(filename, expected):
parsed = parse_filename_metadata(filename)
assert parsed["artist"] == expected["artist"]
assert parsed["title"] == expected["title"]
assert parsed["album"] == expected["album"]
assert parsed["track_number"] == expected["track_number"]

View file

@ -0,0 +1,47 @@
from types import SimpleNamespace
from core import import_guards as guards
class _FakeDB:
def __init__(self, quality_profile):
self._quality_profile = quality_profile
def get_quality_profile(self):
return self._quality_profile
def test_check_flac_bit_depth_rejects_strict_mismatch(monkeypatch):
monkeypatch.setattr(
guards,
"MusicDatabase",
lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": False}}}),
)
monkeypatch.setattr(
guards,
"_get_config_manager",
lambda: SimpleNamespace(get=lambda _key, default=None: False),
)
context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}}
assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) == (
"FLAC bit depth mismatch: file is 24-bit, preference is 16-bit"
)
def test_check_flac_bit_depth_allows_fallback_when_enabled(monkeypatch):
monkeypatch.setattr(
guards,
"MusicDatabase",
lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": True}}}),
)
monkeypatch.setattr(
guards,
"_get_config_manager",
lambda: SimpleNamespace(get=lambda _key, default=None: False),
)
context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}}
assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) is None

161
tests/test_import_paths.py Normal file
View file

@ -0,0 +1,161 @@
import core.import_paths as import_paths
class _Config:
def __init__(self, values):
self._values = values
def get(self, key, default=None):
return self._values.get(key, default)
def test_sanitize_filename_replaces_illegal_characters():
assert import_paths.sanitize_filename("AC/DC: Song?") == "AC_DC_ Song_"
assert import_paths.sanitize_filename("AUX.txt").startswith("_")
def test_build_simple_download_destination_uses_album_folder(monkeypatch, tmp_path):
config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")})
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config)
destination, album_name, filename = import_paths.build_simple_download_destination(
{
"search_result": {
"filename": "Album Folder/source.flac",
"album": "Album Folder",
}
},
str(tmp_path / "source.flac"),
)
assert destination == tmp_path / "Transfer" / "Album Folder" / "source.flac"
assert album_name == "Album Folder"
assert filename == "source.flac"
assert destination.parent.exists()
def test_build_simple_download_destination_falls_back_to_transfer_root(monkeypatch, tmp_path):
config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")})
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config)
destination, album_name, filename = import_paths.build_simple_download_destination(
{
"search_result": {
"filename": "source.flac",
"album": "Unknown Album",
}
},
str(tmp_path / "source.flac"),
)
assert destination == tmp_path / "Transfer" / "source.flac"
assert album_name == ""
assert filename == "source.flac"
assert destination.parent.exists()
def test_get_file_path_from_template_raw_handles_quality_and_disc_placeholders(monkeypatch):
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config({}))
folder_path, filename = import_paths.get_file_path_from_template_raw(
"$artist/$album/$discnum - $title [$quality]",
{
"artist": "Artist One",
"album": "Album One",
"title": "Song One",
"quality": "FLAC 16bit",
"disc_number": 3,
},
)
assert folder_path == "Artist One/Album One"
assert filename == "3 - Song One [FLAC 16bit]"
def test_resolve_album_group_upgrades_standard_to_deluxe():
import_paths._album_name_cache.clear()
import_paths._album_editions.clear()
artist_context = {"name": "Cache Artist"}
standard_album = {"album_name": "Cache Album"}
deluxe_album = {"album_name": "Cache Album (Deluxe Edition)"}
assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album"
assert import_paths.resolve_album_group(artist_context, deluxe_album) == "Cache Album (Deluxe Edition)"
assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album (Deluxe Edition)"
def test_build_final_path_for_track_uses_template_and_disc_folder(monkeypatch, tmp_path):
config = _Config(
{
"soulseek.transfer_path": str(tmp_path / "Transfer"),
"file_organization.enabled": True,
"file_organization.templates": {
"album_path": "$albumartist/$albumartist - $album/$track - $title",
"single_path": "$artist/$artist - $title",
},
"file_organization.collab_artist_mode": "first",
"file_organization.disc_label": "Disc",
}
)
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config)
calls = []
def fake_get_album_tracks_for_source(source, album_id):
calls.append((source, album_id))
return {
"items": [
{"disc_number": 1},
{"disc_number": 2},
]
}
monkeypatch.setattr(import_paths, "_get_album_tracks_for_source", fake_get_album_tracks_for_source)
context = {
"artist": {"name": "Artist One"},
"album": {
"name": "Album One",
"id": "album-1",
"release_date": "2026-01-01",
"total_tracks": 12,
"album_type": "album",
"artists": [{"name": "Artist One"}],
},
"track_info": {
"name": "Song One",
"id": "track-1",
"track_number": 4,
"disc_number": 1,
"artists": [{"name": "Artist One"}],
},
"original_search_result": {
"title": "Song One",
"clean_title": "Song One",
"clean_album": "Album One",
"clean_artist": "Artist One",
"artists": [{"name": "Artist One"}],
},
"source": "deezer",
"is_album_download": False,
}
final_path, created = import_paths.build_final_path_for_track(
context,
{"name": "Artist One"},
{
"is_album": True,
"album_name": "Album One",
"track_number": 4,
"disc_number": 1,
},
".flac",
)
assert created is True
assert calls == [("deezer", "album-1")]
assert final_path == str(
tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1" / "04 - Song One.flac"
)
assert (tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1").is_dir()

View file

@ -0,0 +1,128 @@
import logging
import sys
import types
import core.import_pipeline as import_pipeline
import core.import_paths as import_paths
import core.import_runtime_state as runtime_state
class _Config:
def __init__(self, transfer_path):
self.transfer_path = transfer_path
def get(self, key, default=None):
if key == "soulseek.transfer_path":
return self.transfer_path
if key in {"post_processing.replaygain_enabled", "lossy_copy.enabled", "lossy_copy.delete_original", "import.replace_lower_quality"}:
return False
return default
class _FakeAcoustidVerifier:
def quick_check_available(self):
return False, "disabled"
class _ImmediateThread:
def __init__(self, target=None, daemon=None):
self._target = target
def start(self):
if self._target:
self._target()
def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch):
transfer_root = tmp_path / "Transfer"
transfer_root.mkdir()
source_path = tmp_path / "source.flac"
source_path.write_bytes(b"audio")
context_key = "ctx-1"
task_id = "task-1"
batch_id = "batch-1"
context = {
"search_result": {
"is_simple_download": True,
"filename": "Album Folder/source.flac",
"album": "Album Folder",
},
"track_info": {},
"original_search_result": {},
"is_album_download": False,
"task_id": task_id,
"batch_id": batch_id,
}
mark_calls = []
completion_calls = []
scan_calls = []
activity_calls = []
original_matched_context = dict(runtime_state.matched_downloads_context)
original_download_tasks = dict(runtime_state.download_tasks)
original_download_batches = dict(runtime_state.download_batches)
original_processed_ids = set(runtime_state._processed_download_ids)
original_post_process_locks = dict(runtime_state._post_process_locks)
runtime_state.matched_downloads_context.clear()
runtime_state.download_tasks.clear()
runtime_state.download_batches.clear()
runtime_state._processed_download_ids.clear()
runtime_state._post_process_locks.clear()
runtime = types.SimpleNamespace(
automation_engine=None,
on_download_completed=lambda batch, task, success: completion_calls.append((batch, task, success)),
web_scan_manager=types.SimpleNamespace(request_scan=lambda reason: scan_calls.append(reason)),
repair_worker=None,
)
fake_acoustid = types.ModuleType("core.acoustid_verification")
fake_acoustid.AcoustIDVerification = _FakeAcoustidVerifier
fake_acoustid.VerificationResult = types.SimpleNamespace(FAIL="FAIL")
monkeypatch.setitem(sys.modules, "core.acoustid_verification", fake_acoustid)
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config(str(transfer_root)))
monkeypatch.setattr(import_pipeline, "add_activity_item", lambda *args, **kwargs: activity_calls.append((args, kwargs)))
monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_library_history_download", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_download_provenance", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "_mark_task_completed", lambda task, track_info: mark_calls.append((task, track_info)))
monkeypatch.setattr(import_pipeline.threading, "Thread", _ImmediateThread)
runtime_state.matched_downloads_context[context_key] = context
runtime_state.download_tasks[task_id] = {"track_info": {}, "status": "running"}
try:
import_pipeline.post_process_matched_download_with_verification(
context_key,
context,
str(source_path),
task_id,
batch_id,
runtime,
)
expected_path = transfer_root / "Album Folder" / "source.flac"
assert expected_path.exists()
assert not source_path.exists()
assert context["_simple_download_completed"] is True
assert context["_final_path"] == str(expected_path)
assert mark_calls == [(task_id, {})]
assert completion_calls == [(batch_id, task_id, True)]
assert context_key not in runtime_state.matched_downloads_context
assert scan_calls == ["Simple download completed"]
assert activity_calls
finally:
runtime_state.matched_downloads_context.clear()
runtime_state.matched_downloads_context.update(original_matched_context)
runtime_state.download_tasks.clear()
runtime_state.download_tasks.update(original_download_tasks)
runtime_state.download_batches.clear()
runtime_state.download_batches.update(original_download_batches)
runtime_state._processed_download_ids.clear()
runtime_state._processed_download_ids.update(original_processed_ids)
runtime_state._post_process_locks.clear()
runtime_state._post_process_locks.update(original_post_process_locks)

View file

@ -0,0 +1,144 @@
import sqlite3
from types import SimpleNamespace
from core import import_side_effects as side_effects
class _FakeDB:
def __init__(self, conn):
self._conn = conn
def _get_connection(self):
return self._conn
def _make_soulsync_db():
conn = sqlite3.connect(":memory:")
conn.row_factory = sqlite3.Row
conn.execute(
"""
CREATE TABLE artists (
id TEXT PRIMARY KEY,
name TEXT,
genres TEXT,
thumb_url TEXT,
server_source TEXT,
created_at TEXT,
updated_at TEXT,
spotify_artist_id TEXT
)
"""
)
conn.execute(
"""
CREATE TABLE albums (
id TEXT PRIMARY KEY,
artist_id TEXT,
title TEXT,
year INTEGER,
thumb_url TEXT,
genres TEXT,
track_count INTEGER,
duration INTEGER,
server_source TEXT,
created_at TEXT,
updated_at TEXT,
spotify_album_id TEXT
)
"""
)
conn.execute(
"""
CREATE TABLE tracks (
id TEXT PRIMARY KEY,
album_id TEXT,
artist_id TEXT,
title TEXT,
track_number INTEGER,
duration INTEGER,
file_path TEXT,
bitrate INTEGER,
track_artist TEXT,
server_source TEXT,
created_at TEXT,
updated_at TEXT,
spotify_track_id TEXT
)
"""
)
return conn
def test_record_soulsync_library_entry_writes_artist_album_and_track(tmp_path, monkeypatch):
conn = _make_soulsync_db()
fake_db = _FakeDB(conn)
final_path = tmp_path / "track.flac"
final_path.write_bytes(b"audio")
monkeypatch.setattr(side_effects, "get_database", lambda: fake_db)
monkeypatch.setattr(
side_effects,
"_get_config_manager",
lambda: SimpleNamespace(get_active_media_server=lambda: "soulsync"),
)
import core.genre_filter as genre_filter
monkeypatch.setattr(genre_filter, "filter_genres", lambda genres, _cfg: [genre.upper() for genre in genres])
context = {
"source": "spotify",
"artist": {"id": "sp-artist", "name": "Artist One"},
"album": {
"id": "sp-album",
"name": "Album One",
"release_date": "2024-02-03",
"total_tracks": 12,
"image_url": "https://img.example/album.jpg",
},
"track_info": {
"id": "sp-track",
"name": "Song One",
"track_number": 7,
"duration_ms": 210000,
"artists": [{"name": "Guest Artist"}],
"_source": "spotify",
},
"original_search_result": {
"title": "Song One",
"artists": [{"name": "Guest Artist"}],
"_source": "spotify",
},
"_final_processed_path": str(final_path),
}
artist_context = {"name": "Artist One", "genres": ["rock", "indie"]}
album_info = {"is_album": True, "album_name": "Album One", "track_number": 7}
side_effects.record_soulsync_library_entry(context, artist_context, album_info)
artist_row = conn.execute("SELECT * FROM artists").fetchone()
album_row = conn.execute("SELECT * FROM albums").fetchone()
track_row = conn.execute("SELECT * FROM tracks").fetchone()
assert artist_row["name"] == "Artist One"
assert artist_row["server_source"] == "soulsync"
assert artist_row["spotify_artist_id"] == "sp-artist"
assert artist_row["genres"] == '["ROCK", "INDIE"]'
assert album_row["title"] == "Album One"
assert album_row["server_source"] == "soulsync"
assert album_row["spotify_album_id"] == "sp-album"
assert album_row["year"] == 2024
assert album_row["track_count"] == 12
assert album_row["duration"] == 210000
assert album_row["artist_id"] == artist_row["id"]
assert track_row["title"] == "Song One"
assert track_row["server_source"] == "soulsync"
assert track_row["spotify_track_id"] == "sp-track"
assert track_row["track_number"] == 7
assert track_row["duration"] == 210000
assert track_row["track_artist"] == "Guest Artist"
assert track_row["album_id"] == album_row["id"]
assert track_row["file_path"] == str(final_path)

View file

@ -0,0 +1,265 @@
from types import SimpleNamespace
import core.import_staging as import_staging
class FakeClient:
def __init__(self, results=None):
self.results = results or []
self.calls = []
def search_albums(self, query, **kwargs):
self.calls.append((query, kwargs))
return self.results
def search_tracks(self, query, **kwargs):
self.calls.append((query, kwargs))
return self.results
def _album_result(album_id, name, artist, release_date="2024-01-01", total_tracks=10, image_url="https://img.example/album.jpg", album_type="album"):
return SimpleNamespace(
id=album_id,
name=name,
artists=[artist],
release_date=release_date,
total_tracks=total_tracks,
image_url=image_url,
album_type=album_type,
)
def test_search_import_albums_prefers_primary_source(monkeypatch):
deezer_client = FakeClient([
_album_result("deezer-1", "Album One", "Artist One"),
])
spotify_client = FakeClient([
_album_result("spotify-1", "Album One", "Artist One"),
])
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
monkeypatch.setattr(
import_staging,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
)
monkeypatch.setattr(
import_staging,
"_search_albums_for_source",
lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit),
)
results = import_staging.search_import_albums("Album One", limit=2)
assert results == [
{
"id": "deezer-1",
"name": "Album One",
"artist": "Artist One",
"release_date": "2024-01-01",
"total_tracks": 10,
"image_url": "https://img.example/album.jpg",
"album_type": "album",
"source": "deezer",
}
]
assert deezer_client.calls == [("Album One", {"limit": 2})]
assert spotify_client.calls == []
def test_search_import_albums_falls_back_when_primary_has_no_results(monkeypatch):
deezer_client = FakeClient([])
spotify_client = FakeClient([
_album_result("spotify-1", "Album Two", "Artist Two"),
])
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
monkeypatch.setattr(
import_staging,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
)
monkeypatch.setattr(
import_staging,
"_search_albums_for_source",
lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit),
)
results = import_staging.search_import_albums("Album Two", limit=2)
assert results == [
{
"id": "spotify-1",
"name": "Album Two",
"artist": "Artist Two",
"release_date": "2024-01-01",
"total_tracks": 10,
"image_url": "https://img.example/album.jpg",
"album_type": "album",
"source": "spotify",
}
]
assert deezer_client.calls == [("Album Two", {"limit": 2})]
assert spotify_client.calls == [("Album Two", {"limit": 2, "allow_fallback": False})]
def test_search_import_tracks_prefers_primary_source(monkeypatch):
deezer_client = FakeClient([
SimpleNamespace(
id="deezer-track-1",
name="Song One",
artists=[{"name": "Artist One"}],
album={"id": "deezer-album-1", "name": "Album One"},
duration_ms=210000,
image_url="https://img.example/track.jpg",
track_number=7,
),
])
spotify_client = FakeClient([
SimpleNamespace(
id="spotify-track-1",
name="Song One",
artists=["Artist One"],
album="Album One",
duration_ms=210000,
image_url="https://img.example/track.jpg",
track_number=7,
),
])
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
monkeypatch.setattr(
import_staging,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
)
monkeypatch.setattr(
import_staging,
"_search_tracks_for_source",
lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit),
)
results = import_staging.search_import_tracks("Song One", limit=2)
assert results == [
{
"id": "deezer-track-1",
"name": "Song One",
"artist": "Artist One",
"album": "Album One",
"album_id": "deezer-album-1",
"duration_ms": 210000,
"image_url": "https://img.example/track.jpg",
"track_number": 7,
"source": "deezer",
}
]
assert deezer_client.calls == [("Song One", {"limit": 2})]
assert spotify_client.calls == []
def test_search_import_tracks_falls_back_when_primary_has_no_results(monkeypatch):
deezer_client = FakeClient([])
spotify_client = FakeClient([
SimpleNamespace(
id="spotify-track-1",
name="Song Two",
artists=["Artist Two"],
album="Album Two",
duration_ms=180000,
image_url="",
track_number=3,
),
])
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
monkeypatch.setattr(
import_staging,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
)
monkeypatch.setattr(
import_staging,
"_search_tracks_for_source",
lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit),
)
results = import_staging.search_import_tracks("Song Two", limit=2)
assert results == [
{
"id": "spotify-track-1",
"name": "Song Two",
"artist": "Artist Two",
"album": "Album Two",
"album_id": "",
"duration_ms": 180000,
"image_url": "",
"track_number": 3,
"source": "spotify",
}
]
assert deezer_client.calls == [("Song Two", {"limit": 2})]
assert spotify_client.calls == [("Song Two", {"limit": 2, "allow_fallback": False})]
def test_build_import_suggestions_background_uses_collected_queries(monkeypatch, tmp_path):
staging_root = tmp_path / "Staging"
staging_root.mkdir()
cache = import_staging.get_import_suggestions_cache()
original_cache = dict(cache)
cache["suggestions"] = []
cache["building"] = False
cache["built"] = False
monkeypatch.setattr(import_staging, "get_staging_path", lambda: str(staging_root))
monkeypatch.setattr(import_staging, "_collect_import_suggestion_queries", lambda staging_path: ["Album One", "Folder Hint"])
monkeypatch.setattr(
import_staging,
"search_import_albums",
lambda query, limit=2: [
{
"id": query.lower().replace(" ", "-"),
"name": query,
"artist": f"{query} Artist",
"release_date": "2024-02-01",
"total_tracks": 12,
"image_url": "",
"album_type": "album",
}
],
)
try:
import_staging._build_import_suggestions_background()
assert cache["built"] is True
assert cache["building"] is False
assert cache["suggestions"] == [
{
"id": "album-one",
"name": "Album One",
"artist": "Album One Artist",
"release_date": "2024-02-01",
"total_tracks": 12,
"image_url": "",
"album_type": "album",
},
{
"id": "folder-hint",
"name": "Folder Hint",
"artist": "Folder Hint Artist",
"release_date": "2024-02-01",
"total_tracks": 12,
"image_url": "",
"album_type": "album",
},
]
finally:
cache.clear()
cache.update(original_cache)

View file

@ -0,0 +1,333 @@
import logging
import types
import pytest
from core import metadata_enrichment as me
class _Config:
def __init__(self, values=None):
self.values = values or {}
def get(self, key, default=None):
return self.values.get(key, default)
class _FakeTag:
def __init__(self, kind, **kwargs):
self.kind = kind
self.kwargs = kwargs
class _FakeID3Tags:
def __init__(self):
self.added = []
def add(self, frame):
self.added.append(frame)
def clear(self):
self.added.clear()
def getall(self, _key):
return []
def keys(self):
return [frame.kind for frame in self.added]
def __len__(self):
return len(self.added)
class _FakeAudio:
def __init__(self):
self.tags = _FakeID3Tags()
self.save_calls = []
self.clear_pictures_calls = 0
def clear_pictures(self):
self.clear_pictures_calls += 1
def add_tags(self):
self.tags = _FakeID3Tags()
def save(self, **kwargs):
self.save_calls.append(kwargs)
class _FakeResponse:
def __init__(self, payload, content_type="image/jpeg"):
self._payload = payload
self._content_type = content_type
def read(self):
return self._payload
def info(self):
return types.SimpleNamespace(get_content_type=lambda: self._content_type)
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def _fake_symbols(audio):
def _tag_factory(kind):
return lambda **kwargs: _FakeTag(kind, **kwargs)
return types.SimpleNamespace(
File=lambda _path: audio,
APEv2=type("FakeAPEv2", (), {}),
APENoHeaderError=Exception,
FLAC=type("FakeFLAC", (), {}),
Picture=type("FakePicture", (), {"__init__": lambda self: None}),
ID3=_FakeID3Tags,
APIC=_tag_factory("APIC"),
TBPM=_tag_factory("TBPM"),
TCOP=_tag_factory("TCOP"),
TDOR=_tag_factory("TDOR"),
TDRC=_tag_factory("TDRC"),
TCON=_tag_factory("TCON"),
TIT2=_tag_factory("TIT2"),
TALB=_tag_factory("TALB"),
TPE1=_tag_factory("TPE1"),
TPE2=_tag_factory("TPE2"),
TPOS=_tag_factory("TPOS"),
TPUB=_tag_factory("TPUB"),
TRCK=_tag_factory("TRCK"),
TSRC=_tag_factory("TSRC"),
TXXX=_tag_factory("TXXX"),
UFID=_tag_factory("UFID"),
TMED=_tag_factory("TMED"),
MP4=type("FakeMP4", (), {}),
MP4Cover=types.SimpleNamespace(FORMAT_JPEG=1, FORMAT_PNG=2, __call__=lambda *args, **kwargs: ("cover", args, kwargs)),
MP4FreeForm=lambda data: ("freeform", data),
OggVorbis=type("FakeOggVorbis", (), {}),
OggOpus=None,
)
def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_for_non_itunes_sources():
class _ItunesClient:
def __init__(self):
self.called = False
def resolve_primary_artist(self, artist_id):
self.called = True
raise AssertionError("itunes fallback should not run for non-itunes sources")
runtime = types.SimpleNamespace(
logger=logging.getLogger("test.metadata_enrichment"),
config_manager=_Config({"file_organization.collab_artist_mode": "first"}),
itunes_enrichment_worker=types.SimpleNamespace(client=_ItunesClient()),
)
context = {
"source": "spotify",
"artist": {"name": "Artist One & Artist Two", "id": "123", "genres": ["rock", "indie"]},
"album": {
"name": "Album One",
"total_tracks": 12,
"release_date": "2024-01-02",
"images": [{"url": "https://img.example/album.jpg"}],
},
"track_info": {
"artists": [{"name": "Artist One"}],
"_source": "spotify",
"track_number": 3,
"disc_number": 2,
"total_tracks": 12,
},
"original_search_result": {
"title": "Song One",
"artists": [{"name": "Artist One"}],
"clean_title": "Song One",
"clean_album": "Album One",
"clean_artist": "Artist One",
"disc_number": 2,
"duration_ms": 180000,
},
}
metadata = me.extract_source_metadata(
context,
context["artist"],
{"is_album": True, "album_name": "Album One", "track_number": 3, "disc_number": 2, "album_image_url": "https://img.example/album.jpg"},
runtime=runtime,
)
assert metadata["source"] == "spotify"
assert metadata["title"] == "Song One"
assert metadata["artist"] == "Artist One"
assert metadata["album_artist"] == "Artist One & Artist Two"
assert metadata["album"] == "Album One"
assert metadata["track_number"] == 3
assert metadata["total_tracks"] == 12
assert metadata["disc_number"] == 2
assert metadata["album_art_url"] == "https://img.example/album.jpg"
assert runtime.itunes_enrichment_worker.client.called is False
def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatch):
runtime = types.SimpleNamespace(
logger=logging.getLogger("test.metadata_enrichment"),
config_manager=_Config(),
mb_worker=None,
deezer_worker=None,
audiodb_worker=None,
tidal_client=None,
qobuz_enrichment_worker=None,
lastfm_worker=None,
genius_worker=None,
itunes_enrichment_worker=None,
get_database=lambda: None,
)
audio = _FakeAudio()
symbols = _fake_symbols(audio)
monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols)
current_metadata = {
"source": "deezer",
"source_track_id": "dz-track",
"source_artist_id": "dz-artist",
"source_album_id": "dz-album",
"title": "Song One",
"artist": "Artist One",
"album_artist": "Artist One",
"album": "Album One",
}
me.embed_source_ids(audio, current_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime)
current_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"]
assert "DEEZER_TRACK_ID" in current_descs
assert "DEEZER_ARTIST_ID" in current_descs
audio = _FakeAudio()
symbols = _fake_symbols(audio)
monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols)
legacy_metadata = {
"source": "",
"spotify_track_id": "sp-track",
"spotify_artist_id": "sp-artist",
"spotify_album_id": "sp-album",
"itunes_track_id": "it-track",
"itunes_artist_id": "it-artist",
"itunes_album_id": "it-album",
"title": "Song One",
"artist": "Artist One",
"album_artist": "Artist One",
"album": "Album One",
}
me.embed_source_ids(audio, legacy_metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime)
legacy_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"]
assert "SPOTIFY_TRACK_ID" in legacy_descs
assert "SPOTIFY_ARTIST_ID" in legacy_descs
assert "SPOTIFY_ALBUM_ID" in legacy_descs
assert "ITUNES_TRACK_ID" in legacy_descs
assert "ITUNES_ARTIST_ID" in legacy_descs
assert "ITUNES_ALBUM_ID" in legacy_descs
def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch):
audio = _FakeAudio()
symbols = _fake_symbols(audio)
runtime = types.SimpleNamespace(
logger=logging.getLogger("test.metadata_enrichment"),
config_manager=_Config(
{
"metadata_enhancement.enabled": True,
"metadata_enhancement.embed_album_art": False,
"metadata_enhancement.tags.write_multi_artist": False,
}
),
mb_worker=None,
deezer_worker=None,
audiodb_worker=None,
tidal_client=None,
qobuz_enrichment_worker=None,
lastfm_worker=None,
genius_worker=None,
itunes_enrichment_worker=None,
get_database=lambda: None,
)
strip_calls = []
verify_calls = []
monkeypatch.setattr(me, "_get_mutagen_symbols", lambda runtime=None: symbols)
monkeypatch.setattr(me, "_strip_all_non_audio_tags", lambda file_path, runtime=None: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0})
monkeypatch.setattr(
me,
"extract_source_metadata",
lambda context, artist, album_info, runtime=None: {
"source": "deezer",
"source_track_id": "dz-track",
"source_artist_id": "dz-artist",
"source_album_id": "dz-album",
"title": "Song One",
"artist": "Artist One",
"album_artist": "Artist One",
"album": "Album One",
"track_number": 3,
"total_tracks": 12,
"disc_number": 2,
"date": "2024",
"genre": "Rock",
"musicbrainz_release_id": "mb-release-1",
},
)
monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None)
monkeypatch.setattr(me, "_verify_metadata_written", lambda file_path, runtime=None: verify_calls.append(file_path) or True)
album_info = {}
result = me.enhance_file_metadata(
"song.flac",
{"_audio_quality": ""},
{"name": "Artist One"},
album_info,
runtime=runtime,
)
assert result is True
assert strip_calls == ["song.flac"]
assert verify_calls == ["song.flac"]
assert audio.clear_pictures_calls == 1
assert len(audio.save_calls) == 2
assert album_info["musicbrainz_release_id"] == "mb-release-1"
assert any(frame.kind == "TIT2" for frame in audio.tags.added)
assert any(frame.kind == "TPE1" for frame in audio.tags.added)
assert any(frame.kind == "TXXX" and frame.kwargs.get("desc") == "DEEZER_TRACK_ID" for frame in audio.tags.added)
def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch):
runtime = types.SimpleNamespace(
logger=logging.getLogger("test.metadata_enrichment"),
config_manager=_Config(
{
"metadata_enhancement.cover_art_download": True,
"metadata_enhancement.prefer_caa_art": False,
}
),
)
monkeypatch.setattr(me.urllib.request, "urlopen", lambda *args, **kwargs: _FakeResponse(b"cover-bytes"))
target_dir = tmp_path / "Album One"
target_dir.mkdir()
me.download_cover_art(
{},
str(target_dir),
{"album": {"image_url": "https://img.example/album.jpg"}},
runtime=runtime,
)
cover_path = target_dir / "cover.jpg"
assert cover_path.exists()
assert cover_path.read_bytes() == b"cover-bytes"

View file

@ -0,0 +1,193 @@
from types import SimpleNamespace
from core import metadata_service
class FakeClient:
def __init__(self, search_results=None, details=None, artist_details=None):
self.search_results = search_results or []
self.details = details or {}
self.artist_details = artist_details or {}
self.calls = []
def search_tracks(self, query, limit=1, allow_fallback=True):
self.calls.append(("search_tracks", query, limit, allow_fallback))
return self.search_results
def get_track_details(self, track_id):
self.calls.append(("get_track_details", track_id))
return self.details.get(str(track_id))
def get_artist(self, artist_id):
self.calls.append(("get_artist", artist_id))
return self.artist_details.get(str(artist_id))
def _track_result(track_id="track-1", name="Song One", artist="Artist One"):
return SimpleNamespace(
id=track_id,
name=name,
artists=[artist],
album="Album One",
duration_ms=123000,
track_number=1,
disc_number=1,
image_url="https://img.example/track.jpg",
)
def _track_details(source, track_id="track-1", name="Song One", artist_name="Artist One", artist_id="artist-1"):
return {
"id": track_id,
"name": name,
"track_number": 7,
"disc_number": 1,
"duration_ms": 210000,
"explicit": True,
"uri": f"{source}:track:{track_id}",
"artists": [{"name": artist_name, "id": artist_id}],
"album": {
"id": f"{source}-album-1",
"name": "Album One",
"release_date": "2024-01-01",
"album_type": "album",
"total_tracks": 10,
"images": [{"url": f"https://img.example/{source}-album.jpg"}],
"artists": [{"name": artist_name, "id": artist_id}],
},
}
def test_get_single_track_import_context_uses_primary_source_priority(monkeypatch):
deezer_client = FakeClient(
search_results=[_track_result(track_id="deezer-track-1")],
details={"deezer-track-1": _track_details("deezer", track_id="deezer-track-1", artist_name="Artist One", artist_id="deezer-artist-1")},
artist_details={"deezer-artist-1": {"id": "deezer-artist-1", "genres": ["electronic"]}},
)
spotify_client = FakeClient()
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
monkeypatch.setattr(
metadata_service,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source),
)
result = metadata_service.get_single_track_import_context("Song One", "Artist One")
assert result["success"] is True
assert result["source"] == "deezer"
assert result["source_priority"] == ["deezer", "spotify", "itunes"]
assert result["context"]["track_info"]["name"] == "Song One"
assert result["context"]["track_info"]["album_id"] == "deezer-album-1"
assert result["context"]["album"]["image_url"] == "https://img.example/deezer-album.jpg"
assert result["context"]["artist"]["genres"] == ["electronic"]
assert result["context"]["original_search_result"]["clean_title"] == "Song One"
assert deezer_client.calls == [
('search_tracks', 'artist:"Artist One" track:"Song One"', 5, True),
("get_track_details", "deezer-track-1"),
("get_artist", "deezer-artist-1"),
]
assert spotify_client.calls == []
def test_get_single_track_import_context_falls_back_to_next_source(monkeypatch):
deezer_client = FakeClient(search_results=[])
spotify_client = FakeClient(
search_results=[_track_result(track_id="spotify-track-1")],
details={"spotify-track-1": _track_details("spotify", track_id="spotify-track-1", artist_name="Artist Two", artist_id="spotify-artist-1")},
artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["indie"]}},
)
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
monkeypatch.setattr(
metadata_service,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source),
)
result = metadata_service.get_single_track_import_context("Song Two", "Artist Two")
assert result["success"] is True
assert result["source"] == "spotify"
assert result["context"]["track_info"]["id"] == "spotify-track-1"
assert result["context"]["album"]["name"] == "Album One"
assert deezer_client.calls == [
('search_tracks', 'artist:"Artist Two" track:"Song Two"', 5, True),
('search_tracks', 'Song Two', 5, True),
('search_tracks', 'Artist Two', 5, True),
]
assert spotify_client.calls == [
("search_tracks", "Song Two Artist Two", 5, False),
("get_track_details", "spotify-track-1"),
("get_artist", "spotify-artist-1"),
]
def test_get_single_track_import_context_uses_explicit_override_first(monkeypatch):
spotify_client = FakeClient(
details={"override-track-1": _track_details("spotify", track_id="override-track-1", artist_name="Override Artist", artist_id="spotify-artist-1")},
artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["pop"]}},
)
deezer_client = FakeClient()
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
monkeypatch.setattr(
metadata_service,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source),
)
result = metadata_service.get_single_track_import_context(
"Ignored Title",
"Ignored Artist",
override_id="override-track-1",
)
assert result["success"] is True
assert result["source"] == "spotify"
assert result["context"]["track_info"]["id"] == "override-track-1"
assert result["context"]["artist"]["genres"] == ["pop"]
assert spotify_client.calls == [
("get_track_details", "override-track-1"),
("get_artist", "spotify-artist-1"),
]
assert deezer_client.calls == []
def test_get_single_track_import_context_uses_explicit_override_source(monkeypatch):
itunes_client = FakeClient(
details={"override-track-2": _track_details("itunes", track_id="override-track-2", artist_name="Override Artist Two", artist_id="itunes-artist-1")},
artist_details={"itunes-artist-1": {"id": "itunes-artist-1", "genres": ["singer-songwriter"]}},
)
spotify_client = FakeClient()
deezer_client = FakeClient()
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
monkeypatch.setattr(
metadata_service,
"get_client_for_source",
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": itunes_client}.get(source),
)
result = metadata_service.get_single_track_import_context(
"Ignored Title",
"Ignored Artist",
override_id="override-track-2",
override_source="itunes",
)
assert result["success"] is True
assert result["source"] == "itunes"
assert result["context"]["track_info"]["id"] == "override-track-2"
assert result["context"]["artist"]["genres"] == ["singer-songwriter"]
assert itunes_client.calls == [
("get_track_details", "override-track-2"),
("get_artist", "itunes-artist-1"),
]
assert deezer_client.calls == []
assert spotify_client.calls == []

View file

@ -2,10 +2,8 @@ import sys
import types
from datetime import datetime, timedelta
_RECENT_RELEASE_DATE = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
if "spotipy" not in sys.modules:
spotipy = types.ModuleType("spotipy")
@ -1045,6 +1043,10 @@ def test_update_discovery_pool_incremental_uses_source_priority(monkeypatch):
assert scanner.database.discovery_pool_calls
assert scanner.database.discovery_pool_calls[0][1] == "deezer"
assert scanner.database.discovery_pool_calls[0][0]["deezer_track_id"] == "dz-track-1"
assert scanner.database.discovery_pool_calls[0][0]["deezer_album_id"] == "dz-release-1"
assert scanner.database.discovery_pool_calls[0][0]["deezer_artist_id"] == "dz-artist"
assert scanner.database.discovery_pool_calls[0][0].get("spotify_track_id") is None
assert deezer_client.search_calls == [("Incremental Artist", 1, {})]
assert deezer_client.album_calls
assert spotify_client.search_calls == []

File diff suppressed because it is too large Load diff