Merge pull request #378 from kettui/refactor/extract-import-pipelines
Refactor import and related post-processing pipelines
This commit is contained in:
commit
f7b01f476a
53 changed files with 9559 additions and 5173 deletions
|
|
@ -5,6 +5,7 @@ Download management endpoints — list, cancel active downloads.
|
|||
from flask import request, current_app
|
||||
from .auth import require_api_key
|
||||
from .helpers import api_success, api_error
|
||||
from core.runtime_state import download_tasks, tasks_lock
|
||||
|
||||
|
||||
def _serialize_download(task_id, task):
|
||||
|
|
@ -53,8 +54,6 @@ def register_routes(bp):
|
|||
descending so newest/in-flight tasks appear first.
|
||||
"""
|
||||
try:
|
||||
from web_server import download_tasks, tasks_lock
|
||||
|
||||
# Parse pagination params
|
||||
try:
|
||||
limit = int(request.args.get("limit", 100))
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ def register_routes(bp):
|
|||
def system_activity():
|
||||
"""Recent activity feed."""
|
||||
try:
|
||||
from web_server import activity_feed
|
||||
from core.runtime_state import activity_feed
|
||||
items = list(activity_feed) if activity_feed else []
|
||||
return api_success({"activities": items})
|
||||
except Exception as e:
|
||||
|
|
@ -74,7 +74,7 @@ def register_routes(bp):
|
|||
# Active download count
|
||||
download_count = 0
|
||||
try:
|
||||
from web_server import download_tasks, tasks_lock
|
||||
from core.runtime_state import download_tasks, tasks_lock
|
||||
with tasks_lock:
|
||||
download_count = sum(
|
||||
1 for t in download_tasks.values()
|
||||
|
|
|
|||
1
core/imports/__init__.py
Normal file
1
core/imports/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
"""Import flow helpers package."""
|
||||
478
core/imports/album.py
Normal file
478
core/imports/album.py
Normal file
|
|
@ -0,0 +1,478 @@
|
|||
"""Album import helpers for staging matching and post-processing context."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, Iterable, List, Optional, Set
|
||||
|
||||
from core.imports.context import normalize_import_context
|
||||
from core.imports.staging import collect_staging_files
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("imports.album")
|
||||
|
||||
def get_client_for_source(source: str):
|
||||
from core.metadata_service import get_client_for_source as _get_client_for_source
|
||||
|
||||
return _get_client_for_source(source)
|
||||
|
||||
|
||||
def get_artist_album_tracks(
|
||||
album_id: str,
|
||||
artist_name: str = "",
|
||||
album_name: str = "",
|
||||
source: Optional[str] = None,
|
||||
):
|
||||
from core.metadata_service import get_artist_album_tracks as _get_artist_album_tracks
|
||||
|
||||
return _get_artist_album_tracks(
|
||||
album_id,
|
||||
artist_name=artist_name,
|
||||
album_name=album_name,
|
||||
source_override=source,
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
from core.matching_engine import MusicMatchingEngine
|
||||
_MATCHING_ENGINE_IMPORT_ERROR = None
|
||||
except Exception as exc: # pragma: no cover - only hits in stripped-down environments
|
||||
MusicMatchingEngine = None # type: ignore[assignment]
|
||||
_MATCHING_ENGINE_IMPORT_ERROR = exc
|
||||
|
||||
|
||||
_MATCHING_ENGINE = None
|
||||
|
||||
|
||||
def _get_matching_engine() -> Any:
|
||||
global _MATCHING_ENGINE
|
||||
if _MATCHING_ENGINE is None:
|
||||
if MusicMatchingEngine is None:
|
||||
raise RuntimeError("Music matching engine is unavailable") from _MATCHING_ENGINE_IMPORT_ERROR
|
||||
_MATCHING_ENGINE = MusicMatchingEngine()
|
||||
return _MATCHING_ENGINE
|
||||
|
||||
|
||||
def _normalize_artist_entries(artists: Any) -> List[Dict[str, Any]]:
|
||||
if not artists:
|
||||
return []
|
||||
|
||||
if isinstance(artists, (str, bytes)):
|
||||
artists = [artists]
|
||||
elif isinstance(artists, dict):
|
||||
artists = [artists]
|
||||
else:
|
||||
try:
|
||||
artists = list(artists)
|
||||
except TypeError:
|
||||
artists = [artists]
|
||||
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
for artist in artists:
|
||||
if isinstance(artist, dict):
|
||||
entry: Dict[str, Any] = {}
|
||||
name = artist.get("name") or artist.get("artist_name") or artist.get("title") or ""
|
||||
artist_id = artist.get("id") or artist.get("artist_id") or ""
|
||||
if name:
|
||||
entry["name"] = str(name)
|
||||
if artist_id:
|
||||
entry["id"] = str(artist_id)
|
||||
genres = artist.get("genres")
|
||||
if genres is not None:
|
||||
entry["genres"] = genres
|
||||
if entry:
|
||||
normalized.append(entry)
|
||||
continue
|
||||
|
||||
name = str(artist).strip()
|
||||
if name:
|
||||
normalized.append({"name": name})
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _normalize_album_source(album: Dict[str, Any], source: str = "") -> str:
|
||||
album_source = source or album.get("source") or ""
|
||||
return str(album_source).strip().lower()
|
||||
|
||||
|
||||
def _strip_legacy_source_fields(payload: Any) -> Any:
|
||||
if not isinstance(payload, dict):
|
||||
return payload
|
||||
|
||||
cleaned = dict(payload)
|
||||
cleaned.pop("_source", None)
|
||||
cleaned.pop("provider", None)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _extract_track_artist_name(track: Dict[str, Any]) -> str:
|
||||
artists = track.get("artists") or []
|
||||
if isinstance(artists, (str, bytes)):
|
||||
artists = [artists]
|
||||
elif isinstance(artists, dict):
|
||||
artists = [artists]
|
||||
else:
|
||||
try:
|
||||
artists = list(artists)
|
||||
except TypeError:
|
||||
artists = [artists]
|
||||
|
||||
if not artists:
|
||||
return ""
|
||||
|
||||
first = artists[0]
|
||||
if isinstance(first, dict):
|
||||
return str(first.get("name") or first.get("artist_name") or first.get("title") or "").strip()
|
||||
return str(first or "").strip()
|
||||
|
||||
|
||||
def _coerce_track_int(value: Any, default: int = 1) -> int:
|
||||
if value in (None, ""):
|
||||
return default
|
||||
try:
|
||||
return int(str(value).split("/")[0].strip() or default)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _normalize_match_track(track: Dict[str, Any], source: str, album: Dict[str, Any]) -> Dict[str, Any]:
|
||||
track_album = track.get("album") if isinstance(track.get("album"), dict) else album
|
||||
if isinstance(track_album, dict):
|
||||
track_album = _strip_legacy_source_fields(track_album)
|
||||
track_source = _normalize_album_source(track, source)
|
||||
track_artists = _normalize_artist_entries(track.get("artists") or [])
|
||||
|
||||
if not track_artists and album.get("artists"):
|
||||
track_artists = _normalize_artist_entries(album.get("artists"))
|
||||
|
||||
return {
|
||||
"id": track.get("id", ""),
|
||||
"name": track.get("name", "Unknown Track"),
|
||||
"track_number": _coerce_track_int(track.get("track_number", 1), default=1),
|
||||
"disc_number": _coerce_track_int(track.get("disc_number", 1), default=1),
|
||||
"duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0),
|
||||
"artists": track_artists,
|
||||
"uri": track.get("uri", ""),
|
||||
"album": track_album,
|
||||
"source": track_source,
|
||||
}
|
||||
|
||||
|
||||
def _score_album_track_match(track: Dict[str, Any], staging_file: Dict[str, Any], album_name: str) -> float:
|
||||
engine = _get_matching_engine()
|
||||
|
||||
track_name = track.get("name", "")
|
||||
staging_title = staging_file.get("title", "")
|
||||
score = 0.0
|
||||
|
||||
title_sim = engine.similarity_score(
|
||||
engine.normalize_string(track_name),
|
||||
engine.normalize_string(staging_title or ""),
|
||||
)
|
||||
score += title_sim * 0.45
|
||||
|
||||
track_artist_name = _extract_track_artist_name(track)
|
||||
staging_artist = staging_file.get("artist") or ""
|
||||
if track_artist_name and staging_artist:
|
||||
artist_sim = engine.similarity_score(
|
||||
engine.normalize_string(track_artist_name),
|
||||
engine.normalize_string(staging_artist),
|
||||
)
|
||||
score += artist_sim * 0.15
|
||||
else:
|
||||
score += 0.075
|
||||
|
||||
track_number = _coerce_track_int(track.get("track_number", 1), default=1)
|
||||
staging_track_number = _coerce_track_int(staging_file.get("track_number", 1), default=1)
|
||||
if staging_track_number and track_number:
|
||||
if staging_track_number == track_number:
|
||||
score += 0.30
|
||||
elif abs(staging_track_number - track_number) <= 1:
|
||||
score += 0.12
|
||||
|
||||
staging_album = staging_file.get("album") or ""
|
||||
if staging_album and album_name:
|
||||
album_sim = engine.similarity_score(
|
||||
engine.normalize_string(staging_album),
|
||||
engine.normalize_string(album_name),
|
||||
)
|
||||
score += album_sim * 0.10
|
||||
|
||||
return score
|
||||
|
||||
|
||||
def _fetch_artist_data_for_source(client: Any, artist_id: str, source: str) -> Any:
|
||||
if source == "spotify":
|
||||
try:
|
||||
return client.get_artist(artist_id, allow_fallback=False)
|
||||
except TypeError:
|
||||
return client.get_artist(artist_id)
|
||||
return client.get_artist(artist_id)
|
||||
|
||||
|
||||
def resolve_album_artist_context(album: Dict[str, Any], source: str = "") -> Dict[str, Any]:
|
||||
"""Build a neutral artist context for album import processing."""
|
||||
album = dict(album or {})
|
||||
source = _normalize_album_source(album, source)
|
||||
|
||||
artists = _normalize_artist_entries(album.get("artists") or [])
|
||||
if not artists:
|
||||
artist_name = album.get("artist") or album.get("artist_name") or ""
|
||||
artist_id = album.get("artist_id") or ""
|
||||
if artist_name or artist_id:
|
||||
artist_entry: Dict[str, Any] = {}
|
||||
if artist_name:
|
||||
artist_entry["name"] = str(artist_name)
|
||||
if artist_id:
|
||||
artist_entry["id"] = str(artist_id)
|
||||
artists = [artist_entry]
|
||||
|
||||
primary_artist = artists[0] if artists else {}
|
||||
artist_name = str(
|
||||
primary_artist.get("name")
|
||||
or album.get("artist")
|
||||
or album.get("artist_name")
|
||||
or "Unknown Artist"
|
||||
).strip()
|
||||
artist_id = str(primary_artist.get("id") or album.get("artist_id") or "").strip()
|
||||
|
||||
genres: List[Any] = []
|
||||
if artist_id and source:
|
||||
client = get_client_for_source(source)
|
||||
if client and hasattr(client, "get_artist"):
|
||||
try:
|
||||
artist_data = _fetch_artist_data_for_source(client, artist_id, source)
|
||||
raw_genres = artist_data.get("genres") if isinstance(artist_data, dict) else getattr(artist_data, "genres", [])
|
||||
if isinstance(raw_genres, str):
|
||||
genres = [raw_genres]
|
||||
elif raw_genres:
|
||||
try:
|
||||
genres = list(raw_genres)
|
||||
except TypeError:
|
||||
genres = [raw_genres]
|
||||
except Exception as exc:
|
||||
logger.debug("Could not resolve artist genres for %s on %s: %s", artist_id, source, exc)
|
||||
|
||||
return {
|
||||
"id": artist_id,
|
||||
"name": artist_name,
|
||||
"genres": genres,
|
||||
"source": source,
|
||||
}
|
||||
|
||||
|
||||
def build_album_import_context(
|
||||
album: Dict[str, Any],
|
||||
track: Dict[str, Any],
|
||||
*,
|
||||
artist_context: Optional[Dict[str, Any]] = None,
|
||||
total_discs: int = 1,
|
||||
source: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
"""Build a neutral post-processing context for one album track."""
|
||||
album = dict(album or {})
|
||||
track = dict(track or {})
|
||||
source = _normalize_album_source(album, source)
|
||||
|
||||
album_artists = _normalize_artist_entries(album.get("artists") or [])
|
||||
if not album_artists and artist_context:
|
||||
album_artists = _normalize_artist_entries([artist_context])
|
||||
|
||||
if artist_context:
|
||||
artist_ctx = dict(artist_context)
|
||||
else:
|
||||
artist_ctx = resolve_album_artist_context(album, source)
|
||||
|
||||
artist_ctx = _strip_legacy_source_fields(artist_ctx)
|
||||
artist_ctx.setdefault("genres", [])
|
||||
artist_ctx.setdefault("source", source)
|
||||
artist_ctx["genres"] = artist_ctx.get("genres") or []
|
||||
|
||||
track_artists = _normalize_artist_entries(track.get("artists") or [])
|
||||
if not track_artists:
|
||||
track_artists = album_artists or [artist_ctx]
|
||||
|
||||
track_album_value = track.get("album")
|
||||
if isinstance(track_album_value, dict):
|
||||
track_album_name = (
|
||||
track_album_value.get("name")
|
||||
or track_album_value.get("title")
|
||||
or album.get("name")
|
||||
or album.get("album_name")
|
||||
or ""
|
||||
)
|
||||
track_album_id = str(track_album_value.get("id") or track_album_value.get("album_id") or "").strip()
|
||||
track_album_type = track_album_value.get("album_type") or album.get("album_type") or "album"
|
||||
track_album_release = track_album_value.get("release_date") or album.get("release_date") or ""
|
||||
track_album_image = track_album_value.get("image_url") or album.get("image_url") or ""
|
||||
else:
|
||||
track_album_name = str(track_album_value or album.get("name") or album.get("album_name") or "").strip()
|
||||
track_album_id = str(album.get("id") or album.get("album_id") or "").strip()
|
||||
track_album_type = album.get("album_type") or "album"
|
||||
track_album_release = album.get("release_date") or ""
|
||||
track_album_image = album.get("image_url") or ""
|
||||
|
||||
album_name = str(album.get("name") or album.get("album_name") or track_album_name or "Unknown Album").strip()
|
||||
artist_name = str(
|
||||
artist_ctx.get("name")
|
||||
or album.get("artist")
|
||||
or album.get("artist_name")
|
||||
or "Unknown Artist"
|
||||
).strip()
|
||||
|
||||
track_number = _coerce_track_int(track.get("track_number", 1), default=1)
|
||||
disc_number = _coerce_track_int(track.get("disc_number", 1), default=1)
|
||||
|
||||
normalized_track = {
|
||||
"id": str(track.get("id") or track.get("track_id") or "").strip(),
|
||||
"name": str(track.get("name") or "Unknown Track").strip(),
|
||||
"track_number": track_number,
|
||||
"disc_number": disc_number,
|
||||
"duration_ms": _coerce_track_int(track.get("duration_ms", 0), default=0),
|
||||
"artists": track_artists,
|
||||
"uri": str(track.get("uri") or "").strip(),
|
||||
"album": track_album_name,
|
||||
"album_id": track_album_id,
|
||||
"album_type": track_album_type,
|
||||
"release_date": track_album_release,
|
||||
"source": source,
|
||||
}
|
||||
|
||||
normalized_album = {
|
||||
"id": str(album.get("id") or album.get("album_id") or track_album_id or "").strip(),
|
||||
"name": album_name,
|
||||
"artist": artist_name,
|
||||
"artist_name": artist_name,
|
||||
"artist_id": str(artist_ctx.get("id") or album.get("artist_id") or "").strip(),
|
||||
"artists": album_artists,
|
||||
"release_date": str(album.get("release_date") or track_album_release or "").strip(),
|
||||
"total_tracks": int(album.get("total_tracks") or track.get("total_tracks") or 0) or 1,
|
||||
"total_discs": int(total_discs or 1) if str(total_discs or 1).isdigit() else total_discs or 1,
|
||||
"album_type": str(album.get("album_type") or track_album_type or "album").strip() or "album",
|
||||
"image_url": str(album.get("image_url") or track_album_image or "").strip(),
|
||||
"images": album.get("images") or ([] if not track_album_image else [{"url": track_album_image}]),
|
||||
"source": source,
|
||||
}
|
||||
|
||||
original_search = {
|
||||
"title": normalized_track["name"],
|
||||
"artist": artist_name,
|
||||
"album": album_name,
|
||||
"track_number": track_number,
|
||||
"disc_number": disc_number,
|
||||
"clean_title": normalized_track["name"],
|
||||
"clean_album": album_name,
|
||||
"clean_artist": artist_name,
|
||||
"artists": track_artists,
|
||||
"duration_ms": normalized_track["duration_ms"],
|
||||
"id": normalized_track["id"],
|
||||
"source": source,
|
||||
}
|
||||
|
||||
context = {
|
||||
"artist": artist_ctx,
|
||||
"album": normalized_album,
|
||||
"track_info": normalized_track,
|
||||
"original_search_result": original_search,
|
||||
"is_album_download": True,
|
||||
"has_clean_metadata": bool(normalized_track["id"]),
|
||||
"has_full_metadata": bool(normalized_track["id"]),
|
||||
"source": source,
|
||||
}
|
||||
|
||||
normalized_context = normalize_import_context(context)
|
||||
normalized_context["artist"] = _strip_legacy_source_fields(normalized_context.get("artist"))
|
||||
normalized_context["album"] = _strip_legacy_source_fields(normalized_context.get("album"))
|
||||
normalized_context["track_info"] = _strip_legacy_source_fields(normalized_context.get("track_info"))
|
||||
normalized_context["original_search_result"] = _strip_legacy_source_fields(normalized_context.get("original_search_result"))
|
||||
return normalized_context
|
||||
|
||||
|
||||
def build_album_import_match_payload(
|
||||
album_id: str,
|
||||
*,
|
||||
album_name: str = "",
|
||||
album_artist: str = "",
|
||||
file_paths: Optional[Iterable[str]] = None,
|
||||
source: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build the album import match payload using provider-priority metadata lookup."""
|
||||
album_response = get_artist_album_tracks(
|
||||
album_id,
|
||||
artist_name=album_artist,
|
||||
album_name=album_name,
|
||||
source=source,
|
||||
)
|
||||
|
||||
album = _strip_legacy_source_fields(dict(album_response.get("album") or {}))
|
||||
source = _normalize_album_source(album, album_response.get("source") or source or "")
|
||||
tracks = list(album_response.get("tracks") or [])
|
||||
if not album_response.get("success") or not tracks:
|
||||
return {
|
||||
"success": False,
|
||||
"error": album_response.get("error", "Album not found"),
|
||||
"status_code": album_response.get("status_code", 404),
|
||||
"album": {
|
||||
"id": album_id,
|
||||
"name": album_name or album_id,
|
||||
"artist": album_artist or "Unknown Artist",
|
||||
"artist_name": album_artist or "Unknown Artist",
|
||||
"artist_id": "",
|
||||
"artists": [],
|
||||
"release_date": "",
|
||||
"total_tracks": 0,
|
||||
"total_discs": 1,
|
||||
"album_type": "album",
|
||||
"image_url": "",
|
||||
"images": [],
|
||||
"source": source,
|
||||
},
|
||||
"matches": [],
|
||||
"unmatched_files": [],
|
||||
"source": source,
|
||||
"source_priority": album_response.get("source_priority", []),
|
||||
"resolved_album_id": album_response.get("resolved_album_id") or album_id,
|
||||
}
|
||||
|
||||
staging_files = collect_staging_files(file_paths)
|
||||
album_name_for_match = album.get("name") or album_name or ""
|
||||
matches: List[Dict[str, Any]] = []
|
||||
used_files: Set[int] = set()
|
||||
|
||||
for track in tracks:
|
||||
normalized_track = _normalize_match_track(track, source, album)
|
||||
best_match = None
|
||||
best_score = 0.0
|
||||
|
||||
for index, staging_file in enumerate(staging_files):
|
||||
if index in used_files:
|
||||
continue
|
||||
|
||||
score = _score_album_track_match(normalized_track, staging_file, album_name_for_match)
|
||||
if score > best_score and score >= 0.4:
|
||||
best_score = score
|
||||
best_match = index
|
||||
|
||||
matches.append(
|
||||
{
|
||||
"track": normalized_track,
|
||||
"staging_file": staging_files[best_match] if best_match is not None else None,
|
||||
"confidence": round(best_score, 2) if best_match is not None else 0,
|
||||
}
|
||||
)
|
||||
|
||||
if best_match is not None:
|
||||
used_files.add(best_match)
|
||||
|
||||
unmatched_files = [sf for index, sf in enumerate(staging_files) if index not in used_files]
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"album": album,
|
||||
"matches": matches,
|
||||
"unmatched_files": unmatched_files,
|
||||
"source": source,
|
||||
"source_priority": album_response.get("source_priority", []),
|
||||
"resolved_album_id": album_response.get("resolved_album_id") or album_id,
|
||||
}
|
||||
184
core/imports/album_naming.py
Normal file
184
core/imports/album_naming.py
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
"""Album naming and grouping helpers used by import flows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import threading
|
||||
from typing import Any, Dict
|
||||
|
||||
from core.imports.context import extract_artist_name
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("imports.album_naming")
|
||||
|
||||
_album_cache_lock = threading.Lock()
|
||||
_album_editions: dict[str, str] = {}
|
||||
_album_name_cache: dict[str, str] = {}
|
||||
|
||||
|
||||
def clear_album_grouping_cache() -> None:
|
||||
"""Clear cached album grouping decisions.
|
||||
|
||||
Useful for tests and for any future config reload flows.
|
||||
"""
|
||||
with _album_cache_lock:
|
||||
_album_editions.clear()
|
||||
_album_name_cache.clear()
|
||||
|
||||
|
||||
def get_base_album_name(album_name: str) -> str:
|
||||
"""Extract the base album name without edition indicators."""
|
||||
base_name = album_name or ""
|
||||
base_name = re.sub(
|
||||
r"\s*[\[\(][^)\]]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^)\]]*[\]\)]\s*$",
|
||||
"",
|
||||
base_name,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
base_name = re.sub(r"\s*[\[\(][^)\]]*\bedition\b[^)\]]*[\]\)]\s*$", "", base_name, flags=re.IGNORECASE)
|
||||
base_name = re.sub(
|
||||
r"\s+(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$",
|
||||
"",
|
||||
base_name,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
return base_name.strip()
|
||||
|
||||
|
||||
def detect_deluxe_edition(album_name: str) -> bool:
|
||||
"""Detect if an album name indicates a deluxe/special edition."""
|
||||
if not album_name:
|
||||
return False
|
||||
|
||||
album_lower = album_name.lower()
|
||||
deluxe_indicators = [
|
||||
"deluxe",
|
||||
"deluxe edition",
|
||||
"special edition",
|
||||
"expanded edition",
|
||||
"extended edition",
|
||||
"bonus",
|
||||
"remastered",
|
||||
"anniversary",
|
||||
"collectors edition",
|
||||
"limited edition",
|
||||
"silver edition",
|
||||
"gold edition",
|
||||
"platinum edition",
|
||||
]
|
||||
for indicator in deluxe_indicators:
|
||||
if indicator in album_lower:
|
||||
logger.info("Detected deluxe edition: %r contains %r", album_name, indicator)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def normalize_base_album_name(base_album: str, artist_name: str) -> str:
|
||||
"""Normalize the base album name to handle case variations and known corrections."""
|
||||
normalized_lower = (base_album or "").lower().strip()
|
||||
known_corrections = {
|
||||
# Add specific album name corrections here as needed.
|
||||
}
|
||||
|
||||
for variant, correction in known_corrections.items():
|
||||
if normalized_lower == variant.lower():
|
||||
logger.info("Album correction applied: %r -> %r", base_album, correction)
|
||||
return correction
|
||||
|
||||
normalized = base_album or ""
|
||||
normalized = re.sub(r"\s*&\s*", " & ", normalized)
|
||||
normalized = re.sub(r"\s+", " ", normalized)
|
||||
normalized = normalized.strip()
|
||||
logger.info("Album variant normalization: %r -> %r", base_album, normalized)
|
||||
return normalized
|
||||
|
||||
|
||||
def clean_album_title(album_title: str, artist_name: str) -> str:
|
||||
"""Clean up album title by removing common prefixes, suffixes, and artist redundancy."""
|
||||
original = (album_title or "").strip()
|
||||
cleaned = original
|
||||
logger.info("Album Title Cleaning: %r (artist: %r)", original, artist_name)
|
||||
|
||||
cleaned = re.sub(r"^Album\s*-\s*", "", cleaned, flags=re.IGNORECASE)
|
||||
artist_pattern = re.escape(artist_name or "") + r"\s*-\s*"
|
||||
cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r"\s*[\[\(]\d{4}[\]\)]\s*", " ", cleaned)
|
||||
|
||||
quality_patterns = [
|
||||
r"\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*",
|
||||
r"\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*",
|
||||
r"\s*[\[\(].*?flac.*?[\]\)]\s*",
|
||||
r"\s*[\[\(].*?mp3.*?[\]\)]\s*",
|
||||
r"\s*[\[\(].*?itunes.*?[\]\)]\s*",
|
||||
r"\s*[\[\(].*?web.*?[\]\)]\s*",
|
||||
r"\s*[\[\(].*?cd.*?[\]\)]\s*",
|
||||
]
|
||||
for pattern in quality_patterns:
|
||||
cleaned = re.sub(pattern, " ", cleaned, flags=re.IGNORECASE)
|
||||
|
||||
cleaned = re.sub(r"\s*[\[\(][^\]\)]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r"\s*[\[\(][^\]\)]*\bedition\b[^\]\)]*[\]\)]\s*", " ", cleaned, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r"\s*(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$", "", cleaned, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r"^[-\s\.]+", "", cleaned)
|
||||
cleaned = re.sub(r"[-\s\.]+$", "", cleaned)
|
||||
cleaned = re.sub(r"\s+", " ", cleaned).strip()
|
||||
return cleaned if cleaned else original
|
||||
|
||||
|
||||
def resolve_album_group(artist_context: dict, album_info: dict, original_album: str = None) -> str:
|
||||
"""Smart album grouping: upgrade to deluxe if any track is deluxe."""
|
||||
try:
|
||||
with _album_cache_lock:
|
||||
artist_name = extract_artist_name(artist_context)
|
||||
detected_album = (album_info or {}).get("album_name", "")
|
||||
|
||||
if detected_album:
|
||||
base_album = get_base_album_name(detected_album)
|
||||
elif original_album:
|
||||
cleaned_original = clean_album_title(original_album, artist_name)
|
||||
base_album = get_base_album_name(cleaned_original)
|
||||
else:
|
||||
base_album = get_base_album_name(detected_album)
|
||||
|
||||
base_album = normalize_base_album_name(base_album, artist_name)
|
||||
album_key = f"{artist_name}::{base_album}"
|
||||
is_deluxe_track = False
|
||||
if detected_album:
|
||||
is_deluxe_track = detect_deluxe_edition(detected_album)
|
||||
elif original_album:
|
||||
is_deluxe_track = detect_deluxe_edition(original_album)
|
||||
|
||||
if album_key in _album_name_cache:
|
||||
cached_name = _album_name_cache[album_key]
|
||||
current_edition = _album_editions.get(album_key, "standard")
|
||||
if is_deluxe_track and current_edition == "standard":
|
||||
final_album_name = f"{base_album} (Deluxe Edition)"
|
||||
_album_editions[album_key] = "deluxe"
|
||||
_album_name_cache[album_key] = final_album_name
|
||||
logger.info("Album cache upgrade: %r -> %r", album_key, final_album_name)
|
||||
return final_album_name
|
||||
logger.info("Using cached album name for %r: %r", album_key, cached_name)
|
||||
return cached_name
|
||||
|
||||
logger.info("Album grouping - Key: %r, Detected: %r", album_key, detected_album)
|
||||
|
||||
current_edition = _album_editions.get(album_key, "standard")
|
||||
if is_deluxe_track and current_edition == "standard":
|
||||
logger.info("UPGRADE: Album %r upgraded from standard to deluxe!", base_album)
|
||||
_album_editions[album_key] = "deluxe"
|
||||
current_edition = "deluxe"
|
||||
|
||||
if current_edition == "deluxe":
|
||||
final_album_name = f"{base_album} (Deluxe Edition)"
|
||||
else:
|
||||
final_album_name = base_album
|
||||
|
||||
_album_name_cache[album_key] = final_album_name
|
||||
|
||||
logger.info("Album resolution: %r -> %r (edition: %s)", detected_album, final_album_name, current_edition)
|
||||
return final_album_name
|
||||
except Exception as e:
|
||||
logger.error("Error resolving album group: %s", e)
|
||||
album_name = (album_info or {}).get("album_name", "Unknown Album")
|
||||
return album_name
|
||||
391
core/imports/context.py
Normal file
391
core/imports/context.py
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
"""Helpers for normalizing and reading import contexts.
|
||||
|
||||
These functions keep the single-import pipeline source-agnostic while still
|
||||
accepting legacy `spotify_*` payloads from older callers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
def _as_dict(value: Any) -> Dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _first_value(mapping: Dict[str, Any], *keys: str, default: Any = "") -> Any:
|
||||
for key in keys:
|
||||
if key in mapping:
|
||||
value = mapping.get(key)
|
||||
if value not in (None, ""):
|
||||
return value
|
||||
return default
|
||||
|
||||
|
||||
def _first_id_value(*values: Any) -> str:
|
||||
for value in values:
|
||||
if value in (None, ""):
|
||||
continue
|
||||
text = str(value).strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def extract_artist_name(artist: Any) -> str:
|
||||
if isinstance(artist, dict):
|
||||
return str(artist.get("name", "") or "")
|
||||
if hasattr(artist, "name"):
|
||||
return str(getattr(artist, "name") or "")
|
||||
return str(artist) if artist else ""
|
||||
|
||||
|
||||
def normalize_import_context(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""Normalize an import context to neutral fields in place and drop legacy aliases."""
|
||||
if not isinstance(context, dict):
|
||||
return {}
|
||||
|
||||
source = context.get("source") or context.get("_source") or ""
|
||||
artist = _as_dict(context.get("artist") or context.get("spotify_artist"))
|
||||
album = _as_dict(context.get("album") or context.get("spotify_album"))
|
||||
track_info = _as_dict(context.get("track_info"))
|
||||
original_search = _as_dict(context.get("original_search_result"))
|
||||
search_result = _as_dict(context.get("search_result"))
|
||||
normalized_search = original_search or search_result
|
||||
|
||||
if source:
|
||||
context["source"] = source
|
||||
context["artist"] = artist
|
||||
context["album"] = album
|
||||
context["track_info"] = track_info
|
||||
context["original_search_result"] = normalized_search
|
||||
context.pop("_source", None)
|
||||
context.pop("spotify_artist", None)
|
||||
context.pop("spotify_album", None)
|
||||
|
||||
for clean_key, legacy_key in (
|
||||
("clean_title", "spotify_clean_title"),
|
||||
("clean_album", "spotify_clean_album"),
|
||||
("clean_artist", "spotify_clean_artist"),
|
||||
):
|
||||
if clean_key not in normalized_search or normalized_search.get(clean_key) in (None, ""):
|
||||
legacy_value = normalized_search.get(legacy_key)
|
||||
if legacy_value not in (None, ""):
|
||||
normalized_search[clean_key] = legacy_value
|
||||
normalized_search.pop(legacy_key, None)
|
||||
|
||||
has_clean = bool(context.get("has_clean_metadata", context.get("has_clean_spotify_data", False)))
|
||||
has_full = bool(context.get("has_full_metadata", context.get("has_full_spotify_metadata", False)))
|
||||
context["has_clean_metadata"] = has_clean
|
||||
context["has_full_metadata"] = has_full
|
||||
context.pop("has_clean_spotify_data", None)
|
||||
context.pop("has_full_spotify_metadata", None)
|
||||
|
||||
return context
|
||||
|
||||
|
||||
def get_import_context_artist(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
if not isinstance(context, dict):
|
||||
return {}
|
||||
return _as_dict(context.get("artist") or context.get("spotify_artist"))
|
||||
|
||||
|
||||
def get_import_context_album(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
if not isinstance(context, dict):
|
||||
return {}
|
||||
return _as_dict(context.get("album") or context.get("spotify_album"))
|
||||
|
||||
|
||||
def get_import_track_info(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
if not isinstance(context, dict):
|
||||
return {}
|
||||
return _as_dict(context.get("track_info"))
|
||||
|
||||
|
||||
def get_import_original_search(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
if not isinstance(context, dict):
|
||||
return {}
|
||||
return _as_dict(context.get("original_search_result"))
|
||||
|
||||
|
||||
def get_import_search_result(context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
if not isinstance(context, dict):
|
||||
return {}
|
||||
return _as_dict(context.get("search_result"))
|
||||
|
||||
|
||||
def get_import_source(context: Optional[Dict[str, Any]]) -> str:
|
||||
if not isinstance(context, dict):
|
||||
return ""
|
||||
|
||||
source = context.get("source")
|
||||
if source:
|
||||
return str(source)
|
||||
|
||||
track_info = get_import_track_info(context)
|
||||
source = _first_value(track_info, "source", default="")
|
||||
if source:
|
||||
return str(source)
|
||||
|
||||
original_search = get_import_original_search(context)
|
||||
source = _first_value(original_search, "source", default="")
|
||||
if source:
|
||||
return str(source)
|
||||
|
||||
album = get_import_context_album(context)
|
||||
source = _first_value(album, "source", default="")
|
||||
if source:
|
||||
return str(source)
|
||||
|
||||
artist = get_import_context_artist(context)
|
||||
source = _first_value(artist, "source", default="")
|
||||
return str(source) if source else ""
|
||||
|
||||
|
||||
def get_import_clean_title(
|
||||
context: Optional[Dict[str, Any]],
|
||||
album_info: Optional[Dict[str, Any]] = None,
|
||||
default: str = "Unknown Track",
|
||||
) -> str:
|
||||
original_search = get_import_original_search(context)
|
||||
title = _first_value(
|
||||
original_search,
|
||||
"clean_title",
|
||||
"title",
|
||||
default="",
|
||||
)
|
||||
if not title and album_info:
|
||||
title = _first_value(album_info, "clean_track_name", "track_name", default="")
|
||||
if not title:
|
||||
track_info = get_import_track_info(context)
|
||||
title = _first_value(track_info, "name", "title", default="")
|
||||
return str(title or default)
|
||||
|
||||
|
||||
def get_import_clean_album(
|
||||
context: Optional[Dict[str, Any]],
|
||||
album_info: Optional[Dict[str, Any]] = None,
|
||||
default: str = "Unknown Album",
|
||||
) -> str:
|
||||
original_search = get_import_original_search(context)
|
||||
album = _first_value(
|
||||
original_search,
|
||||
"clean_album",
|
||||
"album",
|
||||
default="",
|
||||
)
|
||||
if not album and album_info:
|
||||
album = _first_value(album_info, "album_name", "clean_album_name", default="")
|
||||
if not album:
|
||||
album_ctx = get_import_context_album(context)
|
||||
album = _first_value(album_ctx, "name", default="")
|
||||
return str(album or default)
|
||||
|
||||
|
||||
def get_import_clean_artist(context: Optional[Dict[str, Any]], default: str = "Unknown Artist") -> str:
|
||||
original_search = get_import_original_search(context)
|
||||
artist = _first_value(
|
||||
original_search,
|
||||
"clean_artist",
|
||||
"artist",
|
||||
default="",
|
||||
)
|
||||
if not artist:
|
||||
artist_ctx = get_import_context_artist(context)
|
||||
artist = _first_value(artist_ctx, "name", default="")
|
||||
return str(artist or default)
|
||||
|
||||
|
||||
def get_import_has_clean_metadata(context: Optional[Dict[str, Any]]) -> bool:
|
||||
if not isinstance(context, dict):
|
||||
return False
|
||||
return bool(context.get("has_clean_metadata", False))
|
||||
|
||||
|
||||
def get_import_has_full_metadata(context: Optional[Dict[str, Any]]) -> bool:
|
||||
if not isinstance(context, dict):
|
||||
return False
|
||||
return bool(context.get("has_full_metadata", False))
|
||||
|
||||
|
||||
def get_import_source_ids(context: Optional[Dict[str, Any]]) -> Dict[str, str]:
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
search_result = get_import_search_result(context)
|
||||
artist = get_import_context_artist(context)
|
||||
album = get_import_context_album(context)
|
||||
|
||||
return {
|
||||
"track_id": _first_id_value(
|
||||
_first_value(track_info, "id", "track_id", "trackId", "source_track_id", default=""),
|
||||
_first_value(track_info, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""),
|
||||
_first_value(original_search, "id", "track_id", "source_track_id", default=""),
|
||||
_first_value(original_search, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""),
|
||||
_first_value(search_result, "id", "track_id", "source_track_id", default=""),
|
||||
_first_value(search_result, "spotify_track_id", "itunes_track_id", "deezer_id", "deezer_track_id", "discogs_id", "soul_id", default=""),
|
||||
),
|
||||
"artist_id": _first_id_value(
|
||||
_first_value(artist, "id", "artist_id", "source_artist_id", default=""),
|
||||
_first_value(artist, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""),
|
||||
_first_value(original_search, "artist_id", "source_artist_id", default=""),
|
||||
_first_value(original_search, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""),
|
||||
_first_value(search_result, "artist_id", "source_artist_id", default=""),
|
||||
_first_value(search_result, "spotify_artist_id", "itunes_artist_id", "deezer_id", "deezer_artist_id", "discogs_id", "soul_id", default=""),
|
||||
),
|
||||
"album_id": _first_id_value(
|
||||
_first_value(album, "id", "album_id", "collectionId", "source_album_id", default=""),
|
||||
_first_value(album, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""),
|
||||
_first_value(original_search, "album_id", "source_album_id", default=""),
|
||||
_first_value(original_search, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""),
|
||||
_first_value(track_info, "album_id", "source_album_id", default=""),
|
||||
_first_value(track_info, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""),
|
||||
_first_value(search_result, "album_id", "source_album_id", default=""),
|
||||
_first_value(search_result, "spotify_album_id", "itunes_album_id", "deezer_id", "deezer_album_id", "discogs_id", "soul_id", "album_soul_id", "hydrabase_album_id", default=""),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_source_tag_names(source: str) -> Dict[str, Optional[str]]:
|
||||
source_name = (source or "").strip().lower()
|
||||
if source_name == "spotify":
|
||||
return {"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"}
|
||||
if source_name == "itunes":
|
||||
return {"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"}
|
||||
if source_name == "deezer":
|
||||
return {"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None}
|
||||
if source_name == "hydrabase":
|
||||
return {"track": None, "artist": None, "album": None}
|
||||
if source_name == "discogs":
|
||||
return {"track": None, "artist": None, "album": None}
|
||||
return {"track": None, "artist": None, "album": None}
|
||||
|
||||
|
||||
def get_library_source_id_columns(source: str) -> Dict[str, Optional[str]]:
|
||||
source_name = (source or "").strip().lower()
|
||||
if source_name == "spotify":
|
||||
return {"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"}
|
||||
if source_name == "itunes":
|
||||
return {"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"}
|
||||
if source_name == "deezer":
|
||||
return {"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"}
|
||||
if source_name == "hydrabase":
|
||||
return {"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"}
|
||||
if source_name == "discogs":
|
||||
return {"artist": "discogs_id", "album": "discogs_id", "track": None}
|
||||
return {}
|
||||
|
||||
|
||||
def build_import_album_info(
|
||||
context: Optional[Dict[str, Any]],
|
||||
*,
|
||||
album_info: Optional[Dict[str, Any]] = None,
|
||||
force_album: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build the album-info payload used by post-processing."""
|
||||
album_ctx = get_import_context_album(context)
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
artist_ctx = get_import_context_artist(context)
|
||||
|
||||
track_number = (
|
||||
(album_info or {}).get("track_number")
|
||||
or track_info.get("track_number")
|
||||
or original_search.get("track_number")
|
||||
or 1
|
||||
)
|
||||
disc_number = (
|
||||
(album_info or {}).get("disc_number")
|
||||
or track_info.get("disc_number")
|
||||
or original_search.get("disc_number")
|
||||
or 1
|
||||
)
|
||||
|
||||
clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track"))
|
||||
album_name = get_import_clean_album(context, album_info=album_info, default=original_search.get("album", "Unknown Album"))
|
||||
album_image_url = (
|
||||
(album_info or {}).get("album_image_url")
|
||||
or album_ctx.get("image_url")
|
||||
or ""
|
||||
)
|
||||
total_tracks = (
|
||||
album_ctx.get("total_tracks")
|
||||
or track_info.get("total_tracks")
|
||||
or (album_info or {}).get("total_tracks")
|
||||
or 0
|
||||
)
|
||||
album_type = (album_ctx.get("album_type") or track_info.get("album_type") or "album")
|
||||
source = get_import_source(context)
|
||||
|
||||
artist_name = artist_ctx.get("name") or original_search.get("artist") or get_import_clean_artist(context)
|
||||
normalized_album = str(album_name or "").strip().lower()
|
||||
normalized_title = str(clean_track_name or "").strip().lower()
|
||||
normalized_artist = str(artist_name or "").strip().lower()
|
||||
is_album = bool(
|
||||
force_album
|
||||
or (
|
||||
normalized_album
|
||||
and total_tracks
|
||||
and int(total_tracks) > 1
|
||||
and normalized_album != normalized_title
|
||||
and normalized_album != normalized_artist
|
||||
)
|
||||
)
|
||||
|
||||
return {
|
||||
"is_album": is_album,
|
||||
"album_name": album_name,
|
||||
"track_number": int(track_number) if str(track_number).isdigit() else track_number,
|
||||
"disc_number": int(disc_number) if str(disc_number).isdigit() else disc_number,
|
||||
"clean_track_name": clean_track_name,
|
||||
"album_image_url": album_image_url,
|
||||
"confidence": (album_info or {}).get("confidence", 1.0 if is_album or force_album else 0.0),
|
||||
"source": source,
|
||||
"album_type": album_type,
|
||||
"total_tracks": int(total_tracks) if str(total_tracks).isdigit() else total_tracks,
|
||||
}
|
||||
|
||||
|
||||
def detect_album_info_web(context, artist_context=None):
|
||||
"""Best-effort album detection for single-track downloads."""
|
||||
context = normalize_import_context(context)
|
||||
if artist_context is None:
|
||||
artist_context = context.get("artist") or {}
|
||||
|
||||
album_info = build_import_album_info(context)
|
||||
if album_info.get("is_album"):
|
||||
return album_info
|
||||
|
||||
album_ctx = get_import_context_album(context)
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
|
||||
album_name = (
|
||||
album_ctx.get("name")
|
||||
or track_info.get("album")
|
||||
or original_search.get("album")
|
||||
or ""
|
||||
)
|
||||
track_name = (
|
||||
track_info.get("name")
|
||||
or original_search.get("title")
|
||||
or ""
|
||||
)
|
||||
artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="")
|
||||
|
||||
if album_name and track_name and album_name.strip().lower() not in {
|
||||
track_name.strip().lower(),
|
||||
artist_name.strip().lower(),
|
||||
}:
|
||||
return build_import_album_info(
|
||||
context,
|
||||
album_info={
|
||||
"album_name": album_name,
|
||||
"track_number": track_info.get("track_number", 1),
|
||||
"disc_number": track_info.get("disc_number", 1),
|
||||
"album_image_url": album_ctx.get("image_url", ""),
|
||||
"confidence": 0.5,
|
||||
},
|
||||
force_album=True,
|
||||
)
|
||||
|
||||
return None
|
||||
368
core/imports/file_ops.py
Normal file
368
core/imports/file_ops.py
Normal file
|
|
@ -0,0 +1,368 @@
|
|||
"""File operation helpers for the import flow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from config.settings import config_manager
|
||||
|
||||
logger = logging.getLogger("imports.file_ops")
|
||||
|
||||
|
||||
def safe_move_file(src, dst):
|
||||
"""Move a file safely across filesystems."""
|
||||
src = Path(src)
|
||||
dst = Path(dst)
|
||||
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if not src.exists():
|
||||
if dst.exists():
|
||||
logger.info(f"Source gone but destination exists, file already transferred: {dst.name}")
|
||||
return
|
||||
raise FileNotFoundError(f"Source file not found and destination does not exist: {src}")
|
||||
|
||||
if dst.exists():
|
||||
for _attempt in range(3):
|
||||
try:
|
||||
dst.unlink()
|
||||
break
|
||||
except PermissionError:
|
||||
if _attempt < 2:
|
||||
time.sleep(1)
|
||||
else:
|
||||
logger.warning(f"Could not remove locked destination after 3 attempts: {dst.name}")
|
||||
except Exception:
|
||||
break
|
||||
|
||||
try:
|
||||
shutil.move(str(src), str(dst))
|
||||
return
|
||||
except FileNotFoundError:
|
||||
if dst.exists():
|
||||
logger.info(f"Source moved by another thread, destination exists: {dst.name}")
|
||||
return
|
||||
raise
|
||||
except (OSError, PermissionError) as e:
|
||||
error_msg = str(e).lower()
|
||||
|
||||
if dst.exists() and dst.stat().st_size > 0:
|
||||
logger.warning(f"Move raised {type(e).__name__} but destination exists, treating as success: {e}")
|
||||
try:
|
||||
src.unlink()
|
||||
except Exception:
|
||||
logger.info(f"Could not delete source file (may be owned by another process): {src}")
|
||||
return
|
||||
|
||||
if "cross-device" in error_msg or "operation not permitted" in error_msg or "permission denied" in error_msg:
|
||||
logger.warning(f"Cross-device move detected, using fallback copy method: {e}")
|
||||
try:
|
||||
with open(src, "rb") as f_src:
|
||||
with open(dst, "wb") as f_dst:
|
||||
shutil.copyfileobj(f_src, f_dst)
|
||||
f_dst.flush()
|
||||
os.fsync(f_dst.fileno())
|
||||
|
||||
try:
|
||||
src.unlink()
|
||||
except PermissionError:
|
||||
logger.info(f"Could not delete source file (may be owned by another process): {src}")
|
||||
logger.info(f"Successfully moved file using fallback method: {src} -> {dst}")
|
||||
return
|
||||
except Exception as fallback_error:
|
||||
logger.error(f"Fallback copy also failed: {fallback_error}")
|
||||
raise
|
||||
raise
|
||||
|
||||
|
||||
def cleanup_empty_directories(download_path, moved_file_path):
|
||||
"""Remove empty directories after a move, ignoring hidden files."""
|
||||
try:
|
||||
current_dir = os.path.dirname(moved_file_path)
|
||||
while current_dir != download_path and current_dir.startswith(download_path):
|
||||
is_empty = not any(not f.startswith(".") for f in os.listdir(current_dir))
|
||||
if is_empty:
|
||||
logger.warning(f"Removing empty directory: {current_dir}")
|
||||
os.rmdir(current_dir)
|
||||
current_dir = os.path.dirname(current_dir)
|
||||
else:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"An error occurred during directory cleanup: {e}")
|
||||
|
||||
|
||||
def get_audio_quality_string(file_path):
|
||||
"""Return a compact audio quality string for the given file."""
|
||||
try:
|
||||
ext = os.path.splitext(file_path)[1].lower()
|
||||
|
||||
if ext == ".flac":
|
||||
from mutagen.flac import FLAC
|
||||
audio = FLAC(file_path)
|
||||
return f"FLAC {audio.info.bits_per_sample}bit"
|
||||
|
||||
if ext == ".mp3":
|
||||
from mutagen.mp3 import MP3, BitrateMode
|
||||
|
||||
audio = MP3(file_path)
|
||||
bitrate_kbps = audio.info.bitrate // 1000
|
||||
if audio.info.bitrate_mode == BitrateMode.VBR:
|
||||
return "MP3-VBR"
|
||||
return f"MP3-{bitrate_kbps}"
|
||||
|
||||
if ext in (".m4a", ".aac", ".mp4"):
|
||||
from mutagen.mp4 import MP4
|
||||
audio = MP4(file_path)
|
||||
return f"M4A-{audio.info.bitrate // 1000}"
|
||||
|
||||
if ext == ".ogg":
|
||||
from mutagen.oggvorbis import OggVorbis
|
||||
audio = OggVorbis(file_path)
|
||||
return f"OGG-{audio.info.bitrate // 1000}"
|
||||
|
||||
if ext == ".opus":
|
||||
from mutagen.oggopus import OggOpus
|
||||
|
||||
audio = OggOpus(file_path)
|
||||
return f"OPUS-{audio.info.bitrate // 1000}"
|
||||
|
||||
return ""
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not determine audio quality for {file_path}: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
def get_quality_tier_from_extension(file_path):
|
||||
"""Classify a file extension into a quality tier."""
|
||||
if not file_path:
|
||||
return ("unknown", 999)
|
||||
|
||||
ext = os.path.splitext(file_path)[1].lower()
|
||||
quality_tiers = {
|
||||
"lossless": {
|
||||
"extensions": [".flac", ".ape", ".wav", ".alac", ".dsf", ".dff", ".aiff", ".aif"],
|
||||
"tier": 1,
|
||||
},
|
||||
"high_lossy": {
|
||||
"extensions": [".opus", ".ogg"],
|
||||
"tier": 2,
|
||||
},
|
||||
"standard_lossy": {
|
||||
"extensions": [".m4a", ".aac"],
|
||||
"tier": 3,
|
||||
},
|
||||
"low_lossy": {
|
||||
"extensions": [".mp3", ".wma"],
|
||||
"tier": 4,
|
||||
},
|
||||
}
|
||||
|
||||
for tier_name, tier_data in quality_tiers.items():
|
||||
if ext in tier_data["extensions"]:
|
||||
return (tier_name, tier_data["tier"])
|
||||
|
||||
return ("unknown", 999)
|
||||
|
||||
|
||||
def downsample_hires_flac(final_path, context):
|
||||
"""Downsample a hi-res FLAC to 16-bit/44.1kHz if enabled."""
|
||||
from mutagen.flac import FLAC
|
||||
|
||||
if not config_manager.get("lossy_copy.downsample_hires", False):
|
||||
return None
|
||||
|
||||
if os.path.splitext(final_path)[1].lower() != ".flac":
|
||||
return None
|
||||
|
||||
try:
|
||||
audio = FLAC(final_path)
|
||||
original_bits = audio.info.bits_per_sample
|
||||
original_rate = audio.info.sample_rate
|
||||
except Exception as e:
|
||||
logger.error(f"[Downsample] Could not read FLAC info: {e}")
|
||||
return None
|
||||
|
||||
if original_bits <= 16 and original_rate <= 44100:
|
||||
return None
|
||||
|
||||
logger.info(f"[Downsample] Converting {original_bits}-bit/{original_rate}Hz -> 16-bit/44100Hz: {os.path.basename(final_path)}")
|
||||
|
||||
ffmpeg_bin = shutil.which("ffmpeg")
|
||||
if not ffmpeg_bin:
|
||||
local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg")
|
||||
if os.path.isfile(local):
|
||||
ffmpeg_bin = local
|
||||
else:
|
||||
logger.warning("[Downsample] ffmpeg not found - skipping hi-res conversion")
|
||||
return None
|
||||
|
||||
temp_path = final_path + ".tmp.flac"
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
ffmpeg_bin, "-i", final_path,
|
||||
"-sample_fmt", "s16",
|
||||
"-ar", "44100",
|
||||
"-map_metadata", "0",
|
||||
"-compression_level", "8",
|
||||
"-y", temp_path,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300,
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.error(f"[Downsample] ffmpeg failed: {result.stderr[:200]}")
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
return None
|
||||
|
||||
if not os.path.isfile(temp_path) or os.path.getsize(temp_path) == 0:
|
||||
logger.warning("[Downsample] Output file missing or empty")
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
return None
|
||||
|
||||
verify_audio = FLAC(temp_path)
|
||||
if verify_audio.info.bits_per_sample != 16:
|
||||
logger.info(f"[Downsample] Output not 16-bit ({verify_audio.info.bits_per_sample}-bit), aborting")
|
||||
os.remove(temp_path)
|
||||
return None
|
||||
|
||||
os.replace(temp_path, final_path)
|
||||
logger.info(f"[Downsample] Converted to 16-bit/44.1kHz: {os.path.basename(final_path)}")
|
||||
|
||||
new_quality = "FLAC 16bit"
|
||||
try:
|
||||
updated_audio = FLAC(final_path)
|
||||
updated_audio["QUALITY"] = new_quality
|
||||
updated_audio.save()
|
||||
except Exception as tag_err:
|
||||
logger.error(f"[Downsample] Could not update QUALITY tag: {tag_err}")
|
||||
|
||||
old_quality = context.get("_audio_quality", "")
|
||||
context["_audio_quality"] = new_quality
|
||||
|
||||
if old_quality and old_quality != new_quality and old_quality in os.path.basename(final_path):
|
||||
new_basename = os.path.basename(final_path).replace(old_quality, new_quality)
|
||||
new_path = os.path.join(os.path.dirname(final_path), new_basename)
|
||||
try:
|
||||
os.rename(final_path, new_path)
|
||||
logger.info(f"[Downsample] Renamed: {os.path.basename(final_path)} -> {new_basename}")
|
||||
for lyrics_ext in (".lrc", ".txt"):
|
||||
old_lyrics = os.path.splitext(final_path)[0] + lyrics_ext
|
||||
if os.path.isfile(old_lyrics):
|
||||
new_lyrics = os.path.splitext(new_path)[0] + lyrics_ext
|
||||
os.rename(old_lyrics, new_lyrics)
|
||||
return new_path
|
||||
except Exception as rename_err:
|
||||
logger.error(f"[Downsample] Could not rename file: {rename_err}")
|
||||
|
||||
return final_path
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.info(f"[Downsample] Conversion timed out for: {os.path.basename(final_path)}")
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
except Exception as e:
|
||||
logger.error(f"[Downsample] Conversion error: {e}")
|
||||
if os.path.exists(temp_path):
|
||||
try:
|
||||
os.remove(temp_path)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def create_lossy_copy(final_path):
|
||||
"""Convert a FLAC file to a lossy copy using the configured codec."""
|
||||
from mutagen.flac import FLAC
|
||||
|
||||
if not config_manager.get("lossy_copy.enabled", False):
|
||||
return None
|
||||
|
||||
if os.path.splitext(final_path)[1].lower() != ".flac":
|
||||
return None
|
||||
|
||||
codec = config_manager.get("lossy_copy.codec", "mp3").lower()
|
||||
bitrate = config_manager.get("lossy_copy.bitrate", "320")
|
||||
|
||||
if codec == "opus" and int(bitrate) > 256:
|
||||
bitrate = "256"
|
||||
|
||||
codec_map = {
|
||||
"mp3": ("libmp3lame", ".mp3", f"MP3-{bitrate}", ["-vn", "-id3v2_version", "3"]),
|
||||
"opus": ("libopus", ".opus", f"OPUS-{bitrate}", ["-vn", "-map", "0:a", "-vbr", "on"]),
|
||||
"aac": ("aac", ".m4a", f"AAC-{bitrate}", ["-vn", "-movflags", "+faststart"]),
|
||||
}
|
||||
|
||||
if codec not in codec_map:
|
||||
logger.info(f"[Lossy Copy] Unknown codec '{codec}' - skipping conversion")
|
||||
return None
|
||||
|
||||
ffmpeg_codec, out_ext, quality_label, extra_args = codec_map[codec]
|
||||
out_path = os.path.splitext(final_path)[0] + out_ext
|
||||
|
||||
original_quality = get_audio_quality_string(final_path)
|
||||
if original_quality:
|
||||
out_basename = os.path.basename(out_path)
|
||||
if original_quality in out_basename:
|
||||
out_basename = out_basename.replace(original_quality, quality_label)
|
||||
out_path = os.path.join(os.path.dirname(out_path), out_basename)
|
||||
|
||||
ffmpeg_bin = shutil.which("ffmpeg")
|
||||
if not ffmpeg_bin:
|
||||
local = os.path.join(os.path.dirname(__file__), "tools", "ffmpeg")
|
||||
if os.path.isfile(local):
|
||||
ffmpeg_bin = local
|
||||
else:
|
||||
logger.warning(f"[Lossy Copy] ffmpeg not found - skipping {codec.upper()} conversion")
|
||||
return None
|
||||
|
||||
try:
|
||||
logger.info(f"[Lossy Copy] Converting to {quality_label}: {os.path.basename(final_path)}")
|
||||
cmd = [
|
||||
ffmpeg_bin, "-i", final_path,
|
||||
"-codec:a", ffmpeg_codec,
|
||||
"-b:a", f"{bitrate}k",
|
||||
"-map_metadata", "0",
|
||||
] + extra_args + ["-y", out_path]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
|
||||
if result.returncode == 0:
|
||||
logger.info(f"[Lossy Copy] Created {quality_label} copy: {os.path.basename(out_path)}")
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
audio = MutagenFile(out_path)
|
||||
if audio is not None:
|
||||
if codec == "mp3":
|
||||
from mutagen.id3 import TXXX
|
||||
audio.tags.add(TXXX(encoding=3, desc="QUALITY", text=[quality_label]))
|
||||
elif codec == "opus":
|
||||
audio["QUALITY"] = [quality_label]
|
||||
elif codec == "aac":
|
||||
from mutagen.mp4 import MP4FreeForm
|
||||
audio["----:com.apple.iTunes:QUALITY"] = [MP4FreeForm(quality_label.encode("utf-8"))]
|
||||
audio.save()
|
||||
except Exception as tag_err:
|
||||
logger.error(f"[Lossy Copy] Could not update QUALITY tag: {tag_err}")
|
||||
return out_path
|
||||
|
||||
logger.error(f"[Lossy Copy] ffmpeg failed: {result.stderr[:200]}")
|
||||
if os.path.exists(out_path):
|
||||
try:
|
||||
os.remove(out_path)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning(f"[Lossy Copy] Conversion timed out for: {os.path.basename(final_path)}")
|
||||
except Exception as e:
|
||||
logger.error(f"[Lossy Copy] Conversion error: {e}")
|
||||
return None
|
||||
92
core/imports/filename.py
Normal file
92
core/imports/filename.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
"""Filename parsing helpers used by import flows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
_TRACK_PATTERNS = (
|
||||
r"^(\d+)\s*[-\.]\s*(.+?)\s*[-–]\s*(.+)$",
|
||||
r"^(.+?)\s*[-–]\s*(.+)$",
|
||||
r"^(\d+)\s*[-\.]\s*(.+)$",
|
||||
)
|
||||
|
||||
|
||||
def extract_track_number_from_filename(filename: str, title: str = None) -> int:
|
||||
"""Extract track number from a filename. Returns 1 if not found."""
|
||||
basename = os.path.splitext(os.path.basename(filename))[0].strip()
|
||||
|
||||
match = re.match(r"^\d[\-\.](\d{1,2})\s*[\-\.]\s*", basename)
|
||||
if match:
|
||||
num = int(match.group(1))
|
||||
if 1 <= num <= 99:
|
||||
return num
|
||||
|
||||
match = re.match(r"^\(?(\d{1,3})\)?\s*[\-\.)\]]\s*", basename)
|
||||
if match:
|
||||
num = int(match.group(1))
|
||||
if 1 <= num <= 999:
|
||||
return num
|
||||
|
||||
return 1
|
||||
|
||||
|
||||
def parse_filename_metadata(filename: str) -> Dict[str, Any]:
|
||||
"""Extract artist/title/album hints from a loose filename."""
|
||||
raw_path = str(filename or "")
|
||||
normalized_path = raw_path.replace("\\", "/")
|
||||
base_name = os.path.splitext(os.path.basename(normalized_path))[0]
|
||||
|
||||
result: Dict[str, Any] = {
|
||||
"artist": "",
|
||||
"title": "",
|
||||
"album": "",
|
||||
"track_number": None,
|
||||
}
|
||||
|
||||
if not base_name:
|
||||
return result
|
||||
|
||||
for pattern in _TRACK_PATTERNS:
|
||||
match = re.match(pattern, base_name)
|
||||
if not match:
|
||||
continue
|
||||
|
||||
groups = match.groups()
|
||||
if len(groups) == 3:
|
||||
try:
|
||||
result["track_number"] = int(groups[0])
|
||||
result["artist"] = result["artist"] or groups[1].strip()
|
||||
result["title"] = result["title"] or groups[2].strip()
|
||||
except ValueError:
|
||||
result["artist"] = result["artist"] or groups[0].strip()
|
||||
result["title"] = result["title"] or f"{groups[1]} - {groups[2]}".strip()
|
||||
elif len(groups) == 2:
|
||||
if groups[0].isdigit():
|
||||
try:
|
||||
result["track_number"] = int(groups[0])
|
||||
result["title"] = result["title"] or groups[1].strip()
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
result["artist"] = result["artist"] or groups[0].strip()
|
||||
result["title"] = result["title"] or groups[1].strip()
|
||||
break
|
||||
|
||||
if not result["title"]:
|
||||
result["title"] = base_name
|
||||
|
||||
if not result["album"] and "/" in normalized_path:
|
||||
path_parts = normalized_path.split("/")
|
||||
for part in reversed(path_parts[:-1]):
|
||||
if not part or part.startswith("@"):
|
||||
continue
|
||||
|
||||
cleaned = re.sub(r"^\d+\s*[-\.]\s*", "", part).strip()
|
||||
if len(cleaned) > 3:
|
||||
result["album"] = cleaned
|
||||
break
|
||||
|
||||
return result
|
||||
118
core/imports/guards.py
Normal file
118
core/imports/guards.py
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
"""Import post-processing guards and quarantine helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from config.settings import config_manager
|
||||
from core.imports.context import (
|
||||
get_import_clean_artist,
|
||||
get_import_clean_title,
|
||||
get_import_context_artist,
|
||||
get_import_original_search,
|
||||
get_import_track_info,
|
||||
normalize_import_context,
|
||||
)
|
||||
from core.imports.file_ops import safe_move_file
|
||||
from database.music_database import MusicDatabase
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("imports.guards")
|
||||
|
||||
|
||||
def _get_config_manager():
|
||||
return config_manager
|
||||
|
||||
|
||||
def move_to_quarantine(file_path: str, context: dict, reason: str, automation_engine=None) -> str:
|
||||
"""Move a file to the quarantine folder and write a metadata sidecar."""
|
||||
download_dir = _get_config_manager().get("soulseek.download_path", "./downloads")
|
||||
quarantine_dir = Path(download_dir) / "ss_quarantine"
|
||||
quarantine_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
original_name = Path(file_path).stem
|
||||
file_ext = Path(file_path).suffix
|
||||
|
||||
quarantine_filename = f"{timestamp}_{original_name}{file_ext}.quarantined"
|
||||
quarantine_path = quarantine_dir / quarantine_filename
|
||||
|
||||
safe_move_file(file_path, str(quarantine_path))
|
||||
|
||||
metadata_path = quarantine_dir / f"{timestamp}_{original_name}.json"
|
||||
context = normalize_import_context(context)
|
||||
original_search = get_import_original_search(context)
|
||||
artist_context = get_import_context_artist(context)
|
||||
|
||||
metadata = {
|
||||
"original_filename": Path(file_path).name,
|
||||
"quarantine_reason": reason,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"expected_track": get_import_clean_title(context, default=original_search.get("title", "Unknown")),
|
||||
"expected_artist": get_import_clean_artist(context, default=(artist_context.get("name", "") if isinstance(artist_context, dict) else "Unknown")),
|
||||
"context_key": context.get("context_key", "unknown"),
|
||||
}
|
||||
|
||||
try:
|
||||
with open(metadata_path, "w", encoding="utf-8") as f:
|
||||
json.dump(metadata, f, indent=2, ensure_ascii=False)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to write quarantine metadata: %s", exc)
|
||||
|
||||
logger.warning("File quarantined: %s - Reason: %s", quarantine_path, reason)
|
||||
|
||||
if automation_engine:
|
||||
try:
|
||||
ti = context.get("track_info", {})
|
||||
artists = ti.get("artists", [])
|
||||
artist_name = ""
|
||||
if artists:
|
||||
first = artists[0]
|
||||
artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first)
|
||||
automation_engine.emit(
|
||||
"download_quarantined",
|
||||
{
|
||||
"artist": artist_name,
|
||||
"title": ti.get("name", ""),
|
||||
"reason": reason or "Unknown",
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return str(quarantine_path)
|
||||
|
||||
|
||||
def check_flac_bit_depth(file_path: str, context: dict) -> Optional[str]:
|
||||
"""Return a rejection message if a FLAC file violates the configured bit depth."""
|
||||
if not context.get("_audio_quality", "").startswith("FLAC"):
|
||||
return None
|
||||
|
||||
quality_profile = MusicDatabase().get_quality_profile()
|
||||
flac_config = quality_profile.get("qualities", {}).get("flac", {})
|
||||
flac_pref = flac_config.get("bit_depth", "any")
|
||||
if flac_pref == "any":
|
||||
return None
|
||||
|
||||
actual_bits = context["_audio_quality"].replace("FLAC ", "").replace("bit", "")
|
||||
if actual_bits == flac_pref:
|
||||
return None
|
||||
|
||||
flac_fallback = flac_config.get("bit_depth_fallback", True)
|
||||
downsample_enabled = _get_config_manager().get("lossy_copy.downsample_hires", False)
|
||||
track_info = context.get("track_info", {})
|
||||
track_name = track_info.get("name", os.path.basename(file_path))
|
||||
|
||||
if flac_fallback or downsample_enabled:
|
||||
if downsample_enabled:
|
||||
logger.info("[FLAC Downsample] Accepted %s-bit FLAC (will be downsampled to %s-bit): %s", actual_bits, flac_pref, track_name)
|
||||
else:
|
||||
logger.warning("[FLAC Fallback] Accepted %s-bit FLAC (preferred %s-bit): %s", actual_bits, flac_pref, track_name)
|
||||
return None
|
||||
|
||||
return f"FLAC bit depth mismatch: file is {actual_bits}-bit, preference is {flac_pref}-bit"
|
||||
593
core/imports/paths.py
Normal file
593
core/imports/paths.py
Normal file
|
|
@ -0,0 +1,593 @@
|
|||
"""Shared path and naming helpers for import processing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Album grouping lives in core.imports.album_naming; this module keeps the
|
||||
# imported helper because the path builder still needs it.
|
||||
from core.imports.album_naming import resolve_album_group
|
||||
from core.imports.context import (
|
||||
extract_artist_name,
|
||||
get_import_clean_title,
|
||||
get_import_context_album,
|
||||
get_import_original_search,
|
||||
get_import_source,
|
||||
get_import_track_info,
|
||||
normalize_import_context,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("imports.paths")
|
||||
|
||||
|
||||
def _get_config_manager():
|
||||
try:
|
||||
from config.settings import config_manager
|
||||
return config_manager
|
||||
except Exception:
|
||||
class _FallbackConfig:
|
||||
@staticmethod
|
||||
def get(key, default=None):
|
||||
return default
|
||||
|
||||
return _FallbackConfig()
|
||||
|
||||
|
||||
def _get_itunes_client():
|
||||
try:
|
||||
from core.metadata_service import get_itunes_client
|
||||
return get_itunes_client()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _get_album_tracks_for_source(source: str, album_id: str):
|
||||
try:
|
||||
from core.metadata_service import get_album_tracks_for_source
|
||||
return get_album_tracks_for_source(source, album_id)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def docker_resolve_path(path_str: str) -> str:
|
||||
"""Resolve Docker-hosted Windows paths into container paths."""
|
||||
if os.path.exists("/.dockerenv") and len(path_str) >= 3 and path_str[1] == ":" and path_str[0].isalpha():
|
||||
drive_letter = path_str[0].lower()
|
||||
rest_of_path = path_str[2:].replace("\\", "/")
|
||||
return f"/host/mnt/{drive_letter}{rest_of_path}"
|
||||
return path_str
|
||||
|
||||
|
||||
def build_simple_download_destination(context, file_path: str):
|
||||
"""Build the destination path for a simple download into Transfer."""
|
||||
context = normalize_import_context(context)
|
||||
search_result = context.get("search_result", {}) or {}
|
||||
if not isinstance(search_result, dict):
|
||||
search_result = {}
|
||||
|
||||
transfer_dir = Path(docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer")))
|
||||
album_name = None
|
||||
original_filename = search_result.get("filename", "")
|
||||
if "/" in original_filename or "\\" in original_filename:
|
||||
path_parts = original_filename.replace("\\", "/").split("/")
|
||||
if len(path_parts) >= 2:
|
||||
album_name = path_parts[-2]
|
||||
if not album_name:
|
||||
album_value = search_result.get("album")
|
||||
if isinstance(album_value, dict):
|
||||
album_name = album_value.get("name", "")
|
||||
else:
|
||||
album_name = album_value
|
||||
|
||||
filename = Path(file_path).name
|
||||
if album_name and str(album_name).lower() not in {"unknown", "unknown album", ""}:
|
||||
album_name = sanitize_filename(str(album_name))
|
||||
destination_dir = transfer_dir / album_name
|
||||
else:
|
||||
album_name = ""
|
||||
destination_dir = transfer_dir
|
||||
|
||||
destination_dir.mkdir(parents=True, exist_ok=True)
|
||||
return destination_dir / filename, album_name, filename
|
||||
|
||||
|
||||
def sanitize_filename(filename: str) -> str:
|
||||
"""Sanitize filename for file system compatibility."""
|
||||
sanitized = re.sub(r'[<>:"/\\|?*]', "_", filename)
|
||||
sanitized = re.sub(r"\s+", " ", sanitized).strip()
|
||||
sanitized = sanitized.rstrip(". ") or "_"
|
||||
if re.match(r"^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)", sanitized, re.IGNORECASE):
|
||||
sanitized = "_" + sanitized
|
||||
return sanitized[:200]
|
||||
|
||||
|
||||
def sanitize_context_values(context: dict) -> dict:
|
||||
"""Sanitize all string values in a template context for path safety."""
|
||||
sanitized = {}
|
||||
for key, value in context.items():
|
||||
if isinstance(value, str) and value:
|
||||
sanitized[key] = sanitize_filename(value)
|
||||
else:
|
||||
sanitized[key] = value
|
||||
return sanitized
|
||||
|
||||
|
||||
def clean_track_title(track_title: str, artist_name: str) -> str:
|
||||
"""Clean up track title by removing artist prefix and other noise."""
|
||||
original = (track_title or "").strip()
|
||||
cleaned = original
|
||||
cleaned = re.sub(r"^\d{1,2}[\.\s\-]+", "", cleaned)
|
||||
artist_pattern = re.escape(artist_name or "") + r"\s*-\s*"
|
||||
cleaned = re.sub(f"^{artist_pattern}", "", cleaned, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r"^[A-Za-z0-9\.]+\s*-\s*\d{1,2}\s*-\s*", "", cleaned)
|
||||
quality_patterns = [
|
||||
r"\s*[\[\(][0-9]+\s*kbps[\]\)]\s*",
|
||||
r"\s*[\[\(]flac[\]\)]\s*",
|
||||
r"\s*[\[\(]mp3[\]\)]\s*",
|
||||
]
|
||||
for pattern in quality_patterns:
|
||||
cleaned = re.sub(pattern, "", cleaned, flags=re.IGNORECASE)
|
||||
cleaned = re.sub(r"^[-\s\.]+", "", cleaned)
|
||||
cleaned = re.sub(r"[-\s\.]+$", "", cleaned)
|
||||
cleaned = re.sub(r"\s+", " ", cleaned).strip()
|
||||
return cleaned if cleaned else original
|
||||
|
||||
|
||||
|
||||
|
||||
def get_album_type_display(raw_type, track_count) -> str:
|
||||
"""Return the display form of an album's type for the $albumtype template variable."""
|
||||
raw = (raw_type or "").strip().lower()
|
||||
try:
|
||||
tc = int(track_count or 0)
|
||||
except (TypeError, ValueError):
|
||||
tc = 0
|
||||
|
||||
if raw in ("compilation", "compile"):
|
||||
return "Compilation"
|
||||
if raw == "album":
|
||||
return "Album"
|
||||
if raw in ("single", "ep"):
|
||||
if tc <= 3:
|
||||
return "Single"
|
||||
if tc <= 6:
|
||||
return "EP"
|
||||
return "Album"
|
||||
|
||||
if tc <= 0:
|
||||
return "Album"
|
||||
if tc <= 3:
|
||||
return "Single"
|
||||
if tc <= 6:
|
||||
return "EP"
|
||||
return "Album"
|
||||
|
||||
|
||||
def _replace_template_variables(template: str, context: dict) -> str:
|
||||
clean_context = sanitize_context_values(context)
|
||||
result = template
|
||||
|
||||
album_artist_value = clean_context.get("albumartist", clean_context.get("artist", "Unknown Artist"))
|
||||
collab_mode = _get_config_manager().get("file_organization.collab_artist_mode", "first")
|
||||
if collab_mode == "first" and album_artist_value:
|
||||
artists_list = context.get("_artists_list")
|
||||
if artists_list and len(artists_list) > 1:
|
||||
first = artists_list[0]
|
||||
album_artist_value = first.get("name", first) if isinstance(first, dict) else str(first)
|
||||
elif artists_list and len(artists_list) == 1:
|
||||
itunes_artist_id = context.get("_itunes_artist_id")
|
||||
if itunes_artist_id and ("," in album_artist_value or " & " in album_artist_value):
|
||||
try:
|
||||
resolved_client = _get_itunes_client()
|
||||
if resolved_client and hasattr(resolved_client, "resolve_primary_artist"):
|
||||
resolved = resolved_client.resolve_primary_artist(itunes_artist_id)
|
||||
if resolved and resolved != album_artist_value:
|
||||
album_artist_value = resolved
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
bracket_map = {
|
||||
"albumartist": album_artist_value,
|
||||
"albumtype": clean_context.get("albumtype", "Album"),
|
||||
"playlist": clean_context.get("playlist_name", ""),
|
||||
"artistletter": (clean_context.get("artist", "U") or "U")[0].upper(),
|
||||
"artist": clean_context.get("artist", "Unknown Artist"),
|
||||
"album": clean_context.get("album", "Unknown Album"),
|
||||
"title": clean_context.get("title", "Unknown Track"),
|
||||
"track": f"{_coerce_int(clean_context.get('track_number', 1), 1):02d}",
|
||||
"disc": str(_coerce_int(clean_context.get("disc_number", 1), 1)),
|
||||
"discnum": str(_coerce_int(clean_context.get("disc_number", 1), 1)),
|
||||
"year": str(clean_context.get("year", "")),
|
||||
"quality": clean_context.get("quality", ""),
|
||||
}
|
||||
for var_name, val in bracket_map.items():
|
||||
result = result.replace("${" + var_name + "}", val)
|
||||
|
||||
result = result.replace("$albumartist", album_artist_value)
|
||||
result = result.replace("$albumtype", clean_context.get("albumtype", "Album"))
|
||||
result = result.replace("$playlist", clean_context.get("playlist_name", ""))
|
||||
result = result.replace("$artistletter", (clean_context.get("artist", "U") or "U")[0].upper())
|
||||
result = result.replace("$artist", clean_context.get("artist", "Unknown Artist"))
|
||||
result = result.replace("$album", clean_context.get("album", "Unknown Album"))
|
||||
result = result.replace("$title", clean_context.get("title", "Unknown Track"))
|
||||
result = result.replace("$track", f"{clean_context.get('track_number', 1):02d}")
|
||||
result = result.replace("$year", str(clean_context.get("year", "")))
|
||||
|
||||
result = re.sub(r"\s+", " ", result)
|
||||
result = re.sub(r"\s*-\s*-\s*", " - ", result)
|
||||
result = result.strip()
|
||||
return result
|
||||
|
||||
|
||||
def apply_path_template(template: str, context: dict) -> str:
|
||||
"""Apply a template to build a path string."""
|
||||
return _replace_template_variables(template, context)
|
||||
|
||||
|
||||
def get_file_path_from_template_raw(template: str, context: dict) -> tuple[str, str]:
|
||||
"""Build file path using a user-provided template string directly."""
|
||||
full_path = apply_path_template(template, context)
|
||||
|
||||
quality_value = context.get("quality", "")
|
||||
disc_number = _coerce_int(context.get("disc_number", 1), 1)
|
||||
disc_value = f"{disc_number:02d}"
|
||||
disc_value_raw = str(disc_number)
|
||||
|
||||
path_parts = full_path.split("/")
|
||||
if len(path_parts) > 1:
|
||||
folder_parts = path_parts[:-1]
|
||||
filename_base = path_parts[-1]
|
||||
|
||||
cleaned_folders = []
|
||||
for part in folder_parts:
|
||||
part = part.replace("$quality", "")
|
||||
part = part.replace("$discnum", "")
|
||||
part = part.replace("$disc", "")
|
||||
part = re.sub(r"\s*\[\s*\]", "", part)
|
||||
part = re.sub(r"\s*\(\s*\)", "", part)
|
||||
part = re.sub(r"\s*\{\s*\}", "", part)
|
||||
part = re.sub(r"\s*-\s*$", "", part)
|
||||
part = re.sub(r"^\s*-\s*", "", part)
|
||||
part = re.sub(r"\s+", " ", part).strip()
|
||||
if part:
|
||||
cleaned_folders.append(part)
|
||||
|
||||
filename_base = filename_base.replace("$quality", quality_value)
|
||||
filename_base = filename_base.replace("$discnum", disc_value_raw)
|
||||
filename_base = filename_base.replace("$disc", disc_value)
|
||||
filename_base = re.sub(r"\s*\[\s*\]", "", filename_base)
|
||||
filename_base = re.sub(r"\s*\(\s*\)", "", filename_base)
|
||||
filename_base = re.sub(r"\s*\{\s*\}", "", filename_base)
|
||||
filename_base = re.sub(r"\s*-\s*$", "", filename_base)
|
||||
filename_base = re.sub(r"\s+", " ", filename_base).strip()
|
||||
|
||||
sanitized_folders = [sanitize_filename(part) for part in cleaned_folders]
|
||||
folder_path = os.path.join(*sanitized_folders) if sanitized_folders else ""
|
||||
return folder_path, sanitize_filename(filename_base)
|
||||
|
||||
full_path = full_path.replace("$quality", quality_value)
|
||||
full_path = full_path.replace("$discnum", disc_value_raw)
|
||||
full_path = full_path.replace("$disc", disc_value)
|
||||
full_path = re.sub(r"\s*\[\s*\]", "", full_path)
|
||||
full_path = re.sub(r"\s*\(\s*\)", "", full_path)
|
||||
full_path = re.sub(r"\s*\{\s*\}", "", full_path)
|
||||
full_path = re.sub(r"\s*-\s*$", "", full_path)
|
||||
full_path = re.sub(r"\s+", " ", full_path).strip()
|
||||
return "", sanitize_filename(full_path)
|
||||
|
||||
|
||||
def get_file_path_from_template(context: dict, template_type: str = "album_path") -> tuple[str, str]:
|
||||
"""Build complete file path using configured templates."""
|
||||
if not _get_config_manager().get("file_organization.enabled", True):
|
||||
return None, None
|
||||
|
||||
templates = _get_config_manager().get("file_organization.templates", {})
|
||||
template = templates.get(template_type)
|
||||
if not template:
|
||||
default_templates = {
|
||||
"album_path": "$albumartist/$albumartist - $album/$track - $title",
|
||||
"single_path": "$artist/$artist - $title/$title",
|
||||
"compilation_path": "Compilations/$album/$track - $artist - $title",
|
||||
"playlist_path": "$playlist/$artist - $title",
|
||||
}
|
||||
template = default_templates.get(template_type, "$artist/$album/$track - $title")
|
||||
|
||||
full_path = apply_path_template(template, context)
|
||||
|
||||
path_parts = full_path.split("/")
|
||||
quality_value = context.get("quality", "")
|
||||
disc_number = _coerce_int(context.get("disc_number", 1), 1)
|
||||
disc_value = f"{disc_number:02d}"
|
||||
disc_value_raw = str(disc_number)
|
||||
|
||||
if len(path_parts) > 1:
|
||||
folder_parts = path_parts[:-1]
|
||||
filename_base = path_parts[-1]
|
||||
|
||||
cleaned_folders = []
|
||||
for part in folder_parts:
|
||||
part = part.replace("$quality", "")
|
||||
part = part.replace("$discnum", "")
|
||||
part = part.replace("$disc", "")
|
||||
part = re.sub(r"\s*\[\s*\]", "", part)
|
||||
part = re.sub(r"\s*\(\s*\)", "", part)
|
||||
part = re.sub(r"\s*\{\s*\}", "", part)
|
||||
part = re.sub(r"\s*-\s*$", "", part)
|
||||
part = re.sub(r"^\s*-\s*", "", part)
|
||||
part = re.sub(r"\s+", " ", part).strip()
|
||||
if part:
|
||||
cleaned_folders.append(part)
|
||||
|
||||
filename_base = filename_base.replace("$quality", quality_value)
|
||||
filename_base = filename_base.replace("$discnum", disc_value_raw)
|
||||
filename_base = filename_base.replace("$disc", disc_value)
|
||||
filename_base = re.sub(r"\s*\[\s*\]", "", filename_base)
|
||||
filename_base = re.sub(r"\s*\(\s*\)", "", filename_base)
|
||||
filename_base = re.sub(r"\s*\{\s*\}", "", filename_base)
|
||||
filename_base = re.sub(r"\s*-\s*$", "", filename_base)
|
||||
filename_base = re.sub(r"\s+", " ", filename_base).strip()
|
||||
|
||||
sanitized_folders = [sanitize_filename(part) for part in cleaned_folders]
|
||||
folder_path = os.path.join(*sanitized_folders) if sanitized_folders else ""
|
||||
filename = sanitize_filename(filename_base)
|
||||
return folder_path, filename
|
||||
|
||||
full_path = full_path.replace("$quality", quality_value)
|
||||
full_path = full_path.replace("$discnum", disc_value_raw)
|
||||
full_path = full_path.replace("$disc", disc_value)
|
||||
full_path = re.sub(r"\s*\[\s*\]", "", full_path)
|
||||
full_path = re.sub(r"\s*\(\s*\)", "", full_path)
|
||||
full_path = re.sub(r"\s*\{\s*\}", "", full_path)
|
||||
full_path = re.sub(r"\s*-\s*$", "", full_path)
|
||||
full_path = re.sub(r"\s+", " ", full_path).strip()
|
||||
return "", sanitize_filename(full_path)
|
||||
|
||||
|
||||
def _max_disc_number(album_tracks: Any) -> int:
|
||||
items = []
|
||||
if isinstance(album_tracks, dict):
|
||||
items = album_tracks.get("items") or album_tracks.get("tracks") or []
|
||||
elif isinstance(album_tracks, list):
|
||||
items = album_tracks
|
||||
|
||||
max_disc = 1
|
||||
for track in items:
|
||||
if not isinstance(track, dict):
|
||||
continue
|
||||
try:
|
||||
disc_number = int(track.get("disc_number", 1) or 1)
|
||||
except (TypeError, ValueError):
|
||||
disc_number = 1
|
||||
if disc_number > max_disc:
|
||||
max_disc = disc_number
|
||||
return max_disc
|
||||
|
||||
|
||||
def _coerce_int(value: Any, default: int = 1) -> int:
|
||||
try:
|
||||
coerced = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return coerced if coerced > 0 else default
|
||||
|
||||
|
||||
def build_final_path_for_track(context, artist_context, album_info, file_ext):
|
||||
"""Shared path builder used by both post-processing and verification."""
|
||||
transfer_dir = docker_resolve_path(_get_config_manager().get("soulseek.transfer_path", "./Transfer"))
|
||||
context = normalize_import_context(context)
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
album_context = get_import_context_album(context)
|
||||
source = get_import_source(context)
|
||||
playlist_folder_mode = track_info.get("_playlist_folder_mode", False)
|
||||
artist_name = extract_artist_name(artist_context)
|
||||
|
||||
source_info = track_info.get("source_info") or {}
|
||||
if isinstance(source_info, str):
|
||||
try:
|
||||
source_info = json.loads(source_info)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
source_info = {}
|
||||
if source_info.get("enhance") and source_info.get("original_file_path"):
|
||||
original_path = source_info["original_file_path"]
|
||||
original_dir = os.path.dirname(original_path)
|
||||
original_stem = os.path.splitext(os.path.basename(original_path))[0]
|
||||
final_path = os.path.join(original_dir, original_stem + file_ext)
|
||||
os.makedirs(original_dir, exist_ok=True)
|
||||
logger.info("[Enhance] Using original file location: %s", final_path)
|
||||
return final_path, True
|
||||
|
||||
year = ""
|
||||
if album_context and album_context.get("release_date"):
|
||||
release_date = album_context["release_date"]
|
||||
if release_date and len(release_date) >= 4:
|
||||
year = release_date[:4]
|
||||
|
||||
raw_album_type = ""
|
||||
if album_context:
|
||||
raw_album_type = album_context.get("album_type", "") or ""
|
||||
total_tracks = (album_context.get("total_tracks", 0) or 0) if album_context else 0
|
||||
album_type_display = get_album_type_display(raw_album_type, total_tracks)
|
||||
|
||||
if playlist_folder_mode:
|
||||
playlist_name = track_info.get("_playlist_name", "Unknown Playlist")
|
||||
track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track"))
|
||||
_artists = original_search.get("artists") or track_info.get("artists") or []
|
||||
|
||||
template_context = {
|
||||
"artist": artist_name,
|
||||
"albumartist": artist_name,
|
||||
"album": track_name,
|
||||
"title": track_name,
|
||||
"playlist_name": playlist_name,
|
||||
"track_number": 1,
|
||||
"disc_number": 1,
|
||||
"year": year,
|
||||
"quality": context.get("_audio_quality", ""),
|
||||
"albumtype": album_type_display,
|
||||
"_artists_list": _artists,
|
||||
"_itunes_artist_id": str(artist_context.get("id", "")) if isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source == "itunes" else None,
|
||||
}
|
||||
|
||||
folder_path, filename_base = get_file_path_from_template(template_context, "playlist_path")
|
||||
if folder_path and filename_base:
|
||||
final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
|
||||
os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
|
||||
return final_path, True
|
||||
|
||||
playlist_name_sanitized = sanitize_filename(playlist_name)
|
||||
playlist_dir = os.path.join(transfer_dir, playlist_name_sanitized)
|
||||
os.makedirs(playlist_dir, exist_ok=True)
|
||||
artist_name_sanitized = sanitize_filename(template_context["artist"])
|
||||
track_name_sanitized = sanitize_filename(track_name)
|
||||
new_filename = f"{artist_name_sanitized} - {track_name_sanitized}{file_ext}"
|
||||
return os.path.join(playlist_dir, new_filename), True
|
||||
|
||||
if album_info and album_info.get("is_album"):
|
||||
clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track"))
|
||||
track_number = _coerce_int(album_info.get("track_number", 1), 1)
|
||||
disc_number = _coerce_int(album_info.get("disc_number", 1), 1)
|
||||
_artists = original_search.get("artists") or track_info.get("artists") or []
|
||||
_album_ctx = album_context
|
||||
_itunes_aid = None
|
||||
_is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer")
|
||||
if _is_itunes and isinstance(artist_context, dict):
|
||||
_aid = artist_context.get("id", "")
|
||||
if str(_aid).isdigit():
|
||||
_itunes_aid = str(_aid)
|
||||
if not _itunes_aid and _album_ctx:
|
||||
_ext = _album_ctx.get("external_urls", {})
|
||||
if isinstance(_ext, dict) and _ext.get("itunes_artist_id"):
|
||||
_itunes_aid = _ext["itunes_artist_id"]
|
||||
|
||||
_artist_name = artist_name
|
||||
_album_artist_name = _artist_name
|
||||
_album_artists_for_collab = None
|
||||
_explicit_artist_ctx = track_info.get("_explicit_artist_context") if isinstance(track_info, dict) else None
|
||||
if isinstance(_explicit_artist_ctx, dict) and _explicit_artist_ctx.get("name"):
|
||||
_album_artist_name = _explicit_artist_ctx["name"]
|
||||
_album_artists_for_collab = [_explicit_artist_ctx]
|
||||
elif isinstance(_explicit_artist_ctx, str) and _explicit_artist_ctx:
|
||||
_album_artist_name = _explicit_artist_ctx
|
||||
_album_artists_for_collab = [{"name": _explicit_artist_ctx}]
|
||||
else:
|
||||
_sa_artists = _album_ctx.get("artists", []) if _album_ctx else []
|
||||
if _sa_artists:
|
||||
_first_sa = _sa_artists[0]
|
||||
if isinstance(_first_sa, dict) and _first_sa.get("name"):
|
||||
_album_artist_name = _first_sa["name"]
|
||||
elif isinstance(_first_sa, str) and _first_sa:
|
||||
_album_artist_name = _first_sa
|
||||
_album_artists_for_collab = _sa_artists
|
||||
|
||||
template_context = {
|
||||
"artist": _artist_name,
|
||||
"albumartist": _album_artist_name,
|
||||
"album": album_info["album_name"],
|
||||
"title": clean_track_name,
|
||||
"track_number": track_number,
|
||||
"disc_number": disc_number,
|
||||
"year": year,
|
||||
"quality": context.get("_audio_quality", ""),
|
||||
"albumtype": album_type_display,
|
||||
"_artists_list": _album_artists_for_collab if _album_artists_for_collab else _artists,
|
||||
"_itunes_artist_id": _itunes_aid,
|
||||
}
|
||||
total_discs = _coerce_int(album_context.get("total_discs", 1) if album_context else 1, 1)
|
||||
|
||||
if total_discs <= 1 and album_context and album_context.get("id"):
|
||||
if disc_number > 1:
|
||||
total_discs = disc_number
|
||||
else:
|
||||
try:
|
||||
_album_tracks = _get_album_tracks_for_source(source, str(album_context["id"]))
|
||||
if _album_tracks:
|
||||
total_discs = _max_disc_number(_album_tracks)
|
||||
if total_discs > 1:
|
||||
album_context["total_discs"] = total_discs
|
||||
logger.info(
|
||||
"[Multi-Disc] Resolved %s discs for single-track download of %r",
|
||||
total_discs,
|
||||
album_context.get("name"),
|
||||
)
|
||||
except Exception as _disc_err:
|
||||
logger.warning("[Multi-Disc] Could not resolve total_discs: %s", _disc_err)
|
||||
|
||||
album_template = _get_config_manager().get("file_organization.templates.album_path", "")
|
||||
user_controls_disc = "$disc" in album_template
|
||||
disc_label = _get_config_manager().get("file_organization.disc_label", "Disc")
|
||||
|
||||
folder_path, filename_base = get_file_path_from_template(template_context, "album_path")
|
||||
if folder_path and filename_base:
|
||||
if total_discs > 1 and not user_controls_disc:
|
||||
disc_folder = f"{disc_label} {disc_number}"
|
||||
final_path = os.path.join(transfer_dir, folder_path, disc_folder, filename_base + file_ext)
|
||||
os.makedirs(os.path.join(transfer_dir, folder_path, disc_folder), exist_ok=True)
|
||||
else:
|
||||
final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
|
||||
os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
|
||||
return final_path, True
|
||||
|
||||
artist_name_sanitized = sanitize_filename(template_context["albumartist"])
|
||||
album_name_sanitized = sanitize_filename(album_info["album_name"])
|
||||
artist_dir = os.path.join(transfer_dir, artist_name_sanitized)
|
||||
album_folder_name = f"{artist_name_sanitized} - {album_name_sanitized}"
|
||||
album_dir = os.path.join(artist_dir, album_folder_name)
|
||||
if total_discs > 1:
|
||||
album_dir = os.path.join(album_dir, f"{disc_label} {disc_number}")
|
||||
os.makedirs(album_dir, exist_ok=True)
|
||||
final_track_name_sanitized = sanitize_filename(clean_track_name)
|
||||
new_filename = f"{track_number:02d} - {final_track_name_sanitized}{file_ext}"
|
||||
return os.path.join(album_dir, new_filename), True
|
||||
|
||||
clean_track_name = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", "Unknown Track"))
|
||||
_artists = original_search.get("artists") or track_info.get("artists") or []
|
||||
_album_ctx = album_context
|
||||
_itunes_aid = None
|
||||
_is_itunes = source == "itunes" or (isinstance(artist_context, dict) and str(artist_context.get("id", "")).isdigit() and source != "deezer")
|
||||
if _is_itunes and isinstance(artist_context, dict):
|
||||
_aid = artist_context.get("id", "")
|
||||
if str(_aid).isdigit():
|
||||
_itunes_aid = str(_aid)
|
||||
if not _itunes_aid and _album_ctx:
|
||||
_ext = _album_ctx.get("external_urls", {})
|
||||
if isinstance(_ext, dict) and _ext.get("itunes_artist_id"):
|
||||
_itunes_aid = _ext["itunes_artist_id"]
|
||||
|
||||
template_context = {
|
||||
"artist": artist_name,
|
||||
"albumartist": artist_name,
|
||||
"album": album_info.get("album_name", clean_track_name) if album_info else clean_track_name,
|
||||
"title": clean_track_name,
|
||||
"track_number": 1,
|
||||
"disc_number": 1,
|
||||
"year": year,
|
||||
"quality": context.get("_audio_quality", ""),
|
||||
"albumtype": album_type_display,
|
||||
"_artists_list": _artists,
|
||||
"_itunes_artist_id": _itunes_aid,
|
||||
}
|
||||
|
||||
folder_path, filename_base = get_file_path_from_template(template_context, "single_path")
|
||||
if filename_base:
|
||||
if folder_path:
|
||||
final_path = os.path.join(transfer_dir, folder_path, filename_base + file_ext)
|
||||
os.makedirs(os.path.join(transfer_dir, folder_path), exist_ok=True)
|
||||
else:
|
||||
final_path = os.path.join(transfer_dir, filename_base + file_ext)
|
||||
os.makedirs(transfer_dir, exist_ok=True)
|
||||
return final_path, True
|
||||
|
||||
artist_name_sanitized = sanitize_filename(template_context["artist"])
|
||||
final_track_name_sanitized = sanitize_filename(clean_track_name)
|
||||
artist_dir = os.path.join(transfer_dir, artist_name_sanitized)
|
||||
single_folder_name = f"{artist_name_sanitized} - {final_track_name_sanitized}"
|
||||
single_dir = os.path.join(artist_dir, single_folder_name)
|
||||
os.makedirs(single_dir, exist_ok=True)
|
||||
new_filename = f"{final_track_name_sanitized}{file_ext}"
|
||||
return os.path.join(single_dir, new_filename), True
|
||||
952
core/imports/pipeline.py
Normal file
952
core/imports/pipeline.py
Normal file
|
|
@ -0,0 +1,952 @@
|
|||
"""Import/post-processing pipeline for downloads and imported files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from config.settings import config_manager
|
||||
from core.imports.file_ops import (
|
||||
cleanup_empty_directories,
|
||||
create_lossy_copy,
|
||||
downsample_hires_flac,
|
||||
get_audio_quality_string,
|
||||
get_quality_tier_from_extension,
|
||||
safe_move_file,
|
||||
)
|
||||
from core.imports.context import (
|
||||
build_import_album_info,
|
||||
detect_album_info_web,
|
||||
extract_artist_name,
|
||||
get_import_clean_artist,
|
||||
get_import_clean_title,
|
||||
get_import_context_artist,
|
||||
get_import_has_clean_metadata,
|
||||
get_import_original_search,
|
||||
get_import_source,
|
||||
get_import_track_info,
|
||||
normalize_import_context,
|
||||
)
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
from core.imports.guards import check_flac_bit_depth, move_to_quarantine
|
||||
from core.imports.side_effects import (
|
||||
check_and_remove_from_wishlist,
|
||||
emit_track_downloaded,
|
||||
record_download_provenance,
|
||||
record_library_history_download,
|
||||
record_retag_download,
|
||||
record_soulsync_library_entry,
|
||||
)
|
||||
from core.runtime_state import (
|
||||
add_activity_item,
|
||||
download_batches,
|
||||
download_tasks,
|
||||
matched_context_lock,
|
||||
matched_downloads_context,
|
||||
mark_task_completed as _mark_task_completed,
|
||||
post_process_locks,
|
||||
post_process_locks_lock,
|
||||
processed_download_ids,
|
||||
tasks_lock,
|
||||
)
|
||||
from core.metadata.artwork import download_cover_art
|
||||
from core.metadata.common import wipe_source_tags
|
||||
from core.metadata.enrichment import enhance_file_metadata
|
||||
from core.imports.paths import (
|
||||
build_final_path_for_track,
|
||||
build_simple_download_destination,
|
||||
docker_resolve_path,
|
||||
)
|
||||
from core.imports.album_naming import resolve_album_group
|
||||
from core.metadata.lyrics import generate_lrc_file
|
||||
from database.music_database import get_database
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("imports.pipeline")
|
||||
pp_logger = get_logger("post_processing")
|
||||
|
||||
__all__ = [
|
||||
"build_import_pipeline_runtime",
|
||||
"post_process_matched_download",
|
||||
"post_process_matched_download_with_verification",
|
||||
]
|
||||
|
||||
|
||||
def build_import_pipeline_runtime(
|
||||
*,
|
||||
automation_engine: Any | None = None,
|
||||
on_download_completed: Any | None = None,
|
||||
web_scan_manager: Any | None = None,
|
||||
repair_worker: Any | None = None,
|
||||
) -> SimpleNamespace:
|
||||
"""Build the runtime object consumed by core.imports.pipeline."""
|
||||
return SimpleNamespace(
|
||||
automation_engine=automation_engine,
|
||||
on_download_completed=on_download_completed,
|
||||
web_scan_manager=web_scan_manager,
|
||||
repair_worker=repair_worker,
|
||||
)
|
||||
|
||||
|
||||
def post_process_matched_download(context_key, context, file_path, runtime, metadata_runtime=None):
|
||||
on_download_completed = getattr(runtime, "on_download_completed", None)
|
||||
automation_engine = getattr(runtime, "automation_engine", None)
|
||||
web_scan_manager = getattr(runtime, "web_scan_manager", None)
|
||||
repair_worker = getattr(runtime, "repair_worker", None)
|
||||
metadata_runtime = metadata_runtime or runtime
|
||||
|
||||
def _notify_download_completed(batch_id, task_id, success=True):
|
||||
if on_download_completed:
|
||||
on_download_completed(batch_id, task_id, success=success)
|
||||
|
||||
with post_process_locks_lock:
|
||||
if context_key not in post_process_locks:
|
||||
post_process_locks[context_key] = threading.Lock()
|
||||
file_lock = post_process_locks[context_key]
|
||||
|
||||
file_lock.acquire()
|
||||
try:
|
||||
if not os.path.exists(file_path):
|
||||
existing_final = context.get('_final_processed_path')
|
||||
if existing_final and os.path.exists(existing_final):
|
||||
logger.info(
|
||||
f"[Race Guard] Source gone but destination exists — already processed by another thread: "
|
||||
f"{os.path.basename(existing_final)}"
|
||||
)
|
||||
return
|
||||
logger.error(
|
||||
f"[Race Guard] Source file gone and no known destination — marking as failed: "
|
||||
f"{os.path.basename(file_path)}"
|
||||
)
|
||||
context['_race_guard_failed'] = True
|
||||
return
|
||||
|
||||
_basename = os.path.basename(file_path)
|
||||
_prev_size = -1
|
||||
for _stability_check in range(5):
|
||||
try:
|
||||
_cur_size = os.path.getsize(file_path)
|
||||
except OSError:
|
||||
_cur_size = -1
|
||||
if _cur_size == _prev_size and _cur_size > 0:
|
||||
break
|
||||
_prev_size = _cur_size
|
||||
if _stability_check == 0:
|
||||
logger.info(f"Waiting for file to stabilise: {_basename} ({_cur_size} bytes)")
|
||||
time.sleep(1.5)
|
||||
else:
|
||||
logger.info(f"File may still be writing after stability checks: {_basename} ({_prev_size} bytes)")
|
||||
|
||||
_skip_acoustid = False
|
||||
try:
|
||||
from core.acoustid_verification import AcoustIDVerification, VerificationResult
|
||||
|
||||
verifier = AcoustIDVerification()
|
||||
available, available_reason = verifier.quick_check_available()
|
||||
if available and not _skip_acoustid:
|
||||
context = normalize_import_context(context)
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
artist_context = get_import_context_artist(context)
|
||||
|
||||
expected_track = get_import_clean_title(context, default=original_search.get('title', ''))
|
||||
expected_artist = ''
|
||||
track_artists = track_info.get('artists', [])
|
||||
if track_artists:
|
||||
first = track_artists[0]
|
||||
if isinstance(first, dict):
|
||||
expected_artist = first.get('name', '')
|
||||
elif isinstance(first, str):
|
||||
expected_artist = first
|
||||
if not expected_artist:
|
||||
expected_artist = extract_artist_name(artist_context) or get_import_clean_artist(context, default='')
|
||||
|
||||
if expected_track and expected_artist:
|
||||
logger.info(f"Running AcoustID verification for: '{expected_track}' by '{expected_artist}'")
|
||||
verification_result, verification_msg = verifier.verify_audio_file(
|
||||
file_path,
|
||||
expected_track,
|
||||
expected_artist,
|
||||
context,
|
||||
)
|
||||
logger.info(f"AcoustID verification result: {verification_result.value} - {verification_msg}")
|
||||
context['_acoustid_result'] = verification_result.value
|
||||
|
||||
if verification_result == VerificationResult.FAIL:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
verification_msg,
|
||||
automation_engine,
|
||||
)
|
||||
logger.error(f"File quarantined due to verification failure: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting wrong file: {file_path}")
|
||||
logger.error(f"Quarantine failed, deleting wrong file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as del_error:
|
||||
logger.error(f"Could not delete wrong file either: {del_error}")
|
||||
|
||||
context['_acoustid_quarantined'] = True
|
||||
context['_acoustid_failure_msg'] = verification_msg
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = (
|
||||
f"AcoustID verification failed: {verification_msg}"
|
||||
)
|
||||
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
else:
|
||||
logger.warning("AcoustID verification skipped: missing track/artist info")
|
||||
context['_acoustid_result'] = 'skip'
|
||||
else:
|
||||
logger.info(f"ℹ️ AcoustID verification not available: {available_reason}")
|
||||
context['_acoustid_result'] = 'disabled'
|
||||
except Exception as verify_error:
|
||||
logger.error(f"AcoustID verification error (continuing normally): {verify_error}")
|
||||
context['_acoustid_result'] = 'error'
|
||||
|
||||
search_result = context.get('search_result', {}) or {}
|
||||
if not isinstance(search_result, dict):
|
||||
search_result = {}
|
||||
is_simple_download = search_result.get('is_simple_download', False)
|
||||
if is_simple_download:
|
||||
logger.info(f"Processing simple download (no metadata enhancement): {file_path}")
|
||||
|
||||
destination, album_name, filename = build_simple_download_destination(context, file_path)
|
||||
if album_name:
|
||||
logger.info(f"Moving to album folder: {album_name}")
|
||||
else:
|
||||
logger.info("Moving to Transfer root (single track)")
|
||||
|
||||
safe_move_file(file_path, destination)
|
||||
logger.info(f"Moved simple download to: {destination}")
|
||||
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
||||
if web_scan_manager:
|
||||
threading.Thread(
|
||||
target=lambda: web_scan_manager.request_scan("Simple download completed"),
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
activity_target = f"{album_name}/{filename}" if album_name else filename
|
||||
add_activity_item("", "Download Complete", activity_target, "Now")
|
||||
logger.info(f"Simple download post-processing complete: {activity_target}")
|
||||
context['_simple_download_completed'] = True
|
||||
context['_final_path'] = str(destination)
|
||||
emit_track_downloaded(context, automation_engine)
|
||||
record_library_history_download(context)
|
||||
record_download_provenance(context)
|
||||
try:
|
||||
check_and_remove_from_wishlist(context)
|
||||
except Exception as wishlist_error:
|
||||
logger.error(f"[Simple Download] Error checking wishlist removal: {wishlist_error}")
|
||||
return
|
||||
|
||||
logger.info(f"Starting robust post-processing for: {context_key}")
|
||||
|
||||
context = normalize_import_context(context)
|
||||
artist_context = get_import_context_artist(context)
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
has_clean_metadata = get_import_has_clean_metadata(context)
|
||||
|
||||
if not artist_context:
|
||||
logger.error("Post-processing failed: Missing artist context.")
|
||||
return
|
||||
|
||||
_junk_artist_names = {'', 'unknown', 'unknown artist', 'various artists', 'none', 'null'}
|
||||
_artist_name = (artist_context.get('name', '') if isinstance(artist_context, dict) else '').strip()
|
||||
if _artist_name.lower() in _junk_artist_names:
|
||||
logger.info(f"[Unknown Artist Guard] Artist name is '{_artist_name}' — attempting to resolve")
|
||||
_resolved = False
|
||||
track_info_guard = track_info or {}
|
||||
original_search_guard = original_search or {}
|
||||
|
||||
_ti_artists = track_info_guard.get('artists', [])
|
||||
if isinstance(_ti_artists, list) and _ti_artists:
|
||||
_first = _ti_artists[0]
|
||||
_name = _first.get('name', '') if isinstance(_first, dict) else str(_first)
|
||||
if _name and _name.strip().lower() not in _junk_artist_names:
|
||||
artist_context['name'] = _name.strip()
|
||||
logger.info(f"[Unknown Artist Guard] Resolved from track_info.artists: '{_name}'")
|
||||
_resolved = True
|
||||
|
||||
if not _resolved:
|
||||
_os_artist = original_search_guard.get('artist') or original_search_guard.get('artist_name') or ''
|
||||
if isinstance(_os_artist, str) and _os_artist.strip().lower() not in _junk_artist_names:
|
||||
artist_context['name'] = _os_artist.strip()
|
||||
logger.info(f"[Unknown Artist Guard] Resolved from original_search_result: '{_os_artist}'")
|
||||
_resolved = True
|
||||
|
||||
if not _resolved:
|
||||
_track_id = track_info_guard.get('id') or track_info_guard.get('track_id') or ''
|
||||
if _track_id:
|
||||
try:
|
||||
from core.metadata_service import get_client_for_source, get_primary_source
|
||||
|
||||
_guard_source = get_import_source(context) or get_primary_source()
|
||||
_fb_client = get_client_for_source(_guard_source) or get_client_for_source(get_primary_source())
|
||||
if hasattr(_fb_client, 'get_track_details'):
|
||||
_details = _fb_client.get_track_details(str(_track_id))
|
||||
if _details and isinstance(_details, dict):
|
||||
_d_artists = _details.get('artists', [])
|
||||
if isinstance(_d_artists, list) and _d_artists:
|
||||
_d_first = _d_artists[0]
|
||||
_d_name = _d_first.get('name', '') if isinstance(_d_first, dict) else str(_d_first)
|
||||
if _d_name and _d_name.strip().lower() not in _junk_artist_names:
|
||||
artist_context['name'] = _d_name.strip()
|
||||
logger.info(f"[Unknown Artist Guard] Resolved from metadata API: '{_d_name}'")
|
||||
_resolved = True
|
||||
except Exception as _guard_err:
|
||||
logger.error(f"[Unknown Artist Guard] Metadata re-fetch failed: {_guard_err}")
|
||||
|
||||
if not _resolved:
|
||||
logger.error(f"[Unknown Artist Guard] Could not resolve artist — proceeding with '{_artist_name}'")
|
||||
|
||||
context['artist'] = artist_context
|
||||
|
||||
playlist_folder_mode = track_info.get("_playlist_folder_mode", False)
|
||||
logger.debug(f"[Debug] Post-processing - track_info type: {type(track_info)}, is None: {track_info is None}, is empty: {not track_info}")
|
||||
logger.debug(f"[Debug] Post-processing - playlist_folder_mode: {playlist_folder_mode}")
|
||||
if track_info:
|
||||
logger.debug(f"[Debug] Post-processing - track_info keys: {list(track_info.keys())}")
|
||||
|
||||
if playlist_folder_mode:
|
||||
playlist_name = track_info.get("_playlist_name", "Unknown Playlist")
|
||||
logger.info(f"[Playlist Folder Mode] Organizing in playlist folder: {playlist_name}")
|
||||
|
||||
file_ext = os.path.splitext(file_path)[1]
|
||||
final_path, _ = build_final_path_for_track(context, artist_context, None, file_ext)
|
||||
logger.info(f"Playlist mode final path: '{final_path}'")
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
if os.path.exists(final_path):
|
||||
logger.info(
|
||||
f"[Playlist Folder Mode] Source gone but destination exists — already processed by another thread: "
|
||||
f"{os.path.basename(final_path)}"
|
||||
)
|
||||
context['_final_processed_path'] = final_path
|
||||
return
|
||||
pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: Source file not found and destination does not exist: {file_path}")
|
||||
raise FileNotFoundError(f"Source file not found and destination does not exist: {file_path}")
|
||||
|
||||
context['_audio_quality'] = get_audio_quality_string(file_path)
|
||||
if context['_audio_quality']:
|
||||
logger.info(f"Audio quality detected: {context['_audio_quality']}")
|
||||
|
||||
rejection_reason = check_flac_bit_depth(file_path, context)
|
||||
if rejection_reason:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
rejection_reason,
|
||||
automation_engine,
|
||||
)
|
||||
logger.info(f"File quarantined due to bit depth filter: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
context['_bitdepth_rejected'] = True
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}"
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
try:
|
||||
logger.warning(
|
||||
f"[Metadata Input] Playlist mode - artist: '{artist_context.get('name', 'MISSING')}' "
|
||||
f"(id: {artist_context.get('id', 'MISSING')})"
|
||||
)
|
||||
enhance_file_metadata(file_path, context, artist_context, None, runtime=metadata_runtime)
|
||||
except Exception as meta_err:
|
||||
import traceback
|
||||
pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")
|
||||
wipe_source_tags(file_path)
|
||||
|
||||
logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'")
|
||||
safe_move_file(file_path, final_path)
|
||||
context['_final_processed_path'] = final_path
|
||||
|
||||
if config_manager.get('post_processing.replaygain_enabled', False):
|
||||
try:
|
||||
from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF
|
||||
if _rg_ffmpeg_ok():
|
||||
lufs, peak_dbfs = _rg_analyze(final_path)
|
||||
gain_db = _RG_REF - lufs
|
||||
_rg_write(final_path, gain_db, peak_dbfs)
|
||||
pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB — {os.path.basename(final_path)}")
|
||||
except Exception as rg_err:
|
||||
pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}")
|
||||
|
||||
downsampled_path = downsample_hires_flac(final_path, context)
|
||||
if downsampled_path:
|
||||
final_path = downsampled_path
|
||||
context['_final_processed_path'] = final_path
|
||||
|
||||
blasphemy_path = create_lossy_copy(final_path)
|
||||
if blasphemy_path:
|
||||
context['_final_processed_path'] = blasphemy_path
|
||||
|
||||
downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
|
||||
cleanup_empty_directories(downloads_path, file_path)
|
||||
|
||||
logger.info(f"[Playlist Folder Mode] Post-processing complete: {final_path}")
|
||||
|
||||
try:
|
||||
check_and_remove_from_wishlist(context)
|
||||
except Exception as wishlist_error:
|
||||
logger.error(f"[Playlist Folder] Error checking wishlist removal: {wishlist_error}")
|
||||
|
||||
emit_track_downloaded(context, automation_engine)
|
||||
record_library_history_download(context)
|
||||
record_download_provenance(context)
|
||||
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
if task_id and batch_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['stream_processed'] = True
|
||||
download_tasks[task_id]['status'] = 'completed'
|
||||
logger.info(f"[Playlist Folder Mode] Marked task {task_id} as completed")
|
||||
_notify_download_completed(batch_id, task_id, success=True)
|
||||
return
|
||||
|
||||
is_album_download = bool(context.get("is_album_download", False))
|
||||
album_info = build_import_album_info(context, force_album=is_album_download)
|
||||
|
||||
if is_album_download:
|
||||
if has_clean_metadata:
|
||||
logger.info("Album context with clean metadata found - using normalized album info")
|
||||
else:
|
||||
logger.warning("Album context found without clean metadata - using normalized album info")
|
||||
elif not album_info.get('is_album'):
|
||||
logger.info("Single track download - attempting album detection")
|
||||
detected_album_info = detect_album_info_web(context, artist_context)
|
||||
if detected_album_info:
|
||||
album_info = detected_album_info
|
||||
|
||||
if album_info and album_info['is_album'] and not is_album_download:
|
||||
logger.info(
|
||||
"SMART ALBUM GROUPING for track=%r original_album=%r",
|
||||
album_info.get('clean_track_name', 'Unknown'),
|
||||
album_info.get('album_name', 'None'),
|
||||
)
|
||||
original_album = original_search.get("album") if original_search.get("album") else None
|
||||
consistent_album_name = resolve_album_group(artist_context, album_info, original_album)
|
||||
album_info['album_name'] = consistent_album_name
|
||||
logger.info("Album grouping complete: final_album=%r", consistent_album_name)
|
||||
elif album_info and album_info['is_album'] and is_album_download:
|
||||
logger.info(
|
||||
"EXPLICIT ALBUM DOWNLOAD - preserving album name=%r; skipping smart grouping",
|
||||
album_info.get('album_name', 'None'),
|
||||
)
|
||||
|
||||
context['_audio_quality'] = get_audio_quality_string(file_path)
|
||||
if context['_audio_quality']:
|
||||
logger.info(f"Audio quality detected: {context['_audio_quality']}")
|
||||
|
||||
rejection_reason = check_flac_bit_depth(file_path, context)
|
||||
if rejection_reason:
|
||||
try:
|
||||
quarantine_path = move_to_quarantine(
|
||||
file_path,
|
||||
context,
|
||||
rejection_reason,
|
||||
automation_engine,
|
||||
)
|
||||
logger.info(f"File quarantined due to bit depth filter: {quarantine_path}")
|
||||
except Exception as quarantine_error:
|
||||
logger.error(f"Quarantine failed ({quarantine_error}), deleting file: {file_path}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
context['_bitdepth_rejected'] = True
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
if task_id:
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Bit depth filter: {rejection_reason}"
|
||||
if task_id and batch_id:
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
file_ext = os.path.splitext(file_path)[1]
|
||||
clean_track_name = get_import_clean_title(
|
||||
context,
|
||||
album_info=album_info,
|
||||
default=original_search.get('title', 'Unknown Track'),
|
||||
)
|
||||
track_number = album_info.get('track_number', 1)
|
||||
logger.debug(
|
||||
"Final track_number processing: source=%s album_info_track_number=%s track_number=%s",
|
||||
album_info.get('source', 'unknown'),
|
||||
album_info.get('track_number', 'NOT_FOUND'),
|
||||
track_number,
|
||||
)
|
||||
if track_number is None:
|
||||
track_number = extract_track_number_from_filename(file_path)
|
||||
logger.info(
|
||||
"Track number was None; extracted from filename=%r -> %s",
|
||||
os.path.basename(file_path),
|
||||
track_number,
|
||||
)
|
||||
if not isinstance(track_number, int) or track_number < 1:
|
||||
logger.error(f"Invalid track number ({track_number}), defaulting to 1")
|
||||
track_number = 1
|
||||
|
||||
logger.debug(f"FINAL track_number used for filename: {track_number}")
|
||||
album_info['track_number'] = track_number
|
||||
album_info['clean_track_name'] = clean_track_name
|
||||
logger.info(f"[FIX] Updated album_info track_number to {track_number} for consistent metadata")
|
||||
|
||||
final_path, _ = build_final_path_for_track(context, artist_context, album_info, file_ext)
|
||||
logger.info(f"Resolved path: '{final_path}'")
|
||||
context['_final_processed_path'] = final_path
|
||||
|
||||
try:
|
||||
logger.warning(f"[Metadata Input] artist: '{artist_context.get('name', 'MISSING')}' (id: {artist_context.get('id', 'MISSING')})")
|
||||
if album_info:
|
||||
logger.warning(
|
||||
f"[Metadata Input] album: '{album_info.get('album_name', 'MISSING')}', "
|
||||
f"track#: {album_info.get('track_number', 'MISSING')}, disc#: {album_info.get('disc_number', 'MISSING')}, "
|
||||
f"source: {album_info.get('source', 'unknown')}"
|
||||
)
|
||||
else:
|
||||
logger.info("[Metadata Input] album_info: None (single track)")
|
||||
enhance_file_metadata(file_path, context, artist_context, album_info, runtime=metadata_runtime)
|
||||
except Exception as meta_err:
|
||||
import traceback
|
||||
pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")
|
||||
wipe_source_tags(file_path)
|
||||
|
||||
_enhance_source_info = context.get('track_info', {}).get('source_info') or {}
|
||||
if isinstance(_enhance_source_info, str):
|
||||
try:
|
||||
_enhance_source_info = json.loads(_enhance_source_info)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
_enhance_source_info = {}
|
||||
is_enhance_download = _enhance_source_info.get('enhance', False)
|
||||
|
||||
logger.info(f"Moving '{os.path.basename(file_path)}' to '{final_path}'")
|
||||
if os.path.exists(final_path):
|
||||
if not os.path.exists(file_path):
|
||||
logger.info(f"[Protection] Destination exists and source already gone - file already transferred: {os.path.basename(final_path)}")
|
||||
return
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
existing_file = MutagenFile(final_path)
|
||||
has_metadata = existing_file is not None and len(existing_file.tags or {}) > 2
|
||||
if has_metadata and not is_enhance_download:
|
||||
_replace_lower = config_manager.get('import.replace_lower_quality', False)
|
||||
if _replace_lower:
|
||||
_existing_tier = get_quality_tier_from_extension(final_path)
|
||||
_incoming_tier = get_quality_tier_from_extension(file_path)
|
||||
if _incoming_tier[1] < _existing_tier[1]:
|
||||
logger.info(f"[Quality Replace] Replacing {_existing_tier[0]} with {_incoming_tier[0]}: {os.path.basename(final_path)}")
|
||||
try:
|
||||
os.remove(final_path)
|
||||
except Exception as e:
|
||||
logger.error(f"[Quality Replace] Could not remove existing file: {e}")
|
||||
else:
|
||||
logger.info(
|
||||
f"[Protection] Existing file is same or better quality ({_existing_tier[0]} vs {_incoming_tier[0]}) - skipping: "
|
||||
f"{os.path.basename(final_path)}"
|
||||
)
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"[Protection] Error removing redundant file: {e}")
|
||||
return
|
||||
else:
|
||||
logger.info(f"[Protection] Existing file already has metadata enhancement - skipping overwrite: {os.path.basename(final_path)}")
|
||||
logger.info(f"[Protection] Removing redundant download file: {os.path.basename(file_path)}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except FileNotFoundError:
|
||||
logger.error(f"[Protection] Could not remove redundant file (already gone): {file_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"[Protection] Error removing redundant file: {e}")
|
||||
return
|
||||
elif is_enhance_download:
|
||||
logger.info(f"[Enhance] Quality enhance mode — replacing existing file: {os.path.basename(final_path)}")
|
||||
try:
|
||||
os.remove(final_path)
|
||||
except Exception as e:
|
||||
logger.error(f"[Enhance] Could not remove existing file for replacement: {e}")
|
||||
else:
|
||||
logger.info(f"[Protection] Existing file lacks metadata - safe to overwrite: {os.path.basename(final_path)}")
|
||||
try:
|
||||
os.remove(final_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as check_error:
|
||||
logger.error(f"[Protection] Error checking existing file metadata, proceeding with overwrite: {check_error}")
|
||||
try:
|
||||
if os.path.exists(final_path):
|
||||
os.remove(final_path)
|
||||
except Exception as e:
|
||||
logger.error(f"[Protection] Failed to remove existing file for overwrite: {e}")
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
if os.path.exists(final_path):
|
||||
logger.info(f"[Pre-Move] Source already gone and destination exists - another thread completed transfer: {os.path.basename(final_path)}")
|
||||
download_cover_art(album_info, os.path.dirname(final_path), context)
|
||||
generate_lrc_file(final_path, context, artist_context, album_info)
|
||||
return
|
||||
expected_dir = os.path.dirname(final_path)
|
||||
expected_stem = os.path.splitext(os.path.basename(final_path))[0]
|
||||
expected_ext = os.path.splitext(final_path)[1]
|
||||
found_variant = None
|
||||
check_exts = {expected_ext}
|
||||
if expected_ext == '.flac' and config_manager.get('lossy_copy.enabled', False) and config_manager.get('lossy_copy.delete_original', False):
|
||||
_lossy_ext_map = {'mp3': '.mp3', 'opus': '.opus', 'aac': '.m4a'}
|
||||
_lossy_codec = config_manager.get('lossy_copy.codec', 'mp3')
|
||||
check_exts.add(_lossy_ext_map.get(_lossy_codec, '.mp3'))
|
||||
if os.path.exists(expected_dir):
|
||||
for f in os.listdir(expected_dir):
|
||||
f_ext = os.path.splitext(f)[1].lower()
|
||||
if f_ext in check_exts and os.path.splitext(f)[0].startswith(expected_stem):
|
||||
found_variant = os.path.join(expected_dir, f)
|
||||
break
|
||||
if found_variant:
|
||||
logger.debug(f"[Pre-Move] Source gone but found variant in destination (stream processor handled it): {os.path.basename(found_variant)}")
|
||||
context['_final_processed_path'] = found_variant
|
||||
download_cover_art(album_info, expected_dir, context)
|
||||
generate_lrc_file(found_variant, context, artist_context, album_info)
|
||||
return
|
||||
logger.warning(f"[Pre-Move] Source file gone and no matching file in destination: {os.path.basename(file_path)}")
|
||||
raise FileNotFoundError(f"Source file vanished before move and destination does not exist: {file_path}")
|
||||
|
||||
safe_move_file(file_path, final_path)
|
||||
|
||||
if is_enhance_download and _enhance_source_info.get('original_file_path'):
|
||||
original_enhance_path = _enhance_source_info['original_file_path']
|
||||
if os.path.normpath(original_enhance_path) != os.path.normpath(final_path) and os.path.exists(original_enhance_path):
|
||||
try:
|
||||
os.remove(original_enhance_path)
|
||||
old_fmt = os.path.splitext(original_enhance_path)[1]
|
||||
new_fmt = os.path.splitext(final_path)[1]
|
||||
logger.info(f"[Enhance] Upgraded {old_fmt} → {new_fmt}: {os.path.basename(final_path)}")
|
||||
except Exception as e:
|
||||
logger.error(f"[Enhance] Could not remove old-format file: {e}")
|
||||
elif is_enhance_download:
|
||||
old_fmt = _enhance_source_info.get('original_format', 'unknown')
|
||||
new_fmt = os.path.splitext(final_path)[1]
|
||||
logger.info(f"[Enhance] Replaced in-place ({old_fmt} → {new_fmt}): {os.path.basename(final_path)}")
|
||||
|
||||
download_cover_art(album_info, os.path.dirname(final_path), context)
|
||||
generate_lrc_file(final_path, context, artist_context, album_info)
|
||||
|
||||
if config_manager.get('post_processing.replaygain_enabled', False):
|
||||
try:
|
||||
from core.replaygain import analyze_track as _rg_analyze, write_replaygain_tags as _rg_write, is_ffmpeg_available as _rg_ffmpeg_ok, RG_REFERENCE_LUFS as _RG_REF
|
||||
if _rg_ffmpeg_ok():
|
||||
lufs, peak_dbfs = _rg_analyze(final_path)
|
||||
gain_db = _RG_REF - lufs
|
||||
_rg_write(final_path, gain_db, peak_dbfs)
|
||||
pp_logger.info(f"ReplayGain: {gain_db:+.2f} dB, peak {peak_dbfs:.2f} dBFS — {os.path.basename(final_path)}")
|
||||
except Exception as rg_err:
|
||||
pp_logger.debug(f"ReplayGain analysis skipped: {rg_err}")
|
||||
|
||||
downsampled_path = downsample_hires_flac(final_path, context)
|
||||
if downsampled_path:
|
||||
final_path = downsampled_path
|
||||
context['_final_processed_path'] = final_path
|
||||
|
||||
blasphemy_path = create_lossy_copy(final_path)
|
||||
if blasphemy_path:
|
||||
context['_final_processed_path'] = blasphemy_path
|
||||
|
||||
downloads_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads'))
|
||||
cleanup_empty_directories(downloads_path, file_path)
|
||||
|
||||
logger.info(f"Post-processing complete for: {context.get('_final_processed_path', final_path)}")
|
||||
|
||||
emit_track_downloaded(context, automation_engine)
|
||||
record_library_history_download(context)
|
||||
record_download_provenance(context)
|
||||
record_soulsync_library_entry(context, artist_context, album_info)
|
||||
|
||||
try:
|
||||
if not playlist_folder_mode:
|
||||
completed_path = context.get('_final_processed_path', final_path)
|
||||
record_retag_download(context, artist_context, album_info, completed_path)
|
||||
except Exception as retag_err:
|
||||
logger.error(f"[Post-Process] Retag data capture failed (non-fatal): {retag_err}")
|
||||
|
||||
try:
|
||||
completed_path = context.get('_final_processed_path', final_path)
|
||||
batch_id_for_repair = context.get('batch_id')
|
||||
if completed_path and batch_id_for_repair and repair_worker:
|
||||
album_folder = os.path.dirname(str(completed_path))
|
||||
if album_folder:
|
||||
repair_worker.register_folder(batch_id_for_repair, album_folder)
|
||||
except Exception as repair_err:
|
||||
logger.error(f"[Post-Process] Repair folder registration failed: {repair_err}")
|
||||
|
||||
try:
|
||||
completed_path = context.get('_final_processed_path', final_path)
|
||||
batch_id_for_consistency = context.get('batch_id')
|
||||
if completed_path and batch_id_for_consistency and album_info and album_info.get('is_album'):
|
||||
_file_info = {
|
||||
'path': str(completed_path),
|
||||
'track_number': album_info.get('track_number', 1),
|
||||
'disc_number': album_info.get('disc_number', 1),
|
||||
'title': get_import_clean_title(
|
||||
context,
|
||||
album_info=album_info,
|
||||
default=album_info.get('clean_track_name', ''),
|
||||
),
|
||||
}
|
||||
with tasks_lock:
|
||||
if batch_id_for_consistency in download_batches:
|
||||
download_batches[batch_id_for_consistency].setdefault('_consistency_files', []).append(_file_info)
|
||||
except Exception as cons_err:
|
||||
logger.error(f"[Post-Process] Album consistency registration failed: {cons_err}")
|
||||
|
||||
try:
|
||||
check_and_remove_from_wishlist(context)
|
||||
except Exception as wishlist_error:
|
||||
logger.error(f"[Post-Process] Error checking wishlist removal: {wishlist_error}")
|
||||
|
||||
task_id = context.get('task_id')
|
||||
batch_id = context.get('batch_id')
|
||||
if task_id and batch_id:
|
||||
logger.info(f"[Post-Process] Calling completion callback for task {task_id} in batch {batch_id}")
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['stream_processed'] = True
|
||||
download_tasks[task_id]['status'] = 'completed'
|
||||
logger.info(f"[Post-Process] Marked task {task_id} as completed")
|
||||
_notify_download_completed(batch_id, task_id, success=True)
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
pp_logger.info(f"[inner] EXCEPTION in post-processing for {context_key}: {e}")
|
||||
pp_logger.info(traceback.format_exc())
|
||||
logger.error(f"\nCRITICAL ERROR in post-processing for {context_key}: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
source_exists = os.path.exists(file_path) if file_path else False
|
||||
if source_exists:
|
||||
if context_key in processed_download_ids:
|
||||
processed_download_ids.remove(context_key)
|
||||
logger.warning(f"Removed {context_key} from processed set - will retry on next check")
|
||||
with matched_context_lock:
|
||||
if context_key not in matched_downloads_context:
|
||||
matched_downloads_context[context_key] = context
|
||||
logger.warning(f"Re-added {context_key} to context for retry")
|
||||
else:
|
||||
logger.warning(f"Source file gone, not retrying: {context_key}")
|
||||
finally:
|
||||
file_lock.release()
|
||||
with post_process_locks_lock:
|
||||
post_process_locks.pop(context_key, None)
|
||||
|
||||
|
||||
def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime, metadata_runtime=None):
|
||||
on_download_completed = getattr(runtime, "on_download_completed", None)
|
||||
|
||||
def _notify_download_completed(batch_id, task_id, success=True):
|
||||
if on_download_completed:
|
||||
on_download_completed(batch_id, task_id, success=success)
|
||||
|
||||
logger = pp_logger
|
||||
try:
|
||||
original_task_id = context.pop('task_id', None)
|
||||
original_batch_id = context.pop('batch_id', None)
|
||||
post_process_matched_download(context_key, context, file_path, runtime, metadata_runtime=metadata_runtime)
|
||||
if original_task_id:
|
||||
context['task_id'] = original_task_id
|
||||
if original_batch_id:
|
||||
context['batch_id'] = original_batch_id
|
||||
|
||||
if context.get('_race_guard_failed'):
|
||||
logger.info(f"Race guard: source file gone for task {task_id} — marking as failed")
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = 'Source file was already processed or removed by another task'
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
if context.get('_acoustid_quarantined'):
|
||||
failure_msg = context.get('_acoustid_failure_msg', 'AcoustID verification failed')
|
||||
logger.info(f"File was quarantined by AcoustID verification (task={task_id}): {failure_msg}")
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"AcoustID verification failed: {failure_msg}"
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
if context.get('_simple_download_completed'):
|
||||
expected_final_path = context.get('_final_path')
|
||||
if expected_final_path and os.path.exists(expected_final_path):
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
_mark_task_completed(task_id, context.get('track_info'))
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
_notify_download_completed(batch_id, task_id, success=True)
|
||||
return
|
||||
logger.info(
|
||||
f"FAILED simple download file not found at: {expected_final_path} "
|
||||
f"(task={task_id}, context={context_key})"
|
||||
)
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = (
|
||||
f"Downloaded file not found at expected location: {os.path.basename(expected_final_path)}"
|
||||
)
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
return
|
||||
|
||||
expected_final_path = context.get('_final_processed_path')
|
||||
if not expected_final_path:
|
||||
logger.info(f"No _final_processed_path in context for task {task_id} — cannot verify, assuming success")
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
_mark_task_completed(task_id, context.get('track_info'))
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
_notify_download_completed(batch_id, task_id, success=True)
|
||||
return
|
||||
|
||||
if os.path.exists(expected_final_path):
|
||||
redownload_ctx = None
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
_mark_task_completed(task_id, context.get('track_info'))
|
||||
download_tasks[task_id]['metadata_enhanced'] = True
|
||||
redownload_ctx = download_tasks[task_id].get('_redownload_context')
|
||||
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
||||
if redownload_ctx:
|
||||
try:
|
||||
old_path = redownload_ctx.get('old_file_path')
|
||||
lib_track_id = redownload_ctx.get('library_track_id')
|
||||
if redownload_ctx.get('delete_old_file') and old_path and os.path.exists(old_path):
|
||||
if os.path.normpath(old_path) != os.path.normpath(expected_final_path):
|
||||
os.remove(old_path)
|
||||
logger.info(f"[Redownload] Deleted old file: {old_path}")
|
||||
if lib_track_id and expected_final_path:
|
||||
_rd_db = get_database()
|
||||
_rd_conn = _rd_db._get_connection()
|
||||
_rd_cursor = _rd_conn.cursor()
|
||||
_rd_cursor.execute(
|
||||
"""
|
||||
UPDATE tracks SET file_path = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(expected_final_path, lib_track_id),
|
||||
)
|
||||
_rd_conn.commit()
|
||||
_rd_conn.close()
|
||||
logger.info(f"[Redownload] Updated DB path for track {lib_track_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"[Redownload] Post-processing hook error: {e}")
|
||||
|
||||
_notify_download_completed(batch_id, task_id, success=True)
|
||||
else:
|
||||
track_name = get_import_clean_title(context, default=context_key)
|
||||
logger.info(f"FAILED verification for '{track_name}' (task={task_id})")
|
||||
logger.info(f" expected_final_path: {expected_final_path}")
|
||||
logger.info(f" file_path (source): {file_path}, exists={os.path.exists(file_path)}")
|
||||
logger.info(
|
||||
f" is_album={context.get('is_album_download', False)}, "
|
||||
f"has_clean_data={get_import_has_clean_metadata(context)}"
|
||||
)
|
||||
expected_dir = os.path.dirname(expected_final_path)
|
||||
if os.path.exists(expected_dir):
|
||||
dir_contents = os.listdir(expected_dir)
|
||||
logger.info(f" directory contains {len(dir_contents)} files: {dir_contents[:20]}")
|
||||
else:
|
||||
logger.info(f" directory does not exist: {expected_dir}")
|
||||
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = (
|
||||
f'File verification failed: expected file at {os.path.basename(expected_final_path)} but it was not found after processing'
|
||||
)
|
||||
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
except Exception as e:
|
||||
import traceback
|
||||
logger.info(f"EXCEPTION in post-processing for '{context_key}' (task={task_id}): {e}")
|
||||
logger.info(traceback.format_exc())
|
||||
with tasks_lock:
|
||||
if task_id in download_tasks:
|
||||
download_tasks[task_id]['status'] = 'failed'
|
||||
download_tasks[task_id]['error_message'] = f"Post-processing verification failed: {str(e)}"
|
||||
with matched_context_lock:
|
||||
if context_key in matched_downloads_context:
|
||||
del matched_downloads_context[context_key]
|
||||
_notify_download_completed(batch_id, task_id, success=False)
|
||||
412
core/imports/resolution.py
Normal file
412
core/imports/resolution.py
Normal file
|
|
@ -0,0 +1,412 @@
|
|||
"""Single-track import lookup and context-building helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("imports.resolution")
|
||||
|
||||
|
||||
def _get_metadata_service():
|
||||
from core import metadata_service
|
||||
|
||||
return metadata_service
|
||||
|
||||
|
||||
def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any:
|
||||
if value is None:
|
||||
return default
|
||||
|
||||
for name in names:
|
||||
if isinstance(value, dict):
|
||||
if name in value and value[name] is not None:
|
||||
return value[name]
|
||||
else:
|
||||
candidate = getattr(value, name, None)
|
||||
if candidate is not None:
|
||||
return candidate
|
||||
return default
|
||||
|
||||
|
||||
def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]:
|
||||
if not artists:
|
||||
return []
|
||||
|
||||
if isinstance(artists, (str, bytes)):
|
||||
artists = [artists]
|
||||
elif isinstance(artists, dict):
|
||||
artists = [artists]
|
||||
else:
|
||||
try:
|
||||
artists = list(artists)
|
||||
except TypeError:
|
||||
artists = [artists]
|
||||
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
for artist in artists:
|
||||
if isinstance(artist, dict):
|
||||
name = _extract_lookup_value(artist, 'name', 'artist_name', 'title', default='') or ''
|
||||
artist_id = _extract_lookup_value(artist, 'id', 'artist_id', default='') or ''
|
||||
entry: Dict[str, Any] = {}
|
||||
if name:
|
||||
entry['name'] = str(name)
|
||||
if artist_id:
|
||||
entry['id'] = str(artist_id)
|
||||
genres = _extract_lookup_value(artist, 'genres', default=None)
|
||||
if genres is not None:
|
||||
entry['genres'] = genres
|
||||
if entry:
|
||||
normalized.append(entry)
|
||||
continue
|
||||
|
||||
name = str(artist).strip()
|
||||
if name:
|
||||
normalized.append({'name': name})
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _get_source_chain_for_lookup(
|
||||
source_override: Optional[str] = None,
|
||||
allow_fallback: bool = True,
|
||||
) -> List[str]:
|
||||
metadata_service = _get_metadata_service()
|
||||
primary_source = metadata_service.get_primary_source()
|
||||
source_chain = list(metadata_service.get_source_priority(primary_source))
|
||||
override = (source_override or '').strip().lower()
|
||||
|
||||
if override:
|
||||
source_chain = [override] + [source for source in source_chain if source != override]
|
||||
|
||||
if not allow_fallback:
|
||||
source_chain = source_chain[:1]
|
||||
|
||||
return source_chain
|
||||
|
||||
|
||||
def _build_track_search_query(source: str, title: str, artist: str) -> str:
|
||||
base_query = " ".join(part for part in (title, artist) if part).strip()
|
||||
if source == 'deezer' and title:
|
||||
if artist:
|
||||
return f'artist:"{artist}" track:"{title}"'
|
||||
return f'track:"{title}"'
|
||||
return base_query or title or artist
|
||||
|
||||
|
||||
def _pick_best_track_match(search_results: List[Any], title: str, artist: str = '') -> Optional[Any]:
|
||||
if not search_results:
|
||||
return None
|
||||
|
||||
target_title = str(title or '').strip().lower()
|
||||
target_artist = str(artist or '').strip().lower()
|
||||
|
||||
for candidate in search_results:
|
||||
candidate_title = str(_extract_lookup_value(candidate, 'name', 'title', 'track_name', default='') or '').strip().lower()
|
||||
if candidate_title != target_title:
|
||||
continue
|
||||
|
||||
if not target_artist:
|
||||
return candidate
|
||||
|
||||
candidate_artists = _normalize_context_artists(_extract_lookup_value(candidate, 'artists', default=[]))
|
||||
candidate_artist_name = candidate_artists[0]['name'].strip().lower() if candidate_artists else ''
|
||||
if candidate_artist_name == target_artist:
|
||||
return candidate
|
||||
|
||||
return search_results[0]
|
||||
|
||||
|
||||
def search_tracks_for_source(source: str, client: Any, query: str, limit: int = 1) -> List[Any]:
|
||||
if not client or not hasattr(client, 'search_tracks'):
|
||||
return []
|
||||
|
||||
try:
|
||||
kwargs = {'limit': limit}
|
||||
if source == 'spotify':
|
||||
kwargs['allow_fallback'] = False
|
||||
return client.search_tracks(query, **kwargs) or []
|
||||
except Exception as exc:
|
||||
logger.debug("Could not search %s for %s: %s", source, query, exc)
|
||||
return []
|
||||
|
||||
|
||||
def _build_single_import_context_payload(
|
||||
track_data: Any,
|
||||
source: Optional[str],
|
||||
source_priority: List[str],
|
||||
requested_title: str = '',
|
||||
requested_artist: str = '',
|
||||
) -> Dict[str, Any]:
|
||||
album_data = _extract_lookup_value(track_data, 'album', default=None)
|
||||
|
||||
track_id = str(_extract_lookup_value(track_data, 'id', 'track_id', 'trackId', default='') or '')
|
||||
track_name = _extract_lookup_value(track_data, 'name', 'title', 'trackName', default='') or requested_title or 'Unknown Track'
|
||||
track_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'artists', default=[]))
|
||||
if not track_artists and requested_artist:
|
||||
track_artists = [{'name': requested_artist}]
|
||||
|
||||
primary_track_artist = track_artists[0] if track_artists else {}
|
||||
primary_artist_name = primary_track_artist.get('name') or requested_artist or 'Unknown Artist'
|
||||
primary_artist_id = str(primary_track_artist.get('id', '') or _extract_lookup_value(track_data, 'artist_id', 'artistId', default='') or '')
|
||||
|
||||
album_name = _extract_lookup_value(track_data, 'album_name', 'collectionName', default='') or ''
|
||||
album_id = str(_extract_lookup_value(track_data, 'album_id', 'collectionId', 'albumId', default='') or '')
|
||||
release_date = str(_extract_lookup_value(track_data, 'release_date', default='') or '')
|
||||
album_type = str(_extract_lookup_value(track_data, 'album_type', default='album') or 'album')
|
||||
total_tracks = int(_extract_lookup_value(track_data, 'total_tracks', 'track_count', default=0) or 0)
|
||||
album_images: List[Dict[str, Any]] = []
|
||||
album_image_url = str(_extract_lookup_value(track_data, 'image_url', 'thumb_url', default='') or '')
|
||||
album_artists = _normalize_context_artists(_extract_lookup_value(track_data, 'album_artists', 'artists', default=[]))
|
||||
|
||||
if isinstance(album_data, dict):
|
||||
album_name = _extract_lookup_value(album_data, 'name', 'title', 'collectionName', default=album_name) or album_name
|
||||
album_id = str(_extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', default=album_id) or album_id)
|
||||
release_date = str(_extract_lookup_value(album_data, 'release_date', default=release_date) or release_date)
|
||||
album_type = str(_extract_lookup_value(album_data, 'album_type', default=album_type) or album_type)
|
||||
total_tracks = int(_extract_lookup_value(album_data, 'total_tracks', 'track_count', 'nb_tracks', default=total_tracks) or total_tracks)
|
||||
album_images = _extract_lookup_value(album_data, 'images', default=[]) or []
|
||||
if not album_image_url:
|
||||
album_image_url = str(_extract_lookup_value(album_data, 'image_url', 'thumb_url', default='') or '')
|
||||
if not album_image_url and album_images:
|
||||
album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '')
|
||||
album_artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[]))
|
||||
elif album_data:
|
||||
album_name = album_name or str(album_data)
|
||||
|
||||
if not album_artists and primary_artist_name:
|
||||
album_artists = [{'name': primary_artist_name}]
|
||||
|
||||
if not album_image_url and album_images:
|
||||
album_image_url = str(_extract_lookup_value(album_images[0], 'url', default='') or '')
|
||||
|
||||
track_info = {
|
||||
'id': track_id,
|
||||
'name': track_name,
|
||||
'track_number': int(_extract_lookup_value(track_data, 'track_number', 'trackNumber', default=1) or 1),
|
||||
'disc_number': int(_extract_lookup_value(track_data, 'disc_number', 'discNumber', default=1) or 1),
|
||||
'duration_ms': int(_extract_lookup_value(track_data, 'duration_ms', 'duration', 'trackTimeMillis', default=0) or 0),
|
||||
'artists': track_artists or [{'name': primary_artist_name}],
|
||||
'uri': str(_extract_lookup_value(track_data, 'uri', default='') or ''),
|
||||
'album': album_name,
|
||||
'album_id': album_id,
|
||||
'album_type': album_type,
|
||||
'release_date': release_date,
|
||||
'_source': source or '',
|
||||
}
|
||||
|
||||
album_payload = {
|
||||
'id': album_id,
|
||||
'name': album_name,
|
||||
'release_date': release_date,
|
||||
'total_tracks': total_tracks or 1,
|
||||
'album_type': album_type,
|
||||
'image_url': album_image_url,
|
||||
'images': album_images,
|
||||
'artists': album_artists,
|
||||
'_source': source or '',
|
||||
}
|
||||
|
||||
artist_payload = {
|
||||
'id': primary_artist_id,
|
||||
'name': primary_artist_name,
|
||||
'genres': [],
|
||||
'_source': source or '',
|
||||
}
|
||||
|
||||
original_search = {
|
||||
'title': track_name,
|
||||
'artist': primary_artist_name,
|
||||
'album': album_name,
|
||||
'track_number': track_info['track_number'],
|
||||
'disc_number': track_info['disc_number'],
|
||||
'clean_title': track_name,
|
||||
'clean_album': album_name,
|
||||
'clean_artist': primary_artist_name,
|
||||
'artists': track_info['artists'],
|
||||
'duration_ms': track_info['duration_ms'],
|
||||
'id': track_id,
|
||||
'_source': source or '',
|
||||
}
|
||||
|
||||
return {
|
||||
'success': bool(track_id or track_name != requested_title or album_name),
|
||||
'source': source,
|
||||
'source_priority': source_priority,
|
||||
'context': {
|
||||
'artist': artist_payload,
|
||||
'album': album_payload,
|
||||
'track_info': track_info,
|
||||
'original_search_result': original_search,
|
||||
'is_album_download': False,
|
||||
'has_clean_metadata': bool(track_id),
|
||||
'has_full_metadata': bool(track_id),
|
||||
'source': source,
|
||||
'source_priority': source_priority,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_single_import_fallback_context(
|
||||
requested_title: str,
|
||||
requested_artist: str,
|
||||
source_priority: List[str],
|
||||
) -> Dict[str, Any]:
|
||||
artist_name = requested_artist or 'Unknown Artist'
|
||||
title = requested_title or 'Unknown Track'
|
||||
return {
|
||||
'success': False,
|
||||
'source': None,
|
||||
'source_priority': source_priority,
|
||||
'context': {
|
||||
'artist': {
|
||||
'id': '',
|
||||
'name': artist_name,
|
||||
'genres': [],
|
||||
'_source': '',
|
||||
},
|
||||
'album': {
|
||||
'id': '',
|
||||
'name': '',
|
||||
'release_date': '',
|
||||
'total_tracks': 1,
|
||||
'album_type': 'album',
|
||||
'image_url': '',
|
||||
'images': [],
|
||||
'artists': [],
|
||||
'_source': '',
|
||||
},
|
||||
'track_info': {
|
||||
'id': '',
|
||||
'name': title,
|
||||
'track_number': 1,
|
||||
'disc_number': 1,
|
||||
'duration_ms': 0,
|
||||
'artists': [{'name': artist_name}],
|
||||
'uri': '',
|
||||
'album': '',
|
||||
'album_id': '',
|
||||
'album_type': 'album',
|
||||
'release_date': '',
|
||||
'_source': '',
|
||||
},
|
||||
'original_search_result': {
|
||||
'title': title,
|
||||
'artist': artist_name,
|
||||
'album': '',
|
||||
'track_number': 1,
|
||||
'disc_number': 1,
|
||||
'clean_title': title,
|
||||
'clean_album': '',
|
||||
'clean_artist': artist_name,
|
||||
'artists': [{'name': artist_name}],
|
||||
'duration_ms': 0,
|
||||
'id': '',
|
||||
'_source': '',
|
||||
},
|
||||
'is_album_download': False,
|
||||
'has_clean_metadata': False,
|
||||
'has_full_metadata': False,
|
||||
'source': None,
|
||||
'source_priority': source_priority,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_single_track_import_context(
|
||||
title: str,
|
||||
artist: str = '',
|
||||
override_id: Optional[str] = None,
|
||||
override_source: str = 'spotify',
|
||||
source_override: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build an import context for singles using source-priority metadata lookup."""
|
||||
metadata_service = _get_metadata_service()
|
||||
source_priority = _get_source_chain_for_lookup(source_override=source_override, allow_fallback=True)
|
||||
title = (title or '').strip()
|
||||
artist = (artist or '').strip()
|
||||
|
||||
if override_id:
|
||||
chosen_source = (override_source or 'spotify').strip().lower() or 'spotify'
|
||||
client = metadata_service.get_client_for_source(chosen_source)
|
||||
if client and hasattr(client, 'get_track_details'):
|
||||
try:
|
||||
track_data = client.get_track_details(str(override_id))
|
||||
if track_data:
|
||||
payload = _build_single_import_context_payload(
|
||||
track_data,
|
||||
chosen_source,
|
||||
source_priority,
|
||||
requested_title=title,
|
||||
requested_artist=artist,
|
||||
)
|
||||
if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'):
|
||||
try:
|
||||
artist_details = client.get_artist(payload['context']['artist']['id'])
|
||||
if artist_details:
|
||||
payload['context']['artist']['genres'] = _extract_lookup_value(
|
||||
artist_details,
|
||||
'genres',
|
||||
default=[],
|
||||
) or []
|
||||
except Exception:
|
||||
pass
|
||||
return payload
|
||||
except Exception as exc:
|
||||
logger.debug("Override track lookup failed on %s for %s: %s", chosen_source, override_id, exc)
|
||||
|
||||
for source in source_priority:
|
||||
client = metadata_service.get_client_for_source(source)
|
||||
if not client:
|
||||
continue
|
||||
|
||||
search_query = _build_track_search_query(source, title, artist)
|
||||
if not search_query:
|
||||
continue
|
||||
|
||||
search_results = search_tracks_for_source(source, client, search_query, limit=5)
|
||||
if not search_results and search_query != title:
|
||||
search_results = search_tracks_for_source(source, client, title, limit=5)
|
||||
if not search_results and artist and search_query != artist:
|
||||
search_results = search_tracks_for_source(source, client, artist, limit=5)
|
||||
|
||||
if not search_results:
|
||||
continue
|
||||
|
||||
best_match = _pick_best_track_match(search_results, title or search_query, artist)
|
||||
if not best_match:
|
||||
continue
|
||||
|
||||
resolved_track_id = str(_extract_lookup_value(best_match, 'id', 'track_id', 'trackId', default='') or '')
|
||||
resolved_data = best_match
|
||||
if resolved_track_id and hasattr(client, 'get_track_details'):
|
||||
try:
|
||||
detailed = client.get_track_details(resolved_track_id)
|
||||
if detailed:
|
||||
resolved_data = detailed
|
||||
except Exception as exc:
|
||||
logger.debug("Track detail lookup failed on %s for %s: %s", source, resolved_track_id, exc)
|
||||
|
||||
payload = _build_single_import_context_payload(
|
||||
resolved_data,
|
||||
source,
|
||||
source_priority,
|
||||
requested_title=title,
|
||||
requested_artist=artist,
|
||||
)
|
||||
if payload['context']['artist'].get('id') and hasattr(client, 'get_artist'):
|
||||
try:
|
||||
artist_details = client.get_artist(payload['context']['artist']['id'])
|
||||
if artist_details:
|
||||
payload['context']['artist']['genres'] = _extract_lookup_value(
|
||||
artist_details,
|
||||
'genres',
|
||||
default=[],
|
||||
) or []
|
||||
except Exception:
|
||||
pass
|
||||
return payload
|
||||
|
||||
return _build_single_import_fallback_context(title, artist, source_priority)
|
||||
574
core/imports/side_effects.py
Normal file
574
core/imports/side_effects.py
Normal file
|
|
@ -0,0 +1,574 @@
|
|||
"""Import post-processing side effects that do not need web runtime state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from config.settings import config_manager
|
||||
from core.imports.context import (
|
||||
extract_artist_name,
|
||||
get_import_clean_album,
|
||||
get_import_clean_artist,
|
||||
get_import_clean_title,
|
||||
get_import_context_album,
|
||||
get_import_context_artist,
|
||||
get_import_original_search,
|
||||
get_import_search_result,
|
||||
get_import_source,
|
||||
get_import_source_ids,
|
||||
get_import_track_info,
|
||||
normalize_import_context,
|
||||
get_library_source_id_columns,
|
||||
)
|
||||
from core.wishlist_service import get_wishlist_service
|
||||
from database.music_database import get_database
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("imports.side_effects")
|
||||
|
||||
|
||||
def _get_config_manager():
|
||||
return config_manager
|
||||
|
||||
|
||||
def _primary_track_artist_name(track_info: Dict[str, Any]) -> str:
|
||||
artists = (track_info or {}).get("artists", [])
|
||||
if isinstance(artists, list) and artists:
|
||||
first = artists[0]
|
||||
if isinstance(first, dict):
|
||||
return str(first.get("name", "") or "")
|
||||
return str(first or "")
|
||||
if isinstance(artists, str):
|
||||
return artists
|
||||
return str((track_info or {}).get("artist", "") or "")
|
||||
|
||||
|
||||
def _all_profile_wishlist_tracks(wishlist_service) -> List[Dict[str, Any]]:
|
||||
database = get_database()
|
||||
all_profiles = database.get_all_profiles()
|
||||
wishlist_tracks: List[Dict[str, Any]] = []
|
||||
for profile in all_profiles:
|
||||
wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=profile["id"]))
|
||||
return wishlist_tracks
|
||||
|
||||
|
||||
def _stable_soulsync_id(text: str) -> str:
|
||||
return str(abs(int(hashlib.md5(text.encode("utf-8", errors="replace")).hexdigest(), 16)) % (10 ** 9))
|
||||
|
||||
|
||||
def emit_track_downloaded(context: Dict[str, Any], automation_engine=None) -> None:
|
||||
"""Emit the track_downloaded automation event."""
|
||||
try:
|
||||
if not automation_engine:
|
||||
return
|
||||
|
||||
ti = context.get("track_info") or context.get("search_result") or {}
|
||||
artist_name = ""
|
||||
artists = ti.get("artists", [])
|
||||
if artists:
|
||||
first = artists[0]
|
||||
artist_name = first.get("name", str(first)) if isinstance(first, dict) else str(first)
|
||||
|
||||
automation_engine.emit(
|
||||
"track_downloaded",
|
||||
{
|
||||
"artist": artist_name,
|
||||
"title": ti.get("name", ti.get("title", "")),
|
||||
"album": ti.get("album", ""),
|
||||
"quality": context.get("_audio_quality", "Unknown"),
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def record_library_history_download(context: Dict[str, Any]) -> None:
|
||||
"""Record a completed download to the library_history table."""
|
||||
try:
|
||||
search_result = context.get("original_search_result") or context.get("search_result") or {}
|
||||
username = search_result.get("username", context.get("_download_username", ""))
|
||||
source_map = {
|
||||
"youtube": "YouTube",
|
||||
"tidal": "Tidal",
|
||||
"qobuz": "Qobuz",
|
||||
"hifi": "HiFi",
|
||||
"deezer_dl": "Deezer",
|
||||
"lidarr": "Lidarr",
|
||||
}
|
||||
download_source = source_map.get(username, "Soulseek")
|
||||
|
||||
ti = context.get("track_info") or context.get("search_result") or {}
|
||||
artist_name = _primary_track_artist_name(ti)
|
||||
if not artist_name:
|
||||
artist_name = ti.get("artist", "")
|
||||
|
||||
album_raw = ti.get("album", "")
|
||||
album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "")
|
||||
title = ti.get("name", ti.get("title", ""))
|
||||
quality = context.get("_audio_quality", "")
|
||||
file_path = context.get("_final_processed_path", context.get("_final_path", ""))
|
||||
|
||||
thumb_url = ""
|
||||
album_context = get_import_context_album(context)
|
||||
if album_context:
|
||||
thumb_url = album_context.get("image_url", "")
|
||||
if not thumb_url:
|
||||
images = album_context.get("images", [])
|
||||
if images:
|
||||
thumb_url = images[0].get("url", "")
|
||||
if not thumb_url:
|
||||
album_info = context.get("album_info", {})
|
||||
if isinstance(album_info, dict):
|
||||
thumb_url = album_info.get("album_image_url", "")
|
||||
|
||||
source_filename = search_result.get("filename", "")
|
||||
source_track_id = search_result.get("track_id", "") or search_result.get("id", "") or ti.get("id", "")
|
||||
source_track_title = search_result.get("title", "") or search_result.get("name", "")
|
||||
source_artist = search_result.get("artist", "")
|
||||
if source_filename and "||" in source_filename and username in ("tidal", "youtube", "qobuz", "hifi", "deezer_dl", "lidarr"):
|
||||
stream_id = source_filename.split("||")[0]
|
||||
if stream_id and not source_track_id:
|
||||
source_track_id = stream_id
|
||||
|
||||
acoustid_result = context.get("_acoustid_result", "")
|
||||
|
||||
db = get_database()
|
||||
db.add_library_history_entry(
|
||||
event_type="download",
|
||||
title=title,
|
||||
artist_name=artist_name,
|
||||
album_name=album_name,
|
||||
quality=quality,
|
||||
file_path=file_path,
|
||||
thumb_url=thumb_url,
|
||||
download_source=download_source,
|
||||
source_track_id=source_track_id,
|
||||
source_track_title=source_track_title,
|
||||
source_filename=source_filename,
|
||||
acoustid_result=acoustid_result,
|
||||
source_artist=source_artist,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def record_download_provenance(context: Dict[str, Any]) -> None:
|
||||
"""Record source provenance for a completed download."""
|
||||
try:
|
||||
search_result = context.get("original_search_result") or context.get("search_result") or {}
|
||||
username = search_result.get("username", context.get("_download_username", ""))
|
||||
filename = search_result.get("filename", "")
|
||||
source_service = {
|
||||
"youtube": "youtube",
|
||||
"tidal": "tidal",
|
||||
"qobuz": "qobuz",
|
||||
"hifi": "hifi",
|
||||
"deezer_dl": "deezer",
|
||||
"lidarr": "lidarr",
|
||||
}.get(username, "soulseek")
|
||||
|
||||
ti = context.get("track_info") or context.get("search_result") or {}
|
||||
artist_name = _primary_track_artist_name(ti)
|
||||
if not artist_name:
|
||||
artist_name = ti.get("artist", "")
|
||||
|
||||
album_raw = ti.get("album", "")
|
||||
album_name = album_raw.get("name", "") if isinstance(album_raw, dict) else str(album_raw or "")
|
||||
title = ti.get("name", ti.get("title", ""))
|
||||
|
||||
file_path = context.get("_final_processed_path", context.get("_final_path", ""))
|
||||
quality = context.get("_audio_quality", "")
|
||||
size = search_result.get("size", 0)
|
||||
|
||||
bit_depth = None
|
||||
sample_rate = None
|
||||
bitrate = None
|
||||
try:
|
||||
if file_path and os.path.isfile(file_path):
|
||||
from mutagen import File as MutagenFile
|
||||
|
||||
audio = MutagenFile(file_path)
|
||||
if audio and audio.info:
|
||||
sample_rate = getattr(audio.info, "sample_rate", None)
|
||||
bitrate = getattr(audio.info, "bitrate", None)
|
||||
bit_depth = getattr(audio.info, "bits_per_sample", None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
db = get_database()
|
||||
db.record_track_download(
|
||||
file_path=file_path,
|
||||
source_service=source_service,
|
||||
source_username=username,
|
||||
source_filename=filename,
|
||||
source_size=size or 0,
|
||||
audio_quality=quality,
|
||||
track_title=title,
|
||||
track_artist=artist_name,
|
||||
track_album=album_name,
|
||||
bit_depth=bit_depth,
|
||||
sample_rate=sample_rate,
|
||||
bitrate=bitrate,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def record_soulsync_library_entry(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any]) -> None:
|
||||
"""Write imported media to the SoulSync library tables when the active server is SoulSync."""
|
||||
try:
|
||||
if _get_config_manager().get_active_media_server() != "soulsync":
|
||||
return
|
||||
|
||||
context = normalize_import_context(context)
|
||||
final_path = context.get("_final_processed_path")
|
||||
if not final_path:
|
||||
return
|
||||
|
||||
album_ctx = get_import_context_album(context)
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
source = get_import_source(context)
|
||||
source_ids = get_import_source_ids(context)
|
||||
source_columns = get_library_source_id_columns(source)
|
||||
|
||||
artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="")
|
||||
if not artist_name or artist_name in ("Unknown", "Unknown Artist"):
|
||||
return
|
||||
|
||||
album_name = ""
|
||||
if album_info and isinstance(album_info, dict):
|
||||
album_name = album_info.get("album_name", "")
|
||||
if not album_name:
|
||||
album_name = album_ctx.get("name", "") or original_search.get("album", "")
|
||||
if not album_name:
|
||||
album_name = track_info.get("name", "Unknown")
|
||||
|
||||
track_name = get_import_clean_title(
|
||||
context,
|
||||
album_info=album_info,
|
||||
default=track_info.get("name", "") or original_search.get("title", ""),
|
||||
)
|
||||
track_number = (track_info.get("track_number") or (album_info.get("track_number") if isinstance(album_info, dict) else None)) or 1
|
||||
duration_ms = track_info.get("duration_ms", 0) or 0
|
||||
|
||||
year = None
|
||||
release_date = album_ctx.get("release_date", "")
|
||||
if release_date and len(release_date) >= 4:
|
||||
try:
|
||||
year = int(release_date[:4])
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
image_url = album_ctx.get("image_url", "")
|
||||
if not image_url:
|
||||
images = album_ctx.get("images", [])
|
||||
if images and isinstance(images, list) and len(images) > 0:
|
||||
img = images[0]
|
||||
image_url = img.get("url", "") if isinstance(img, dict) else str(img)
|
||||
|
||||
artist_source_id = source_ids.get("artist_id", "")
|
||||
album_source_id = source_ids.get("album_id", "")
|
||||
track_source_id = source_ids.get("track_id", "")
|
||||
for key in ("auto_import", "from_sync_modal", "explicit_artist", "explicit_album", ""):
|
||||
if artist_source_id == key:
|
||||
artist_source_id = ""
|
||||
if album_source_id == key:
|
||||
album_source_id = ""
|
||||
if track_source_id == key:
|
||||
track_source_id = ""
|
||||
|
||||
genres = (artist_context or {}).get("genres", []) if isinstance(artist_context, dict) else []
|
||||
if genres:
|
||||
from core.genre_filter import filter_genres as _filter_genres
|
||||
|
||||
genres = _filter_genres(genres, _get_config_manager())
|
||||
genres_json = json.dumps(genres) if genres else ""
|
||||
|
||||
bitrate = 0
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
|
||||
audio = MutagenFile(final_path)
|
||||
if audio and hasattr(audio, "info") and audio.info and hasattr(audio.info, "bitrate"):
|
||||
bitrate = int(audio.info.bitrate / 1000) if audio.info.bitrate else 0
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
artist_id = _stable_soulsync_id(artist_name.lower().strip())
|
||||
album_id = _stable_soulsync_id(f"{artist_name}::{album_name}".lower().strip())
|
||||
track_id = _stable_soulsync_id(final_path)
|
||||
total_tracks = album_ctx.get("total_tracks", 0) or 0
|
||||
|
||||
db = get_database()
|
||||
with db._get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("SELECT id FROM artists WHERE id = ? AND server_source = 'soulsync'", (artist_id,))
|
||||
if not cursor.fetchone():
|
||||
cursor.execute(
|
||||
"SELECT id FROM artists WHERE name COLLATE NOCASE = ? AND server_source = 'soulsync' LIMIT 1",
|
||||
(artist_name,),
|
||||
)
|
||||
existing_by_name = cursor.fetchone()
|
||||
if existing_by_name:
|
||||
artist_id = existing_by_name[0]
|
||||
else:
|
||||
cursor.execute("SELECT id FROM artists WHERE id = ?", (artist_id,))
|
||||
if cursor.fetchone():
|
||||
artist_id = _stable_soulsync_id(artist_name.lower().strip() + "::soulsync")
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO artists (id, name, genres, thumb_url, server_source, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
""",
|
||||
(artist_id, artist_name, genres_json, image_url),
|
||||
)
|
||||
artist_source_col = source_columns.get("artist")
|
||||
if artist_source_col and artist_source_id:
|
||||
try:
|
||||
cursor.execute(
|
||||
f"UPDATE artists SET {artist_source_col} = ? WHERE id = ?",
|
||||
(artist_source_id, artist_id),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
cursor.execute("SELECT id FROM albums WHERE id = ? AND server_source = 'soulsync'", (album_id,))
|
||||
if not cursor.fetchone():
|
||||
cursor.execute(
|
||||
"SELECT id FROM albums WHERE title COLLATE NOCASE = ? AND artist_id = ? AND server_source = 'soulsync' LIMIT 1",
|
||||
(album_name, artist_id),
|
||||
)
|
||||
existing_album_by_name = cursor.fetchone()
|
||||
if existing_album_by_name:
|
||||
album_id = existing_album_by_name[0]
|
||||
else:
|
||||
cursor.execute("SELECT id FROM albums WHERE id = ?", (album_id,))
|
||||
if cursor.fetchone():
|
||||
album_id = _stable_soulsync_id(f"{artist_name}::{album_name}::soulsync".lower().strip())
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO albums (id, artist_id, title, year, thumb_url, genres, track_count,
|
||||
duration, server_source, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
""",
|
||||
(album_id, artist_id, album_name, year, image_url, genres_json, total_tracks, duration_ms),
|
||||
)
|
||||
album_source_col = source_columns.get("album")
|
||||
if album_source_col and album_source_id:
|
||||
try:
|
||||
cursor.execute(
|
||||
f"UPDATE albums SET {album_source_col} = ? WHERE id = ?",
|
||||
(album_source_id, album_id),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
track_artist = None
|
||||
track_artists_list = track_info.get("artists", []) or original_search.get("artists", [])
|
||||
if track_artists_list:
|
||||
first_track_artist = track_artists_list[0]
|
||||
if isinstance(first_track_artist, dict):
|
||||
ta_name = first_track_artist.get("name", "")
|
||||
else:
|
||||
ta_name = str(first_track_artist)
|
||||
if ta_name and ta_name.lower() != artist_name.lower():
|
||||
track_artist = ta_name
|
||||
|
||||
cursor.execute("SELECT id FROM tracks WHERE file_path = ?", (final_path,))
|
||||
if not cursor.fetchone():
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO tracks (id, album_id, artist_id, title, track_number,
|
||||
duration, file_path, bitrate, track_artist, server_source,
|
||||
created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
""",
|
||||
(
|
||||
track_id,
|
||||
album_id,
|
||||
artist_id,
|
||||
track_name,
|
||||
track_number,
|
||||
duration_ms,
|
||||
final_path,
|
||||
bitrate,
|
||||
track_artist,
|
||||
),
|
||||
)
|
||||
track_source_col = source_columns.get("track")
|
||||
if track_source_col and track_source_id:
|
||||
try:
|
||||
cursor.execute(
|
||||
f"UPDATE tracks SET {track_source_col} = ? WHERE id = ?",
|
||||
(track_source_id, track_id),
|
||||
)
|
||||
track_album_col = source_columns.get("track_album")
|
||||
if track_album_col and album_source_id:
|
||||
cursor.execute(
|
||||
f"UPDATE tracks SET {track_album_col} = ? WHERE id = ?",
|
||||
(album_source_id, track_id),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
conn.commit()
|
||||
logger.info("[SoulSync Library] Added: %s / %s / %s", artist_name, album_name, track_name)
|
||||
except Exception as exc:
|
||||
logger.error("[SoulSync Library] Could not record library entry: %s", exc)
|
||||
|
||||
|
||||
def record_retag_download(context: Dict[str, Any], artist_context: Dict[str, Any], album_info: Dict[str, Any], final_path: str) -> None:
|
||||
"""Record a completed download for later re-tagging."""
|
||||
try:
|
||||
db = get_database()
|
||||
|
||||
context = normalize_import_context(context)
|
||||
artist_context = get_import_context_artist(context) or (artist_context if isinstance(artist_context, dict) else {})
|
||||
album_context = get_import_context_album(context)
|
||||
track_info = get_import_track_info(context)
|
||||
original_search = get_import_original_search(context)
|
||||
source = get_import_source(context)
|
||||
source_ids = get_import_source_ids(context)
|
||||
|
||||
artist_name = extract_artist_name(artist_context) or get_import_clean_artist(context, default="Unknown Artist")
|
||||
is_album = album_info and album_info.get("is_album", False)
|
||||
group_type = "album" if is_album else "single"
|
||||
album_name = album_info.get("album_name", "") if album_info else get_import_clean_album(context, default=original_search.get("album", "Unknown"))
|
||||
|
||||
image_url = album_info.get("album_image_url") if album_info else None
|
||||
if not image_url:
|
||||
image_url = album_context.get("image_url", "")
|
||||
if not image_url and album_context.get("images"):
|
||||
images = album_context.get("images", [])
|
||||
if images and isinstance(images[0], dict):
|
||||
image_url = images[0].get("url", "")
|
||||
|
||||
total_tracks = album_context.get("total_tracks", 1) if album_context else 1
|
||||
release_date = album_context.get("release_date", "") if album_context else ""
|
||||
|
||||
spotify_album_id = None
|
||||
itunes_album_id = None
|
||||
if source == "spotify":
|
||||
spotify_album_id = source_ids.get("album_id", "") or None
|
||||
elif source == "itunes":
|
||||
itunes_album_id = source_ids.get("album_id", "") or None
|
||||
|
||||
group_id = db.find_retag_group(artist_name, album_name)
|
||||
if group_id is None:
|
||||
group_id = db.add_retag_group(
|
||||
group_type=group_type,
|
||||
artist_name=artist_name,
|
||||
album_name=album_name,
|
||||
image_url=image_url,
|
||||
spotify_album_id=spotify_album_id,
|
||||
itunes_album_id=itunes_album_id,
|
||||
total_tracks=total_tracks,
|
||||
release_date=release_date,
|
||||
)
|
||||
if group_id is None:
|
||||
return
|
||||
|
||||
track_number = album_info.get("track_number", 1) if album_info else (track_info.get("track_number", 1) or 1)
|
||||
disc_number = original_search.get("disc_number") or (album_info.get("disc_number", 1) if album_info else track_info.get("disc_number", 1) or 1)
|
||||
title = get_import_clean_title(
|
||||
context,
|
||||
album_info=album_info,
|
||||
default=album_info.get("clean_track_name", "Unknown Track") if album_info else "Unknown Track",
|
||||
)
|
||||
file_format = os.path.splitext(str(final_path))[1].lstrip(".").lower()
|
||||
|
||||
source_track_id = None
|
||||
itunes_track_id = None
|
||||
if source == "spotify":
|
||||
source_track_id = source_ids.get("track_id", "") or None
|
||||
elif source == "itunes":
|
||||
itunes_track_id = source_ids.get("track_id", "") or None
|
||||
|
||||
if not db.retag_track_exists(group_id, str(final_path)):
|
||||
db.add_retag_track(
|
||||
group_id=group_id,
|
||||
track_number=track_number,
|
||||
disc_number=disc_number,
|
||||
title=title,
|
||||
file_path=str(final_path),
|
||||
file_format=file_format,
|
||||
spotify_track_id=source_track_id,
|
||||
itunes_track_id=itunes_track_id,
|
||||
)
|
||||
logger.info("[Retag] Recorded track for retag: '%s' in '%s'", title, album_name)
|
||||
|
||||
db.trim_retag_groups(100)
|
||||
except Exception as exc:
|
||||
logger.error("[Retag] Could not record track for retag: %s", exc)
|
||||
|
||||
|
||||
def check_and_remove_from_wishlist(context: Dict[str, Any]) -> None:
|
||||
"""Check whether a successful download should be removed from the wishlist."""
|
||||
try:
|
||||
wishlist_service = get_wishlist_service()
|
||||
source = get_import_source(context)
|
||||
source_ids = get_import_source_ids(context)
|
||||
source_label = {
|
||||
"spotify": "Spotify",
|
||||
"itunes": "iTunes",
|
||||
"deezer": "Deezer",
|
||||
"discogs": "Discogs",
|
||||
"hydrabase": "Hydrabase",
|
||||
}.get(source, "Source")
|
||||
track_info = get_import_track_info(context) or get_import_search_result(context)
|
||||
search_result = get_import_original_search(context) or get_import_search_result(context)
|
||||
track_id = None
|
||||
|
||||
if source == "spotify":
|
||||
track_id = source_ids.get("track_id") or None
|
||||
if track_id:
|
||||
logger.info("[Wishlist] Found %s track ID from source_ids: %s", source_label, track_id)
|
||||
elif "wishlist_id" in track_info:
|
||||
wishlist_id = track_info["wishlist_id"]
|
||||
logger.info("[Wishlist] Found wishlist_id in context: %s", wishlist_id)
|
||||
wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service)
|
||||
for wishlist_track in wishlist_tracks:
|
||||
if wishlist_track.get("wishlist_id") == wishlist_id:
|
||||
track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id")
|
||||
logger.info("[Wishlist] Found track ID from wishlist entry: %s", track_id)
|
||||
break
|
||||
|
||||
if not track_id:
|
||||
track_name = track_info.get("name") or search_result.get("title", "")
|
||||
artist_name = _primary_track_artist_name(track_info) or _primary_track_artist_name(search_result)
|
||||
|
||||
if track_name and artist_name:
|
||||
logger.warning("[Wishlist] No track ID found, checking for fuzzy match: '%s' by '%s'", track_name, artist_name)
|
||||
|
||||
wishlist_tracks = _all_profile_wishlist_tracks(wishlist_service)
|
||||
for wishlist_track in wishlist_tracks:
|
||||
wl_name = wishlist_track.get("name", "").lower()
|
||||
wl_artists = wishlist_track.get("artists", [])
|
||||
wl_artist_name = ""
|
||||
if wl_artists:
|
||||
if isinstance(wl_artists[0], dict):
|
||||
wl_artist_name = wl_artists[0].get("name", "").lower()
|
||||
else:
|
||||
wl_artist_name = str(wl_artists[0]).lower()
|
||||
if wl_name == track_name.lower() and wl_artist_name == artist_name.lower():
|
||||
track_id = wishlist_track.get("spotify_track_id") or wishlist_track.get("id")
|
||||
logger.info("[Wishlist] Found fuzzy match - track ID: %s", track_id)
|
||||
break
|
||||
|
||||
if track_id:
|
||||
logger.info("[Wishlist] Attempting to remove track from wishlist: %s", track_id)
|
||||
removed = wishlist_service.mark_track_download_result(track_id, success=True)
|
||||
if removed:
|
||||
logger.info("[Wishlist] Successfully removed track from wishlist: %s", track_id)
|
||||
else:
|
||||
logger.warning("ℹ️ [Wishlist] Track not found in wishlist or already removed: %s", track_id)
|
||||
else:
|
||||
logger.warning("ℹ️ [Wishlist] No track ID found for wishlist removal check")
|
||||
except Exception as exc:
|
||||
logger.error("[Wishlist] Error in wishlist removal check: %s", exc)
|
||||
556
core/imports/staging.py
Normal file
556
core/imports/staging.py
Normal file
|
|
@ -0,0 +1,556 @@
|
|||
"""Shared staging folder and import suggestion helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
from core.imports.paths import docker_resolve_path
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("imports.staging")
|
||||
|
||||
AUDIO_EXTENSIONS = {".mp3", ".flac", ".ogg", ".opus", ".m4a", ".aac", ".wav", ".wma", ".aiff", ".aif", ".ape"}
|
||||
|
||||
_import_suggestions_cache_lock = threading.Lock()
|
||||
_import_suggestions_cache: Dict[str, Any] = {
|
||||
"suggestions": [],
|
||||
"building": False,
|
||||
"built": False,
|
||||
}
|
||||
|
||||
|
||||
def _get_config_manager():
|
||||
try:
|
||||
from config.settings import config_manager
|
||||
|
||||
return config_manager
|
||||
except Exception:
|
||||
class _FallbackConfig:
|
||||
@staticmethod
|
||||
def get(key, default=None):
|
||||
return default
|
||||
|
||||
return _FallbackConfig()
|
||||
|
||||
|
||||
def get_staging_path() -> str:
|
||||
"""Resolve the configured staging folder path."""
|
||||
raw = _get_config_manager().get("import.staging_path", "./Staging")
|
||||
return docker_resolve_path(raw)
|
||||
|
||||
|
||||
def get_import_suggestions_cache() -> Dict[str, Any]:
|
||||
"""Expose the shared import suggestions cache."""
|
||||
return _import_suggestions_cache
|
||||
|
||||
|
||||
def get_primary_source() -> str:
|
||||
from core.metadata_service import get_primary_source as _get_primary_source
|
||||
|
||||
return _get_primary_source()
|
||||
|
||||
|
||||
def get_source_priority(preferred_source: str):
|
||||
from core.metadata_service import get_source_priority as _get_source_priority
|
||||
|
||||
return _get_source_priority(preferred_source)
|
||||
|
||||
|
||||
def get_client_for_source(source: str):
|
||||
from core.metadata_service import get_client_for_source as _get_client_for_source
|
||||
|
||||
return _get_client_for_source(source)
|
||||
|
||||
|
||||
def read_staging_file_metadata(file_path: str, filename: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Read common audio tag metadata from a staging file."""
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
|
||||
tags = MutagenFile(file_path, easy=True)
|
||||
except Exception:
|
||||
tags = None
|
||||
|
||||
filename = filename or os.path.basename(file_path)
|
||||
stem = os.path.splitext(os.path.basename(filename))[0]
|
||||
|
||||
def _first_tag(*keys: str) -> str:
|
||||
if not tags:
|
||||
return ""
|
||||
for key in keys:
|
||||
try:
|
||||
value = tags.get(key) # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
value = None
|
||||
if value:
|
||||
if isinstance(value, (list, tuple)):
|
||||
value = value[0] if value else ""
|
||||
text = str(value).strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
title = _first_tag("title")
|
||||
artist = _first_tag("artist")
|
||||
albumartist = _first_tag("albumartist")
|
||||
album = _first_tag("album")
|
||||
|
||||
if not title:
|
||||
title = stem
|
||||
if not albumartist:
|
||||
albumartist = artist
|
||||
|
||||
track_number = extract_track_number_from_filename(filename or file_path)
|
||||
try:
|
||||
# Preserve tag-based numbers when present, but still fall back to the filename parser.
|
||||
tag_track_number = _first_tag("tracknumber", "track_number")
|
||||
if tag_track_number:
|
||||
track_number = int(str(tag_track_number).split("/")[0].strip() or track_number)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
disc_number = 1
|
||||
try:
|
||||
tag_disc_number = _first_tag("discnumber", "disc_number")
|
||||
if tag_disc_number:
|
||||
disc_number = int(str(tag_disc_number).split("/")[0].strip() or 1)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"artist": artist,
|
||||
"albumartist": albumartist,
|
||||
"album": album,
|
||||
"track_number": track_number,
|
||||
"disc_number": disc_number,
|
||||
}
|
||||
|
||||
|
||||
def _search_albums_for_source(source: str, client: Any, query: str, limit: int = 5):
|
||||
from core.metadata_service import _search_albums_for_source as _metadata_search_albums_for_source
|
||||
|
||||
return _metadata_search_albums_for_source(source, client, query, limit=limit)
|
||||
|
||||
|
||||
def _search_tracks_for_source(source: str, client: Any, query: str, limit: int = 5):
|
||||
from core.imports.resolution import search_tracks_for_source
|
||||
|
||||
return search_tracks_for_source(source, client, query, limit=limit)
|
||||
|
||||
|
||||
def _extract_value(value: Any, *names: str, default: Any = None) -> Any:
|
||||
if value is None:
|
||||
return default
|
||||
|
||||
if isinstance(value, (str, bytes)):
|
||||
return default
|
||||
|
||||
for name in names:
|
||||
if isinstance(value, dict):
|
||||
if name in value and value[name] is not None:
|
||||
return value[name]
|
||||
else:
|
||||
candidate = getattr(value, name, None)
|
||||
if candidate is not None:
|
||||
return candidate
|
||||
|
||||
return default
|
||||
|
||||
|
||||
def _extract_artist_names(artists: Any) -> List[str]:
|
||||
if not artists:
|
||||
return []
|
||||
|
||||
if isinstance(artists, (str, bytes)):
|
||||
artist = str(artists).strip()
|
||||
return [artist] if artist else []
|
||||
|
||||
try:
|
||||
items = list(artists)
|
||||
except TypeError:
|
||||
items = [artists]
|
||||
|
||||
names: List[str] = []
|
||||
for artist in items:
|
||||
if isinstance(artist, dict):
|
||||
name = str(_extract_value(artist, "name", "artist_name", "title", default="") or "").strip()
|
||||
else:
|
||||
candidate = getattr(artist, "name", None)
|
||||
if candidate is None:
|
||||
candidate = artist
|
||||
name = str(candidate or "").strip()
|
||||
if name:
|
||||
names.append(name)
|
||||
|
||||
return names
|
||||
|
||||
|
||||
def _normalize_album_result(album: Any, source: str) -> Dict[str, Any]:
|
||||
album_id = str(_extract_value(album, "id", "album_id", "release_id", default="") or "").strip()
|
||||
album_name = str(_extract_value(album, "name", "title", default="") or "").strip()
|
||||
artists = _extract_artist_names(_extract_value(album, "artists", default=[]))
|
||||
artist_name = ", ".join(artists) if artists else str(
|
||||
_extract_value(album, "artist_name", "artist", default="Unknown Artist") or "Unknown Artist"
|
||||
).strip()
|
||||
release_date = str(_extract_value(album, "release_date", "releaseDate", default="") or "").strip()
|
||||
album_type = str(_extract_value(album, "album_type", "type", default="album") or "album").strip() or "album"
|
||||
|
||||
total_tracks = _extract_value(album, "total_tracks", "track_count", default=0)
|
||||
if isinstance(total_tracks, (list, tuple, set)):
|
||||
total_tracks = len(total_tracks)
|
||||
try:
|
||||
total_tracks = int(total_tracks or 0)
|
||||
except (TypeError, ValueError):
|
||||
total_tracks = 0
|
||||
|
||||
image_url = _extract_value(album, "image_url", "thumb_url", "cover_image", "cover_url", default="")
|
||||
if not image_url:
|
||||
images = _extract_value(album, "images", default=[]) or []
|
||||
if isinstance(images, dict):
|
||||
images = [images]
|
||||
elif isinstance(images, (str, bytes)):
|
||||
images = [images]
|
||||
try:
|
||||
images = list(images)
|
||||
except TypeError:
|
||||
images = [images]
|
||||
if images:
|
||||
first_image = images[0]
|
||||
if isinstance(first_image, (str, bytes)):
|
||||
image_url = str(first_image).strip()
|
||||
else:
|
||||
image_url = _extract_value(first_image, "url", "image_url", "src", default="")
|
||||
|
||||
return {
|
||||
"id": album_id or album_name or "unknown-album",
|
||||
"name": album_name or album_id or "Unknown Album",
|
||||
"artist": artist_name or "Unknown Artist",
|
||||
"release_date": release_date,
|
||||
"total_tracks": total_tracks,
|
||||
"image_url": str(image_url or ""),
|
||||
"album_type": album_type,
|
||||
"source": source,
|
||||
}
|
||||
|
||||
|
||||
def _album_fingerprint(album: Dict[str, Any]) -> Tuple[str, str, str, str]:
|
||||
return (
|
||||
str(album.get("name", "") or "").strip().casefold(),
|
||||
str(album.get("artist", "") or "").strip().casefold(),
|
||||
str(album.get("release_date", "") or "").strip()[:10].casefold(),
|
||||
str(album.get("album_type", "") or "").strip().casefold(),
|
||||
)
|
||||
|
||||
|
||||
def _normalize_track_result(track: Any, source: str) -> Dict[str, Any]:
|
||||
track_id = str(_extract_value(track, "id", "track_id", "trackId", default="") or "").strip()
|
||||
track_name = str(_extract_value(track, "name", "title", "track_name", default="") or "").strip()
|
||||
artists = _extract_artist_names(_extract_value(track, "artists", default=[]))
|
||||
artist_name = ", ".join(artists) if artists else str(
|
||||
_extract_value(track, "artist", "artist_name", default="Unknown Artist") or "Unknown Artist"
|
||||
).strip()
|
||||
|
||||
album_value = _extract_value(track, "album", default=None)
|
||||
album_name = ""
|
||||
album_id = str(_extract_value(track, "album_id", "collectionId", "albumId", default="") or "").strip()
|
||||
if isinstance(album_value, dict):
|
||||
album_name = str(_extract_value(album_value, "name", "title", default="") or "").strip()
|
||||
album_id = album_id or str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip()
|
||||
if not album_name:
|
||||
album_name = album_id
|
||||
elif isinstance(album_value, (str, bytes)):
|
||||
album_name = str(album_value).strip()
|
||||
elif album_value is not None:
|
||||
album_name = str(_extract_value(album_value, "name", "title", default=album_value) or "").strip()
|
||||
if not album_id:
|
||||
album_id = str(_extract_value(album_value, "id", "album_id", "collectionId", default="") or "").strip()
|
||||
|
||||
image_url = _extract_value(track, "image_url", "thumb_url", "cover_image", default="")
|
||||
if not image_url:
|
||||
images = _extract_value(track, "images", default=[]) or []
|
||||
if isinstance(images, dict):
|
||||
images = [images]
|
||||
elif isinstance(images, (str, bytes)):
|
||||
images = [images]
|
||||
try:
|
||||
images = list(images)
|
||||
except TypeError:
|
||||
images = [images]
|
||||
if images:
|
||||
first_image = images[0]
|
||||
if isinstance(first_image, (str, bytes)):
|
||||
image_url = str(first_image).strip()
|
||||
else:
|
||||
image_url = _extract_value(first_image, "url", "image_url", "src", default="")
|
||||
if not image_url and album_value is not None:
|
||||
album_images = _extract_value(album_value, "images", default=[]) or []
|
||||
if isinstance(album_images, dict):
|
||||
album_images = [album_images]
|
||||
elif isinstance(album_images, (str, bytes)):
|
||||
album_images = [album_images]
|
||||
try:
|
||||
album_images = list(album_images)
|
||||
except TypeError:
|
||||
album_images = [album_images]
|
||||
if album_images:
|
||||
first_album_image = album_images[0]
|
||||
if isinstance(first_album_image, (str, bytes)):
|
||||
image_url = str(first_album_image).strip()
|
||||
else:
|
||||
image_url = _extract_value(first_album_image, "url", "image_url", "src", default="")
|
||||
|
||||
duration_ms = _extract_value(track, "duration_ms", "duration", "trackTimeMillis", default=0)
|
||||
try:
|
||||
duration_ms = int(duration_ms or 0)
|
||||
except (TypeError, ValueError):
|
||||
duration_ms = 0
|
||||
|
||||
track_number = _extract_value(track, "track_number", "trackNumber", default=1)
|
||||
try:
|
||||
track_number = int(track_number or 1)
|
||||
except (TypeError, ValueError):
|
||||
track_number = 1
|
||||
|
||||
return {
|
||||
"id": track_id or track_name or "unknown-track",
|
||||
"name": track_name or track_id or "Unknown Track",
|
||||
"artist": artist_name or "Unknown Artist",
|
||||
"album": album_name or "",
|
||||
"album_id": album_id or "",
|
||||
"duration_ms": duration_ms,
|
||||
"image_url": str(image_url or ""),
|
||||
"track_number": track_number,
|
||||
"source": source,
|
||||
}
|
||||
|
||||
|
||||
def _read_staging_audio_tags(file_path: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
|
||||
tags = MutagenFile(file_path, easy=True)
|
||||
if not tags:
|
||||
return None, None
|
||||
|
||||
album = (tags.get("album") or [None])[0]
|
||||
artist = (tags.get("artist") or (tags.get("albumartist") or [None]))[0]
|
||||
album_text = str(album).strip() if album else ""
|
||||
artist_text = str(artist).strip() if artist else ""
|
||||
return (album_text or None, artist_text or None)
|
||||
except Exception:
|
||||
return None, None
|
||||
|
||||
|
||||
def _collect_import_suggestion_queries(staging_path: str) -> List[str]:
|
||||
tag_albums: Dict[Tuple[str, str], int] = {}
|
||||
folder_hints: Dict[str, int] = {}
|
||||
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS]
|
||||
if not audio_files:
|
||||
continue
|
||||
|
||||
rel_dir = os.path.relpath(root, staging_path)
|
||||
if rel_dir != ".":
|
||||
top_folder = rel_dir.split(os.sep)[0]
|
||||
folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files)
|
||||
|
||||
for fname in audio_files:
|
||||
full_path = os.path.join(root, fname)
|
||||
album, artist = _read_staging_audio_tags(full_path)
|
||||
if album:
|
||||
key = (album.strip(), (artist or "").strip())
|
||||
tag_albums[key] = tag_albums.get(key, 0) + 1
|
||||
|
||||
queries: List[str] = []
|
||||
seen_lower = set()
|
||||
|
||||
for (album, artist), _count in sorted(tag_albums.items(), key=lambda item: -item[1]):
|
||||
q = f"{album} {artist}".strip() if artist else album
|
||||
if q and q.lower() not in seen_lower:
|
||||
seen_lower.add(q.lower())
|
||||
queries.append(q)
|
||||
|
||||
for folder, _count in sorted(folder_hints.items(), key=lambda item: -item[1]):
|
||||
q = folder.replace("_", " ")
|
||||
if q and q.lower() not in seen_lower:
|
||||
seen_lower.add(q.lower())
|
||||
queries.append(q)
|
||||
|
||||
return queries[:5]
|
||||
|
||||
|
||||
def search_import_albums(query: str, limit: int = 12) -> List[Dict[str, Any]]:
|
||||
"""Search albums using the configured metadata provider first."""
|
||||
query = (query or "").strip()
|
||||
if not query:
|
||||
return []
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
seen = set()
|
||||
source_chain = get_source_priority(get_primary_source())
|
||||
|
||||
for source in source_chain:
|
||||
client = get_client_for_source(source)
|
||||
if not client:
|
||||
continue
|
||||
|
||||
source_results = _search_albums_for_source(source, client, query, limit=limit)
|
||||
if not source_results:
|
||||
continue
|
||||
|
||||
added_for_source = False
|
||||
for album in source_results:
|
||||
suggestion = _normalize_album_result(album, source)
|
||||
fingerprint = _album_fingerprint(suggestion)
|
||||
if fingerprint in seen:
|
||||
continue
|
||||
seen.add(fingerprint)
|
||||
results.append(suggestion)
|
||||
added_for_source = True
|
||||
if len(results) >= limit:
|
||||
return results[:limit]
|
||||
|
||||
if added_for_source:
|
||||
break
|
||||
|
||||
return results[:limit]
|
||||
|
||||
|
||||
def search_import_tracks(query: str, limit: int = 30) -> List[Dict[str, Any]]:
|
||||
"""Search tracks using the configured metadata provider priority order."""
|
||||
query = (query or "").strip()
|
||||
if not query:
|
||||
return []
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
source_chain = get_source_priority(get_primary_source())
|
||||
|
||||
for source in source_chain:
|
||||
client = get_client_for_source(source)
|
||||
if not client:
|
||||
continue
|
||||
|
||||
source_results = _search_tracks_for_source(source, client, query, limit=limit)
|
||||
if not source_results:
|
||||
continue
|
||||
|
||||
for track in source_results:
|
||||
results.append(_normalize_track_result(track, source))
|
||||
if len(results) >= limit:
|
||||
return results[:limit]
|
||||
break
|
||||
|
||||
return results[:limit]
|
||||
|
||||
|
||||
def _build_import_suggestions_background():
|
||||
cache = _import_suggestions_cache
|
||||
|
||||
with _import_suggestions_cache_lock:
|
||||
if cache["building"]:
|
||||
return
|
||||
cache["building"] = True
|
||||
|
||||
try:
|
||||
staging_path = get_staging_path()
|
||||
if not os.path.isdir(staging_path):
|
||||
with _import_suggestions_cache_lock:
|
||||
cache["suggestions"] = []
|
||||
cache["built"] = True
|
||||
return
|
||||
|
||||
queries = _collect_import_suggestion_queries(staging_path)
|
||||
if not queries:
|
||||
with _import_suggestions_cache_lock:
|
||||
cache["suggestions"] = []
|
||||
cache["built"] = True
|
||||
return
|
||||
|
||||
suggestions: List[Dict[str, Any]] = []
|
||||
seen = set()
|
||||
for query in queries:
|
||||
try:
|
||||
albums = search_import_albums(query, limit=2)
|
||||
for album in albums:
|
||||
fingerprint = _album_fingerprint(album)
|
||||
if fingerprint in seen:
|
||||
continue
|
||||
seen.add(fingerprint)
|
||||
suggestions.append(album)
|
||||
except Exception as exc:
|
||||
logger.warning("Import suggestion search failed for %r: %s", query, exc)
|
||||
|
||||
with _import_suggestions_cache_lock:
|
||||
cache["suggestions"] = suggestions[:8]
|
||||
cache["built"] = True
|
||||
|
||||
logger.info(
|
||||
"Import suggestions cache built: %s suggestions from %s hints",
|
||||
len(cache["suggestions"]),
|
||||
len(queries),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Error building import suggestions cache: %s", exc)
|
||||
with _import_suggestions_cache_lock:
|
||||
cache["suggestions"] = []
|
||||
cache["built"] = True
|
||||
finally:
|
||||
with _import_suggestions_cache_lock:
|
||||
cache["building"] = False
|
||||
|
||||
|
||||
def start_import_suggestions_cache():
|
||||
"""Start building the import suggestions cache in a background thread."""
|
||||
threading.Thread(
|
||||
target=_build_import_suggestions_background,
|
||||
daemon=True,
|
||||
name="import-suggestions-cache",
|
||||
).start()
|
||||
|
||||
|
||||
def refresh_import_suggestions_cache():
|
||||
"""Invalidate and rebuild the suggestions cache."""
|
||||
with _import_suggestions_cache_lock:
|
||||
_import_suggestions_cache["built"] = False
|
||||
start_import_suggestions_cache()
|
||||
|
||||
|
||||
def collect_staging_files(file_paths: Optional[Iterable[str]] = None) -> List[Dict[str, Any]]:
|
||||
"""Collect audio files from the staging area with normalized metadata."""
|
||||
staging_path = get_staging_path()
|
||||
file_filter: Optional[set[str]] = set(file_paths) if file_paths else None
|
||||
staging_files: List[Dict[str, Any]] = []
|
||||
|
||||
if not os.path.isdir(staging_path):
|
||||
return staging_files
|
||||
|
||||
for root, _dirs, filenames in os.walk(staging_path):
|
||||
for filename in filenames:
|
||||
ext = os.path.splitext(filename)[1].lower()
|
||||
if ext not in AUDIO_EXTENSIONS:
|
||||
continue
|
||||
|
||||
full_path = os.path.join(root, filename)
|
||||
if file_filter is not None and full_path not in file_filter:
|
||||
continue
|
||||
|
||||
meta = read_staging_file_metadata(full_path, filename)
|
||||
staging_files.append(
|
||||
{
|
||||
"filename": filename,
|
||||
"full_path": full_path,
|
||||
"title": meta.get("title", ""),
|
||||
"artist": meta.get("albumartist") or meta.get("artist") or "",
|
||||
"album": meta.get("album", ""),
|
||||
"albumartist": meta.get("albumartist") or meta.get("artist") or "",
|
||||
"track_number": meta.get("track_number", 1),
|
||||
"disc_number": meta.get("disc_number", 1),
|
||||
}
|
||||
)
|
||||
|
||||
return staging_files
|
||||
2
core/metadata/__init__.py
Normal file
2
core/metadata/__init__.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
"""Metadata helper package."""
|
||||
|
||||
157
core/metadata/artwork.py
Normal file
157
core/metadata/artwork.py
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
"""Album artwork helpers for metadata enrichment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import urllib.request
|
||||
|
||||
from core.imports.context import get_import_context_album
|
||||
from core.metadata.common import (
|
||||
get_config_manager,
|
||||
get_image_dimensions,
|
||||
get_mutagen_symbols,
|
||||
)
|
||||
from utils.logging_config import get_logger as _create_logger
|
||||
|
||||
__all__ = [
|
||||
"embed_album_art_metadata",
|
||||
"download_cover_art",
|
||||
]
|
||||
|
||||
|
||||
logger = _create_logger("metadata.artwork")
|
||||
|
||||
|
||||
def embed_album_art_metadata(audio_file, metadata: dict):
|
||||
cfg = get_config_manager()
|
||||
symbols = get_mutagen_symbols()
|
||||
if not symbols:
|
||||
return
|
||||
|
||||
try:
|
||||
image_data = None
|
||||
mime_type = None
|
||||
|
||||
release_mbid = metadata.get("musicbrainz_release_id")
|
||||
if release_mbid and cfg.get("metadata_enhancement.prefer_caa_art", False):
|
||||
try:
|
||||
caa_url = f"https://coverartarchive.org/release/{release_mbid}/front"
|
||||
req = urllib.request.Request(caa_url, headers={"Accept": "image/*"})
|
||||
with urllib.request.urlopen(req, timeout=10) as response:
|
||||
image_data = response.read()
|
||||
mime_type = response.info().get_content_type() or "image/jpeg"
|
||||
if not image_data or len(image_data) <= 1000:
|
||||
image_data = None
|
||||
except Exception:
|
||||
image_data = None
|
||||
|
||||
if not image_data:
|
||||
art_url = metadata.get("album_art_url")
|
||||
if not art_url:
|
||||
logger.warning("No album art URL available for embedding.")
|
||||
return
|
||||
with urllib.request.urlopen(art_url, timeout=10) as response:
|
||||
image_data = response.read()
|
||||
mime_type = response.info().get_content_type() or "image/jpeg"
|
||||
|
||||
if not image_data:
|
||||
logger.error("Failed to download album art data.")
|
||||
return
|
||||
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.APIC(encoding=3, mime=mime_type, type=3, desc="Cover", data=image_data))
|
||||
elif isinstance(audio_file, symbols.FLAC):
|
||||
picture = symbols.Picture()
|
||||
picture.data = image_data
|
||||
picture.type = 3
|
||||
picture.mime = mime_type
|
||||
width, height = get_image_dimensions(image_data)
|
||||
picture.width = width or 640
|
||||
picture.height = height or 640
|
||||
picture.depth = 24
|
||||
audio_file.add_picture(picture)
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
fmt = symbols.MP4Cover.FORMAT_JPEG if "jpeg" in mime_type else symbols.MP4Cover.FORMAT_PNG
|
||||
audio_file["covr"] = [symbols.MP4Cover(image_data, imageformat=fmt)]
|
||||
|
||||
logger.info("Album art successfully embedded.")
|
||||
except Exception as exc:
|
||||
logger.error("Error embedding album art: %s", exc)
|
||||
|
||||
|
||||
def download_cover_art(album_info: dict, target_dir: str, context: dict = None):
|
||||
cfg = get_config_manager()
|
||||
if cfg.get("metadata_enhancement.cover_art_download", True) is False:
|
||||
return
|
||||
|
||||
try:
|
||||
cover_path = os.path.join(target_dir, "cover.jpg")
|
||||
album_info = album_info or {}
|
||||
release_mbid = album_info.get("musicbrainz_release_id")
|
||||
prefer_caa = cfg.get("metadata_enhancement.prefer_caa_art", False)
|
||||
|
||||
if os.path.exists(cover_path):
|
||||
if release_mbid and prefer_caa:
|
||||
try:
|
||||
existing_size = os.path.getsize(cover_path)
|
||||
if existing_size > 200_000:
|
||||
return
|
||||
is_upgrade = True
|
||||
except Exception:
|
||||
return
|
||||
else:
|
||||
return
|
||||
else:
|
||||
is_upgrade = False
|
||||
|
||||
image_data = None
|
||||
if release_mbid and prefer_caa:
|
||||
try:
|
||||
caa_url = f"https://coverartarchive.org/release/{release_mbid}/front"
|
||||
req = urllib.request.Request(caa_url, headers={"Accept": "image/*"})
|
||||
with urllib.request.urlopen(req, timeout=10) as response:
|
||||
image_data = response.read()
|
||||
if not image_data or len(image_data) <= 1000:
|
||||
image_data = None
|
||||
except Exception:
|
||||
image_data = None
|
||||
|
||||
if is_upgrade and not image_data:
|
||||
logger.error("CAA upgrade failed - keeping existing cover.jpg")
|
||||
return
|
||||
|
||||
if not image_data:
|
||||
art_url = album_info.get("album_image_url")
|
||||
if not art_url and context:
|
||||
album_ctx = get_import_context_album(context)
|
||||
art_url = album_ctx.get("image_url")
|
||||
if not art_url and album_ctx.get("images"):
|
||||
images = album_ctx.get("images", [])
|
||||
if images and isinstance(images[0], dict):
|
||||
art_url = images[0].get("url", "")
|
||||
if art_url:
|
||||
logger.info("Using cover art URL from album context")
|
||||
if art_url and "i.scdn.co" in art_url:
|
||||
try:
|
||||
from core.spotify_client import _upgrade_spotify_image_url
|
||||
|
||||
art_url = _upgrade_spotify_image_url(art_url)
|
||||
except Exception:
|
||||
pass
|
||||
elif art_url and "mzstatic.com" in art_url:
|
||||
art_url = re.sub(r"\d+x\d+bb", "3000x3000bb", art_url)
|
||||
if not art_url:
|
||||
logger.warning("No cover art URL available for download.")
|
||||
return
|
||||
with urllib.request.urlopen(art_url, timeout=10) as response:
|
||||
image_data = response.read()
|
||||
|
||||
if not image_data:
|
||||
return
|
||||
|
||||
with open(cover_path, "wb") as handle:
|
||||
handle.write(image_data)
|
||||
logger.info("Cover art downloaded to: %s", cover_path)
|
||||
except Exception as exc:
|
||||
logger.error("Error downloading cover.jpg: %s", exc)
|
||||
267
core/metadata/common.py
Normal file
267
core/metadata/common.py
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
"""Shared low-level helpers for metadata enrichment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import weakref
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from utils.logging_config import get_logger as _create_logger
|
||||
|
||||
|
||||
logger = _create_logger("metadata.common")
|
||||
|
||||
__all__ = [
|
||||
"get_logger",
|
||||
"get_config_manager",
|
||||
"get_mutagen_symbols",
|
||||
"get_file_lock",
|
||||
"is_ogg_opus",
|
||||
"is_vorbis_like",
|
||||
"save_audio_file",
|
||||
"get_image_dimensions",
|
||||
"strip_all_non_audio_tags",
|
||||
"verify_metadata_written",
|
||||
"wipe_source_tags",
|
||||
]
|
||||
|
||||
_FILE_LOCKS: "weakref.WeakValueDictionary[str, threading.Lock]" = weakref.WeakValueDictionary()
|
||||
_FILE_LOCKS_LOCK = threading.Lock()
|
||||
|
||||
|
||||
class _NullConfigManager:
|
||||
def get(self, _key: str, default: Any = None) -> Any:
|
||||
return default
|
||||
|
||||
|
||||
def get_logger():
|
||||
return logger
|
||||
|
||||
|
||||
def get_config_manager():
|
||||
try:
|
||||
from config.settings import config_manager as settings_config_manager
|
||||
|
||||
return settings_config_manager
|
||||
except Exception:
|
||||
return _NullConfigManager()
|
||||
|
||||
|
||||
def get_mutagen_symbols():
|
||||
"""Lazy mutagen import so tests can monkeypatch this without the package installed."""
|
||||
try:
|
||||
from mutagen import File as MutagenFile
|
||||
from mutagen.apev2 import APEv2, APENoHeaderError
|
||||
from mutagen.flac import FLAC, Picture
|
||||
from mutagen.id3 import (
|
||||
APIC,
|
||||
ID3,
|
||||
TBPM,
|
||||
TCOP,
|
||||
TDOR,
|
||||
TDRC,
|
||||
TCON,
|
||||
TIT2,
|
||||
TALB,
|
||||
TPE1,
|
||||
TPE2,
|
||||
TPOS,
|
||||
TPUB,
|
||||
TRCK,
|
||||
TSRC,
|
||||
TXXX,
|
||||
UFID,
|
||||
TMED,
|
||||
)
|
||||
from mutagen.mp4 import MP4, MP4Cover, MP4FreeForm
|
||||
from mutagen.oggvorbis import OggVorbis
|
||||
try:
|
||||
from mutagen.oggopus import OggOpus
|
||||
except Exception:
|
||||
OggOpus = None
|
||||
except Exception as exc:
|
||||
logger.debug("Mutagen unavailable for metadata enrichment: %s", exc)
|
||||
return None
|
||||
|
||||
return SimpleNamespace(
|
||||
File=MutagenFile,
|
||||
APEv2=APEv2,
|
||||
APENoHeaderError=APENoHeaderError,
|
||||
FLAC=FLAC,
|
||||
Picture=Picture,
|
||||
ID3=ID3,
|
||||
APIC=APIC,
|
||||
TBPM=TBPM,
|
||||
TCOP=TCOP,
|
||||
TDOR=TDOR,
|
||||
TDRC=TDRC,
|
||||
TCON=TCON,
|
||||
TIT2=TIT2,
|
||||
TALB=TALB,
|
||||
TPE1=TPE1,
|
||||
TPE2=TPE2,
|
||||
TPOS=TPOS,
|
||||
TPUB=TPUB,
|
||||
TRCK=TRCK,
|
||||
TSRC=TSRC,
|
||||
TXXX=TXXX,
|
||||
UFID=UFID,
|
||||
TMED=TMED,
|
||||
MP4=MP4,
|
||||
MP4Cover=MP4Cover,
|
||||
MP4FreeForm=MP4FreeForm,
|
||||
OggVorbis=OggVorbis,
|
||||
OggOpus=OggOpus,
|
||||
)
|
||||
|
||||
|
||||
def get_file_lock(file_path: str) -> threading.Lock:
|
||||
# Keep a per-path lock while it is actively referenced, but let it
|
||||
# fall out of the cache once nobody is using it anymore.
|
||||
with _FILE_LOCKS_LOCK:
|
||||
lock = _FILE_LOCKS.get(file_path)
|
||||
if lock is None:
|
||||
lock = threading.Lock()
|
||||
_FILE_LOCKS[file_path] = lock
|
||||
return lock
|
||||
|
||||
|
||||
def is_ogg_opus(audio_file: Any) -> bool:
|
||||
return type(audio_file).__name__ == "OggOpus"
|
||||
|
||||
|
||||
def is_vorbis_like(audio_file: Any, symbols: Any) -> bool:
|
||||
vorbis_classes = tuple(
|
||||
cls for cls in (
|
||||
getattr(symbols, "FLAC", None),
|
||||
getattr(symbols, "OggVorbis", None),
|
||||
) if cls is not None
|
||||
)
|
||||
return bool(vorbis_classes) and isinstance(audio_file, vorbis_classes) or is_ogg_opus(audio_file)
|
||||
|
||||
|
||||
def save_audio_file(audio_file: Any, symbols: Any) -> None:
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.save(v1=0, v2_version=4)
|
||||
elif isinstance(audio_file, symbols.FLAC):
|
||||
audio_file.save(deleteid3=True)
|
||||
else:
|
||||
audio_file.save()
|
||||
|
||||
|
||||
def get_image_dimensions(data: bytes):
|
||||
try:
|
||||
if data[:8] == b"\x89PNG\r\n\x1a\n":
|
||||
import struct
|
||||
|
||||
w, h = struct.unpack(">II", data[16:24])
|
||||
return w, h
|
||||
if data[:2] == b"\xff\xd8":
|
||||
import struct
|
||||
|
||||
i = 2
|
||||
while i < len(data) - 9:
|
||||
if data[i] != 0xFF:
|
||||
break
|
||||
marker = data[i + 1]
|
||||
if marker in (0xC0, 0xC2):
|
||||
h, w = struct.unpack(">HH", data[i + 5 : i + 9])
|
||||
return w, h
|
||||
length = struct.unpack(">H", data[i + 2 : i + 4])[0]
|
||||
i += 2 + length
|
||||
except Exception:
|
||||
pass
|
||||
return None, None
|
||||
|
||||
|
||||
def strip_all_non_audio_tags(file_path: str) -> dict:
|
||||
summary = {"apev2_stripped": False, "apev2_tag_count": 0}
|
||||
if os.path.splitext(file_path)[1].lower() != ".mp3":
|
||||
return summary
|
||||
|
||||
symbols = get_mutagen_symbols()
|
||||
if not symbols:
|
||||
return summary
|
||||
|
||||
try:
|
||||
apev2_tags = symbols.APEv2(file_path)
|
||||
tag_count = len(apev2_tags)
|
||||
tag_keys = list(apev2_tags.keys())
|
||||
apev2_tags.delete(file_path)
|
||||
summary["apev2_stripped"] = True
|
||||
summary["apev2_tag_count"] = tag_count
|
||||
logger.info("Stripped %s APEv2 tags: %s", tag_count, ", ".join(tag_keys[:10]))
|
||||
except symbols.APENoHeaderError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.error("Could not strip APEv2 tags (non-fatal): %s", exc)
|
||||
return summary
|
||||
|
||||
|
||||
def verify_metadata_written(file_path: str) -> bool:
|
||||
symbols = get_mutagen_symbols()
|
||||
if not symbols:
|
||||
return False
|
||||
|
||||
try:
|
||||
check = symbols.File(file_path)
|
||||
if check is None or check.tags is None:
|
||||
logger.info("[VERIFY] Tags are None after save: %s", file_path)
|
||||
return False
|
||||
|
||||
title_found = False
|
||||
artist_found = False
|
||||
if isinstance(check.tags, symbols.ID3):
|
||||
title_found = bool(check.tags.getall("TIT2"))
|
||||
artist_found = bool(check.tags.getall("TPE1"))
|
||||
try:
|
||||
symbols.APEv2(file_path)
|
||||
logger.info("[VERIFY] APEv2 tags still present after processing!")
|
||||
return False
|
||||
except symbols.APENoHeaderError:
|
||||
pass
|
||||
elif is_vorbis_like(check, symbols):
|
||||
title_found = bool(check.get("title"))
|
||||
artist_found = bool(check.get("artist"))
|
||||
elif isinstance(check, symbols.MP4):
|
||||
title_found = bool(check.get("\xa9nam"))
|
||||
artist_found = bool(check.get("\xa9ART"))
|
||||
|
||||
if not title_found or not artist_found:
|
||||
logger.warning("[VERIFY] Missing metadata - title:%s artist:%s", title_found, artist_found)
|
||||
return False
|
||||
|
||||
logger.info("[VERIFY] Metadata verified OK")
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error("[VERIFY] Verification error (non-fatal): %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
def wipe_source_tags(file_path: str) -> bool:
|
||||
try:
|
||||
strip_all_non_audio_tags(file_path)
|
||||
symbols = get_mutagen_symbols()
|
||||
if not symbols:
|
||||
return False
|
||||
|
||||
audio = symbols.File(file_path)
|
||||
if audio is None:
|
||||
return False
|
||||
if hasattr(audio, "clear_pictures"):
|
||||
audio.clear_pictures()
|
||||
if audio.tags is not None:
|
||||
tag_count = len(audio.tags)
|
||||
audio.tags.clear()
|
||||
else:
|
||||
audio.add_tags()
|
||||
tag_count = 0
|
||||
save_audio_file(audio, symbols)
|
||||
if tag_count > 0:
|
||||
logger.info("[Tag Wipe] Stripped %s source tags from: %s", tag_count, os.path.basename(file_path))
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error("[Tag Wipe] Failed (non-fatal): %s", exc)
|
||||
return False
|
||||
194
core/metadata/enrichment.py
Normal file
194
core/metadata/enrichment.py
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
"""Compatibility facade and orchestration for metadata enrichment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from core.metadata.artwork import embed_album_art_metadata
|
||||
from core.metadata.common import (
|
||||
get_config_manager,
|
||||
get_file_lock,
|
||||
get_mutagen_symbols,
|
||||
is_vorbis_like,
|
||||
save_audio_file,
|
||||
strip_all_non_audio_tags,
|
||||
verify_metadata_written,
|
||||
)
|
||||
from core.metadata.source import embed_source_ids, extract_source_metadata
|
||||
from utils.logging_config import get_logger as _create_logger
|
||||
|
||||
|
||||
__all__ = [
|
||||
"build_metadata_enrichment_runtime",
|
||||
"enhance_file_metadata",
|
||||
"extract_source_metadata",
|
||||
"embed_source_ids",
|
||||
]
|
||||
|
||||
|
||||
logger = _create_logger("metadata.enrichment")
|
||||
|
||||
|
||||
def build_metadata_enrichment_runtime(
|
||||
*,
|
||||
mb_worker: Any | None = None,
|
||||
deezer_worker: Any | None = None,
|
||||
audiodb_worker: Any | None = None,
|
||||
tidal_client: Any | None = None,
|
||||
qobuz_enrichment_worker: Any | None = None,
|
||||
lastfm_worker: Any | None = None,
|
||||
genius_worker: Any | None = None,
|
||||
spotify_enrichment_worker: Any | None = None,
|
||||
itunes_enrichment_worker: Any | None = None,
|
||||
) -> SimpleNamespace:
|
||||
"""Build the runtime object consumed by core.metadata.enrichment/source."""
|
||||
return SimpleNamespace(
|
||||
mb_worker=mb_worker,
|
||||
deezer_worker=deezer_worker,
|
||||
audiodb_worker=audiodb_worker,
|
||||
tidal_client=tidal_client,
|
||||
qobuz_enrichment_worker=qobuz_enrichment_worker,
|
||||
lastfm_worker=lastfm_worker,
|
||||
genius_worker=genius_worker,
|
||||
spotify_enrichment_worker=spotify_enrichment_worker,
|
||||
itunes_enrichment_worker=itunes_enrichment_worker,
|
||||
)
|
||||
|
||||
|
||||
def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool:
|
||||
cfg = get_config_manager()
|
||||
if cfg.get("metadata_enhancement.enabled", True) is False:
|
||||
logger.warning("Metadata enhancement disabled in config.")
|
||||
return True
|
||||
|
||||
if album_info is None:
|
||||
album_info = {}
|
||||
|
||||
symbols = get_mutagen_symbols()
|
||||
if not symbols:
|
||||
logger.error("Mutagen is unavailable, cannot enhance metadata.")
|
||||
return False
|
||||
|
||||
file_lock = get_file_lock(file_path)
|
||||
with file_lock:
|
||||
logger.info("Enhancing metadata for: %s", os.path.basename(file_path))
|
||||
try:
|
||||
strip_all_non_audio_tags(file_path)
|
||||
audio_file = symbols.File(file_path)
|
||||
if audio_file is None:
|
||||
logger.error("Could not load audio file with Mutagen: %s", file_path)
|
||||
return False
|
||||
|
||||
if hasattr(audio_file, "clear_pictures"):
|
||||
audio_file.clear_pictures()
|
||||
|
||||
if audio_file.tags is not None:
|
||||
if len(audio_file.tags) > 0:
|
||||
tag_keys = list(audio_file.tags.keys())[:15]
|
||||
logger.info("Clearing %s existing tags: %s", len(audio_file.tags), ", ".join(str(k) for k in tag_keys))
|
||||
audio_file.tags.clear()
|
||||
else:
|
||||
audio_file.add_tags()
|
||||
|
||||
save_audio_file(audio_file, symbols)
|
||||
|
||||
metadata = extract_source_metadata(context, artist, album_info)
|
||||
if not metadata:
|
||||
logger.error("Could not extract source metadata, saving with cleared tags.")
|
||||
save_audio_file(audio_file, symbols)
|
||||
return True
|
||||
|
||||
track_num_str = f"{metadata.get('track_number', 1)}/{metadata.get('total_tracks', 1)}"
|
||||
write_multi = cfg.get("metadata_enhancement.tags.write_multi_artist", False)
|
||||
artists_list = metadata.get("_artists_list", [])
|
||||
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
if metadata.get("title"):
|
||||
audio_file.tags.add(symbols.TIT2(encoding=3, text=[metadata["title"]]))
|
||||
if metadata.get("artist"):
|
||||
audio_file.tags.add(symbols.TPE1(encoding=3, text=[metadata["artist"]]))
|
||||
if write_multi and len(artists_list) > 1:
|
||||
audio_file.tags.add(symbols.TPE1(encoding=3, text=artists_list))
|
||||
if metadata.get("album_artist"):
|
||||
audio_file.tags.add(symbols.TPE2(encoding=3, text=[metadata["album_artist"]]))
|
||||
if metadata.get("album"):
|
||||
audio_file.tags.add(symbols.TALB(encoding=3, text=[metadata["album"]]))
|
||||
if metadata.get("date"):
|
||||
audio_file.tags.add(symbols.TDRC(encoding=3, text=[metadata["date"]]))
|
||||
if metadata.get("genre"):
|
||||
audio_file.tags.add(symbols.TCON(encoding=3, text=[metadata["genre"]]))
|
||||
audio_file.tags.add(symbols.TRCK(encoding=3, text=[track_num_str]))
|
||||
if metadata.get("disc_number"):
|
||||
audio_file.tags.add(symbols.TPOS(encoding=3, text=[str(metadata["disc_number"])]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
if metadata.get("title"):
|
||||
audio_file["title"] = [metadata["title"]]
|
||||
if metadata.get("artist"):
|
||||
audio_file["artist"] = [metadata["artist"]]
|
||||
if write_multi and len(artists_list) > 1:
|
||||
audio_file["artists"] = artists_list
|
||||
if metadata.get("album_artist"):
|
||||
audio_file["albumartist"] = [metadata["album_artist"]]
|
||||
if metadata.get("album"):
|
||||
audio_file["album"] = [metadata["album"]]
|
||||
if metadata.get("date"):
|
||||
audio_file["date"] = [metadata["date"]]
|
||||
if metadata.get("genre"):
|
||||
audio_file["genre"] = [metadata["genre"]]
|
||||
audio_file["tracknumber"] = [track_num_str]
|
||||
if metadata.get("disc_number"):
|
||||
audio_file["discnumber"] = [str(metadata["disc_number"])]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
if metadata.get("title"):
|
||||
audio_file["\xa9nam"] = [metadata["title"]]
|
||||
if metadata.get("artist"):
|
||||
audio_file["\xa9ART"] = artists_list if (write_multi and len(artists_list) > 1) else [metadata["artist"]]
|
||||
if metadata.get("album_artist"):
|
||||
audio_file["aART"] = [metadata["album_artist"]]
|
||||
if metadata.get("album"):
|
||||
audio_file["\xa9alb"] = [metadata["album"]]
|
||||
if metadata.get("date"):
|
||||
audio_file["\xa9day"] = [metadata["date"]]
|
||||
if metadata.get("genre"):
|
||||
audio_file["\xa9gen"] = [metadata["genre"]]
|
||||
audio_file["trkn"] = [(metadata.get("track_number", 1), metadata.get("total_tracks", 1))]
|
||||
if metadata.get("disc_number"):
|
||||
audio_file["disk"] = [(metadata["disc_number"], 0)]
|
||||
|
||||
embed_source_ids(audio_file, metadata, context, runtime=runtime)
|
||||
|
||||
if album_info is not None and metadata.get("musicbrainz_release_id"):
|
||||
album_info["musicbrainz_release_id"] = metadata["musicbrainz_release_id"]
|
||||
|
||||
if cfg.get("metadata_enhancement.embed_album_art", True):
|
||||
embed_album_art_metadata(audio_file, metadata)
|
||||
|
||||
quality = context.get("_audio_quality", "")
|
||||
if quality and cfg.get("metadata_enhancement.tags.quality_tag", True) is not False:
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.TXXX(encoding=3, desc="QUALITY", text=[quality]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
audio_file["quality"] = [quality]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
audio_file["----:com.apple.iTunes:QUALITY"] = [symbols.MP4FreeForm(quality.encode("utf-8"))]
|
||||
|
||||
save_audio_file(audio_file, symbols)
|
||||
|
||||
verified = verify_metadata_written(file_path)
|
||||
if verified:
|
||||
logger.info("Metadata enhanced successfully.")
|
||||
else:
|
||||
logger.info("Metadata saved but verification found issues (see above).")
|
||||
return True
|
||||
except Exception as exc:
|
||||
import traceback
|
||||
|
||||
logger.error("Error enhancing metadata for %s: %s", file_path, exc)
|
||||
logger.error("[Metadata Debug] Exception type: %s", type(exc).__name__)
|
||||
logger.info("[Metadata Debug] File exists: %s", os.path.exists(file_path))
|
||||
logger.warning("[Metadata Debug] Artist: %s", artist.get("name", "MISSING") if artist else "None")
|
||||
logger.warning("[Metadata Debug] Album info: %s", album_info.get("album_name", "MISSING") if album_info else "None")
|
||||
logger.error("[Metadata Debug] Traceback:\n%s", traceback.format_exc())
|
||||
return False
|
||||
70
core/metadata/lyrics.py
Normal file
70
core/metadata/lyrics.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
"""Lyrics export helpers for metadata enrichment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.imports.context import (
|
||||
get_import_clean_album,
|
||||
get_import_clean_title,
|
||||
get_import_context_album,
|
||||
get_import_original_search,
|
||||
normalize_import_context,
|
||||
)
|
||||
from core.metadata.common import get_config_manager
|
||||
from utils.logging_config import get_logger as _create_logger
|
||||
|
||||
__all__ = [
|
||||
"generate_lrc_file",
|
||||
]
|
||||
|
||||
|
||||
logger = _create_logger("metadata.lyrics")
|
||||
|
||||
|
||||
def generate_lrc_file(file_path: str, context: dict, artist: dict, album_info: dict) -> bool:
|
||||
cfg = get_config_manager()
|
||||
if cfg.get("metadata_enhancement.lrclib_enabled", True) is False:
|
||||
return False
|
||||
|
||||
try:
|
||||
from core.lyrics_client import lyrics_client
|
||||
|
||||
context = normalize_import_context(context)
|
||||
original_search = get_import_original_search(context)
|
||||
album_context = get_import_context_album(context)
|
||||
track_name = get_import_clean_title(context, default=original_search.get("title", "Unknown Track"))
|
||||
|
||||
if isinstance(artist, dict):
|
||||
artist_name = artist.get("name", "Unknown Artist")
|
||||
elif hasattr(artist, "name"):
|
||||
artist_name = artist.name
|
||||
else:
|
||||
artist_name = str(artist) if artist else "Unknown Artist"
|
||||
|
||||
album_name = None
|
||||
duration_seconds = None
|
||||
if album_info and album_info.get("is_album"):
|
||||
album_name = (
|
||||
get_import_clean_album(context, album_info=album_info, default="")
|
||||
or album_info.get("album_name")
|
||||
or album_context.get("name")
|
||||
)
|
||||
|
||||
if original_search.get("duration_ms"):
|
||||
duration_seconds = int(original_search["duration_ms"] / 1000)
|
||||
|
||||
success = lyrics_client.create_lrc_file(
|
||||
audio_file_path=file_path,
|
||||
track_name=track_name,
|
||||
artist_name=artist_name,
|
||||
album_name=album_name,
|
||||
duration_seconds=duration_seconds,
|
||||
)
|
||||
|
||||
if success:
|
||||
logger.info("LRC file generated for: %s", track_name)
|
||||
else:
|
||||
logger.warning("No lyrics found for: %s", track_name)
|
||||
return success
|
||||
except Exception as exc:
|
||||
logger.error("Error generating LRC file for %s: %s", file_path, exc)
|
||||
return False
|
||||
974
core/metadata/source.py
Normal file
974
core/metadata/source.py
Normal file
|
|
@ -0,0 +1,974 @@
|
|||
"""Source metadata extraction and source-ID embedding helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from typing import Any, Dict
|
||||
|
||||
import requests
|
||||
|
||||
from core.imports.context import (
|
||||
extract_artist_name,
|
||||
get_import_clean_artist,
|
||||
get_import_clean_title,
|
||||
get_import_context_album,
|
||||
get_import_original_search,
|
||||
get_import_source,
|
||||
get_import_source_ids,
|
||||
get_import_track_info,
|
||||
get_source_tag_names,
|
||||
normalize_import_context,
|
||||
)
|
||||
from core.metadata_service import get_itunes_client
|
||||
from database.music_database import get_database
|
||||
from core.metadata.common import (
|
||||
get_config_manager,
|
||||
get_mutagen_symbols,
|
||||
is_vorbis_like,
|
||||
)
|
||||
from utils.logging_config import get_logger as _create_logger
|
||||
|
||||
__all__ = [
|
||||
"extract_source_metadata",
|
||||
"embed_source_ids",
|
||||
"normalize_album_cache_key",
|
||||
"mb_release_cache",
|
||||
"mb_release_cache_lock",
|
||||
"mb_release_detail_cache",
|
||||
"mb_release_detail_cache_lock",
|
||||
]
|
||||
|
||||
_MB_RELEASE_CACHE_MAX_ENTRIES = 4096
|
||||
_MB_RELEASE_DETAIL_CACHE_MAX_ENTRIES = 4096
|
||||
|
||||
mb_release_cache: "OrderedDict[tuple, str]" = OrderedDict()
|
||||
mb_release_cache_lock = threading.RLock()
|
||||
mb_release_detail_cache: "OrderedDict[str, Dict[str, Any]]" = OrderedDict()
|
||||
mb_release_detail_cache_lock = threading.RLock()
|
||||
logger = _create_logger("metadata.source")
|
||||
|
||||
_SOURCE_NETWORK_EXCEPTIONS = (requests.RequestException, socket.timeout, TimeoutError)
|
||||
|
||||
_EDITION_PAREN_RE = re.compile(
|
||||
r'\s*[\(\[]\s*(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|'
|
||||
r'limited|bonus|platinum|gold|super\s*deluxe|standard)'
|
||||
r'(?:\s+(?:edition|version))?[^)\]]*[\)\]]',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_EDITION_BARE_RE = re.compile(
|
||||
r'\s+(?:-\s+)?(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|'
|
||||
r'limited|bonus|platinum|gold|super\s*deluxe|standard)'
|
||||
r'(?:\s+(?:edition|version))?\s*$',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def normalize_album_cache_key(album_name: str) -> str:
|
||||
result = _EDITION_PAREN_RE.sub("", album_name or "")
|
||||
result = _EDITION_BARE_RE.sub("", result)
|
||||
return result.lower().strip()
|
||||
|
||||
|
||||
def _bounded_cache_get(cache, key):
|
||||
value = cache.get(key)
|
||||
if value is not None and hasattr(cache, "move_to_end"):
|
||||
cache.move_to_end(key)
|
||||
return value
|
||||
|
||||
|
||||
def _bounded_cache_set(cache, key, value, max_entries: int) -> None:
|
||||
cache[key] = value
|
||||
if hasattr(cache, "move_to_end"):
|
||||
cache.move_to_end(key)
|
||||
while len(cache) > max_entries:
|
||||
cache.popitem(last=False)
|
||||
|
||||
|
||||
def _call_source_lookup(label: str, func, *args, **kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except _SOURCE_NETWORK_EXCEPTIONS as exc:
|
||||
logger.warning("%s lookup failed (network): %s", label, exc)
|
||||
return None
|
||||
|
||||
|
||||
SOURCE_TAG_CONFIG = {
|
||||
"SPOTIFY_TRACK_ID": "spotify.tags.track_id",
|
||||
"SPOTIFY_ARTIST_ID": "spotify.tags.artist_id",
|
||||
"SPOTIFY_ALBUM_ID": "spotify.tags.album_id",
|
||||
"ITUNES_TRACK_ID": "itunes.tags.track_id",
|
||||
"ITUNES_ARTIST_ID": "itunes.tags.artist_id",
|
||||
"ITUNES_ALBUM_ID": "itunes.tags.album_id",
|
||||
"MUSICBRAINZ_RECORDING_ID": "musicbrainz.tags.recording_id",
|
||||
"MUSICBRAINZ_ARTIST_ID": "musicbrainz.tags.artist_id",
|
||||
"MUSICBRAINZ_RELEASE_ID": "musicbrainz.tags.release_id",
|
||||
"MUSICBRAINZ_RELEASEGROUPID": "musicbrainz.tags.release_group_id",
|
||||
"MUSICBRAINZ_ALBUMARTISTID": "musicbrainz.tags.album_artist_id",
|
||||
"MUSICBRAINZ_RELEASETRACKID": "musicbrainz.tags.release_track_id",
|
||||
"RELEASETYPE": "musicbrainz.tags.release_type",
|
||||
"ORIGINALDATE": "musicbrainz.tags.original_date",
|
||||
"RELEASESTATUS": "musicbrainz.tags.release_status",
|
||||
"RELEASECOUNTRY": "musicbrainz.tags.release_country",
|
||||
"BARCODE": "musicbrainz.tags.barcode",
|
||||
"MEDIA": "musicbrainz.tags.media",
|
||||
"TOTALDISCS": "musicbrainz.tags.total_discs",
|
||||
"CATALOGNUMBER": "musicbrainz.tags.catalog_number",
|
||||
"SCRIPT": "musicbrainz.tags.script",
|
||||
"ASIN": "musicbrainz.tags.asin",
|
||||
"DEEZER_TRACK_ID": "deezer.tags.track_id",
|
||||
"DEEZER_ARTIST_ID": "deezer.tags.artist_id",
|
||||
"AUDIODB_TRACK_ID": "audiodb.tags.track_id",
|
||||
"TIDAL_TRACK_ID": "tidal.tags.track_id",
|
||||
"TIDAL_ARTIST_ID": "tidal.tags.artist_id",
|
||||
"QOBUZ_TRACK_ID": "qobuz.tags.track_id",
|
||||
"QOBUZ_ARTIST_ID": "qobuz.tags.artist_id",
|
||||
"GENIUS_TRACK_ID": "genius.tags.track_id",
|
||||
}
|
||||
|
||||
DEFAULT_SOURCE_ORDER = ["musicbrainz", "deezer", "audiodb", "tidal", "qobuz", "lastfm", "genius"]
|
||||
|
||||
ID3_TAG_MAP = {
|
||||
"MUSICBRAINZ_RECORDING_ID": ("UFID", "http://musicbrainz.org"),
|
||||
"MUSICBRAINZ_ARTIST_ID": ("TXXX", "MusicBrainz Artist Id"),
|
||||
"MUSICBRAINZ_RELEASE_ID": ("TXXX", "MusicBrainz Album Id"),
|
||||
"MUSICBRAINZ_RELEASEGROUPID": ("TXXX", "MusicBrainz Release Group Id"),
|
||||
"MUSICBRAINZ_ALBUMARTISTID": ("TXXX", "MusicBrainz Album Artist Id"),
|
||||
"MUSICBRAINZ_RELEASETRACKID": ("TXXX", "MusicBrainz Release Track Id"),
|
||||
"RELEASETYPE": ("TXXX", "MusicBrainz Album Type"),
|
||||
"RELEASESTATUS": ("TXXX", "MusicBrainz Album Status"),
|
||||
"RELEASECOUNTRY": ("TXXX", "MusicBrainz Album Release Country"),
|
||||
"ORIGINALDATE": ("TDOR", None),
|
||||
"MEDIA": ("TMED", None),
|
||||
}
|
||||
|
||||
VORBIS_TAG_MAP = {
|
||||
"MUSICBRAINZ_RECORDING_ID": "MUSICBRAINZ_TRACKID",
|
||||
"MUSICBRAINZ_ARTIST_ID": "MUSICBRAINZ_ARTISTID",
|
||||
"MUSICBRAINZ_RELEASE_ID": "MUSICBRAINZ_ALBUMID",
|
||||
"MUSICBRAINZ_RELEASEGROUPID": "MUSICBRAINZ_RELEASEGROUPID",
|
||||
"MUSICBRAINZ_ALBUMARTISTID": "MUSICBRAINZ_ALBUMARTISTID",
|
||||
"MUSICBRAINZ_RELEASETRACKID": "MUSICBRAINZ_RELEASETRACKID",
|
||||
}
|
||||
|
||||
MP4_TAG_MAP = {
|
||||
"MUSICBRAINZ_RECORDING_ID": "MusicBrainz Track Id",
|
||||
"MUSICBRAINZ_ARTIST_ID": "MusicBrainz Artist Id",
|
||||
"MUSICBRAINZ_RELEASE_ID": "MusicBrainz Album Id",
|
||||
"MUSICBRAINZ_RELEASEGROUPID": "MusicBrainz Release Group Id",
|
||||
"MUSICBRAINZ_ALBUMARTISTID": "MusicBrainz Album Artist Id",
|
||||
"MUSICBRAINZ_RELEASETRACKID": "MusicBrainz Release Track Id",
|
||||
"RELEASETYPE": "MusicBrainz Album Type",
|
||||
"RELEASESTATUS": "MusicBrainz Album Status",
|
||||
"RELEASECOUNTRY": "MusicBrainz Album Release Country",
|
||||
}
|
||||
|
||||
|
||||
def _tag_enabled(cfg, path: str) -> bool:
|
||||
return cfg.get(path, True) is not False
|
||||
|
||||
|
||||
def _names_match(a: str, b: str, threshold: float = 0.75) -> bool:
|
||||
if not a or not b:
|
||||
return False
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
norm = lambda s: re.sub(r"[^a-z0-9 ]", "", re.sub(r"\(.*?\)", "", s).lower()).strip()
|
||||
return SequenceMatcher(None, norm(a), norm(b)).ratio() >= threshold
|
||||
|
||||
|
||||
def _collect_source_ids(metadata: dict, cfg) -> dict:
|
||||
source_ids = {}
|
||||
source = (metadata.get("source") or "").strip().lower()
|
||||
if source:
|
||||
source_tag_names = get_source_tag_names(source)
|
||||
source_track_id = metadata.get("source_track_id")
|
||||
source_artist_id = metadata.get("source_artist_id")
|
||||
source_album_id = metadata.get("source_album_id")
|
||||
if cfg.get(f"{source}.embed_tags", True) is not False:
|
||||
if source_tag_names.get("track") and source_track_id:
|
||||
source_ids[source_tag_names["track"]] = source_track_id
|
||||
if source_tag_names.get("artist") and source_artist_id:
|
||||
source_ids[source_tag_names["artist"]] = source_artist_id
|
||||
if source_tag_names.get("album") and source_album_id:
|
||||
source_ids[source_tag_names["album"]] = source_album_id
|
||||
|
||||
if not source_ids:
|
||||
if cfg.get("spotify.embed_tags", True) is not False:
|
||||
if metadata.get("spotify_track_id"):
|
||||
source_ids["SPOTIFY_TRACK_ID"] = metadata["spotify_track_id"]
|
||||
if metadata.get("spotify_artist_id"):
|
||||
source_ids["SPOTIFY_ARTIST_ID"] = metadata["spotify_artist_id"]
|
||||
if metadata.get("spotify_album_id"):
|
||||
source_ids["SPOTIFY_ALBUM_ID"] = metadata["spotify_album_id"]
|
||||
if cfg.get("itunes.embed_tags", True) is not False:
|
||||
if metadata.get("itunes_track_id"):
|
||||
source_ids["ITUNES_TRACK_ID"] = metadata["itunes_track_id"]
|
||||
if metadata.get("itunes_artist_id"):
|
||||
source_ids["ITUNES_ARTIST_ID"] = metadata["itunes_artist_id"]
|
||||
if metadata.get("itunes_album_id"):
|
||||
source_ids["ITUNES_ALBUM_ID"] = metadata["itunes_album_id"]
|
||||
|
||||
return source_ids
|
||||
|
||||
|
||||
def _process_musicbrainz_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
|
||||
if cfg.get("musicbrainz.embed_tags", True) is False:
|
||||
return
|
||||
if not track_title or not artist_name:
|
||||
return
|
||||
|
||||
mb_worker = getattr(runtime, "mb_worker", None)
|
||||
mb_service = mb_worker.mb_service if mb_worker else None
|
||||
if not mb_service:
|
||||
return
|
||||
|
||||
result = _call_source_lookup("MusicBrainz recording", mb_service.match_recording, track_title, artist_name)
|
||||
if result and result.get("mbid"):
|
||||
pp["recording_mbid"] = result["mbid"]
|
||||
pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = pp["recording_mbid"]
|
||||
details = _call_source_lookup(
|
||||
"MusicBrainz recording details",
|
||||
mb_service.mb_client.get_recording,
|
||||
pp["recording_mbid"],
|
||||
includes=["isrcs", "genres"],
|
||||
)
|
||||
if details:
|
||||
isrcs = details.get("isrcs", [])
|
||||
if isrcs:
|
||||
pp["isrc"] = isrcs[0]
|
||||
pp["mb_genres"] = [g["name"] for g in sorted(details.get("genres", []), key=lambda x: x.get("count", 0), reverse=True)]
|
||||
|
||||
track_artist_name = metadata.get("artist", "") or artist_name
|
||||
if ", " in track_artist_name:
|
||||
track_artist_name = track_artist_name.split(", ")[0]
|
||||
artist_result = _call_source_lookup("MusicBrainz artist", mb_service.match_artist, track_artist_name)
|
||||
if artist_result and artist_result.get("mbid"):
|
||||
pp["artist_mbid"] = artist_result["mbid"]
|
||||
pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = pp["artist_mbid"]
|
||||
|
||||
album_name_for_mb = metadata.get("album", "")
|
||||
if album_name_for_mb:
|
||||
artist_key = (pp.get("batch_artist_name") or artist_name).lower().strip()
|
||||
rc_key_norm = (normalize_album_cache_key(album_name_for_mb), artist_key)
|
||||
rc_key_exact = (album_name_for_mb.lower().strip(), artist_key)
|
||||
release_mbid = None
|
||||
with mb_release_cache_lock:
|
||||
cached = _bounded_cache_get(mb_release_cache, rc_key_norm)
|
||||
if cached is None:
|
||||
cached = _bounded_cache_get(mb_release_cache, rc_key_exact)
|
||||
if cached:
|
||||
release_mbid = cached
|
||||
else:
|
||||
rc_result = _call_source_lookup("MusicBrainz release", mb_service.match_release, album_name_for_mb, artist_name)
|
||||
if rc_result and rc_result.get("mbid"):
|
||||
release_mbid = rc_result["mbid"]
|
||||
if release_mbid:
|
||||
_bounded_cache_set(mb_release_cache, rc_key_norm, release_mbid, _MB_RELEASE_CACHE_MAX_ENTRIES)
|
||||
_bounded_cache_set(mb_release_cache, rc_key_exact, release_mbid, _MB_RELEASE_CACHE_MAX_ENTRIES)
|
||||
pp["release_mbid"] = release_mbid or ""
|
||||
if pp["release_mbid"]:
|
||||
pp["id_tags"]["MUSICBRAINZ_RELEASE_ID"] = pp["release_mbid"]
|
||||
|
||||
if pp["release_mbid"]:
|
||||
with mb_release_detail_cache_lock:
|
||||
release_detail = _bounded_cache_get(mb_release_detail_cache, pp["release_mbid"])
|
||||
if release_detail is None:
|
||||
release_detail = _call_source_lookup(
|
||||
"MusicBrainz release details",
|
||||
mb_service.mb_client.get_release,
|
||||
pp["release_mbid"],
|
||||
includes=["release-groups", "labels", "media", "artist-credits", "recordings"],
|
||||
) or {}
|
||||
with mb_release_detail_cache_lock:
|
||||
_bounded_cache_set(mb_release_detail_cache, pp["release_mbid"], release_detail, _MB_RELEASE_DETAIL_CACHE_MAX_ENTRIES)
|
||||
if release_detail:
|
||||
rg = release_detail.get("release-group", {})
|
||||
if rg.get("id"):
|
||||
pp["id_tags"]["MUSICBRAINZ_RELEASEGROUPID"] = rg["id"]
|
||||
ac = release_detail.get("artist-credit", [])
|
||||
if ac and isinstance(ac[0], dict):
|
||||
aa = ac[0].get("artist", {})
|
||||
if aa.get("id"):
|
||||
pp["id_tags"]["MUSICBRAINZ_ALBUMARTISTID"] = aa["id"]
|
||||
if rg.get("primary-type"):
|
||||
pp["id_tags"]["RELEASETYPE"] = rg["primary-type"]
|
||||
if rg.get("first-release-date"):
|
||||
pp["id_tags"]["ORIGINALDATE"] = rg["first-release-date"]
|
||||
if not pp["release_year"] and len(rg["first-release-date"]) >= 4:
|
||||
year = rg["first-release-date"][:4]
|
||||
if year.isdigit():
|
||||
pp["release_year"] = year
|
||||
if release_detail.get("status"):
|
||||
pp["id_tags"]["RELEASESTATUS"] = release_detail["status"]
|
||||
if release_detail.get("country"):
|
||||
pp["id_tags"]["RELEASECOUNTRY"] = release_detail["country"]
|
||||
if release_detail.get("barcode"):
|
||||
pp["id_tags"]["BARCODE"] = release_detail["barcode"]
|
||||
media_list = release_detail.get("media", [])
|
||||
if media_list:
|
||||
fmt = media_list[0].get("format", "")
|
||||
if fmt:
|
||||
pp["id_tags"]["MEDIA"] = fmt
|
||||
pp["id_tags"]["TOTALDISCS"] = str(len(media_list))
|
||||
label_info = release_detail.get("label-info", [])
|
||||
if label_info and isinstance(label_info[0], dict):
|
||||
cat = label_info[0].get("catalog-number", "")
|
||||
if cat:
|
||||
pp["id_tags"]["CATALOGNUMBER"] = cat
|
||||
text_rep = release_detail.get("text-representation", {})
|
||||
if isinstance(text_rep, dict) and text_rep.get("script"):
|
||||
pp["id_tags"]["SCRIPT"] = text_rep["script"]
|
||||
if release_detail.get("asin"):
|
||||
pp["id_tags"]["ASIN"] = release_detail["asin"]
|
||||
track_num = metadata.get("track_number")
|
||||
disc_num = metadata.get("disc_number") or 1
|
||||
if track_num and media_list:
|
||||
try:
|
||||
track_num_int = int(track_num)
|
||||
disc_num_int = int(disc_num)
|
||||
for medium in media_list:
|
||||
if medium.get("position", 1) == disc_num_int:
|
||||
for mtrack in (medium.get("tracks") or medium.get("track-list", [])):
|
||||
if mtrack.get("position") == track_num_int:
|
||||
if mtrack.get("id"):
|
||||
pp["id_tags"]["MUSICBRAINZ_RELEASETRACKID"] = mtrack["id"]
|
||||
release_recording = mtrack.get("recording", {})
|
||||
if release_recording.get("id"):
|
||||
pp["recording_mbid"] = release_recording["id"]
|
||||
pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = release_recording["id"]
|
||||
break
|
||||
break
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
def _process_deezer_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
|
||||
if cfg.get("deezer.embed_tags", True) is False:
|
||||
return
|
||||
if not track_title or not artist_name:
|
||||
return
|
||||
|
||||
deezer_worker = getattr(runtime, "deezer_worker", None)
|
||||
dz_client = deezer_worker.client if deezer_worker else None
|
||||
if not dz_client:
|
||||
return
|
||||
dz_result = _call_source_lookup("Deezer track", dz_client.search_track, artist_name, track_title)
|
||||
if dz_result and _names_match(dz_result.get("title", ""), track_title) and _names_match(dz_result.get("artist", {}).get("name", ""), artist_name):
|
||||
dz_track_id = dz_result["id"]
|
||||
pp["id_tags"]["DEEZER_TRACK_ID"] = str(dz_track_id)
|
||||
dz_artist_id = dz_result.get("artist", {}).get("id")
|
||||
if dz_artist_id:
|
||||
pp["id_tags"]["DEEZER_ARTIST_ID"] = str(dz_artist_id)
|
||||
dz_details = _call_source_lookup("Deezer track details", dz_client.get_track_details, dz_track_id)
|
||||
if dz_details:
|
||||
bpm_val = dz_details.get("bpm")
|
||||
if bpm_val and bpm_val > 0:
|
||||
pp["deezer_bpm"] = bpm_val
|
||||
dz_isrc = dz_details.get("isrc")
|
||||
if dz_isrc:
|
||||
pp["deezer_isrc"] = dz_isrc
|
||||
if not pp["release_year"]:
|
||||
dz_album = dz_result.get("album", {})
|
||||
dz_release = (dz_album.get("release_date", "") if isinstance(dz_album, dict) else "") or ""
|
||||
if len(dz_release) >= 4 and dz_release[:4].isdigit():
|
||||
pp["release_year"] = dz_release[:4]
|
||||
|
||||
|
||||
def _process_audiodb_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
|
||||
if cfg.get("audiodb.embed_tags", True) is False:
|
||||
return
|
||||
if not track_title or not artist_name:
|
||||
return
|
||||
|
||||
audiodb_worker = getattr(runtime, "audiodb_worker", None)
|
||||
adb_client = audiodb_worker.client if audiodb_worker else None
|
||||
if not adb_client:
|
||||
return
|
||||
adb_result = _call_source_lookup("AudioDB track", adb_client.search_track, artist_name, track_title)
|
||||
if adb_result and _names_match(adb_result.get("strTrack", ""), track_title) and _names_match(adb_result.get("strArtist", ""), artist_name):
|
||||
adb_track_id = adb_result.get("idTrack")
|
||||
if adb_track_id:
|
||||
pp["id_tags"]["AUDIODB_TRACK_ID"] = str(adb_track_id)
|
||||
adb_mb_track = adb_result.get("strMusicBrainzID")
|
||||
if adb_mb_track and "MUSICBRAINZ_RECORDING_ID" not in pp["id_tags"]:
|
||||
pp["id_tags"]["MUSICBRAINZ_RECORDING_ID"] = adb_mb_track
|
||||
pp["recording_mbid"] = adb_mb_track
|
||||
adb_mb_artist = adb_result.get("strMusicBrainzArtistID")
|
||||
if adb_mb_artist and "MUSICBRAINZ_ARTIST_ID" not in pp["id_tags"]:
|
||||
pp["id_tags"]["MUSICBRAINZ_ARTIST_ID"] = adb_mb_artist
|
||||
pp["artist_mbid"] = adb_mb_artist
|
||||
pp["audiodb_mood"] = adb_result.get("strMood") or None
|
||||
pp["audiodb_style"] = adb_result.get("strStyle") or None
|
||||
pp["audiodb_genre"] = adb_result.get("strGenre") or None
|
||||
|
||||
|
||||
def _process_tidal_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
|
||||
if cfg.get("tidal.embed_tags", True) is False:
|
||||
return
|
||||
if not track_title or not artist_name:
|
||||
return
|
||||
|
||||
tidal_client = getattr(runtime, "tidal_client", None)
|
||||
if not (tidal_client and tidal_client.is_authenticated()):
|
||||
return
|
||||
td_result = _call_source_lookup("Tidal track", tidal_client.search_track, artist_name, track_title)
|
||||
if td_result and _names_match(td_result.get("title", ""), track_title):
|
||||
td_track_id = td_result.get("id")
|
||||
if td_track_id:
|
||||
pp["id_tags"]["TIDAL_TRACK_ID"] = str(td_track_id)
|
||||
td_artist = td_result.get("artist", {})
|
||||
if isinstance(td_artist, dict) and td_artist.get("id"):
|
||||
pp["id_tags"]["TIDAL_ARTIST_ID"] = str(td_artist["id"])
|
||||
if td_track_id:
|
||||
td_details = _call_source_lookup("Tidal track details", tidal_client.get_track, str(td_track_id))
|
||||
if td_details:
|
||||
pp["tidal_isrc"] = td_details.get("isrc")
|
||||
td_copyright = td_details.get("copyright")
|
||||
if isinstance(td_copyright, dict):
|
||||
td_copyright = td_copyright.get("text", td_copyright.get("name", ""))
|
||||
pp["tidal_copyright"] = td_copyright or None
|
||||
if not pp["release_year"]:
|
||||
td_album = td_result.get("album", {})
|
||||
td_release = ""
|
||||
if isinstance(td_album, dict):
|
||||
td_release = str(td_album.get("release_date", "") or td_album.get("releaseDate", "") or "")
|
||||
if len(td_release) >= 4 and td_release[:4].isdigit():
|
||||
pp["release_year"] = td_release[:4]
|
||||
|
||||
|
||||
def _process_qobuz_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
|
||||
if cfg.get("qobuz.embed_tags", True) is False:
|
||||
return
|
||||
if not track_title or not artist_name:
|
||||
return
|
||||
|
||||
qobuz_worker = getattr(runtime, "qobuz_enrichment_worker", None)
|
||||
qz_client = qobuz_worker.client if qobuz_worker else None
|
||||
if not (qz_client and qz_client.is_authenticated()):
|
||||
return
|
||||
qz_result = _call_source_lookup("Qobuz track", qz_client.search_track, artist_name, track_title)
|
||||
if qz_result:
|
||||
qz_performer = qz_result.get("performer") or {}
|
||||
if not isinstance(qz_performer, dict):
|
||||
qz_performer = {}
|
||||
qz_artist_name = qz_performer.get("name", "")
|
||||
if _names_match(qz_result.get("title", ""), track_title) and _names_match(qz_artist_name, artist_name):
|
||||
qz_track_id = qz_result.get("id")
|
||||
if qz_track_id:
|
||||
pp["id_tags"]["QOBUZ_TRACK_ID"] = str(qz_track_id)
|
||||
if qz_performer.get("id"):
|
||||
pp["id_tags"]["QOBUZ_ARTIST_ID"] = str(qz_performer["id"])
|
||||
qz_isrc = qz_result.get("isrc")
|
||||
if isinstance(qz_isrc, dict):
|
||||
qz_isrc = qz_isrc.get("value", qz_isrc.get("id", ""))
|
||||
if qz_isrc:
|
||||
pp["qobuz_isrc"] = qz_isrc
|
||||
qz_copyright = qz_result.get("copyright")
|
||||
if isinstance(qz_copyright, dict):
|
||||
qz_copyright = qz_copyright.get("text", qz_copyright.get("name", ""))
|
||||
if isinstance(qz_copyright, str):
|
||||
pp["qobuz_copyright"] = qz_copyright
|
||||
qz_album = qz_result.get("album", {})
|
||||
if isinstance(qz_album, dict):
|
||||
qz_label_info = qz_album.get("label", {})
|
||||
if isinstance(qz_label_info, dict) and qz_label_info.get("name"):
|
||||
pp["qobuz_label"] = qz_label_info["name"]
|
||||
if not pp["release_year"]:
|
||||
qz_release = str(qz_album.get("release_date_original", "") or "")
|
||||
if not qz_release:
|
||||
qz_ts = qz_album.get("released_at")
|
||||
if qz_ts and isinstance(qz_ts, (int, float)) and qz_ts > 0:
|
||||
import datetime as _dt
|
||||
|
||||
qz_release = str(_dt.datetime.utcfromtimestamp(qz_ts).year)
|
||||
if len(qz_release) >= 4 and qz_release[:4].isdigit():
|
||||
pp["release_year"] = qz_release[:4]
|
||||
|
||||
|
||||
def _process_lastfm_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
|
||||
if cfg.get("lastfm.embed_tags", True) is False:
|
||||
return
|
||||
if not track_title or not artist_name:
|
||||
return
|
||||
|
||||
lastfm_worker = getattr(runtime, "lastfm_worker", None)
|
||||
lf_client = lastfm_worker.client if lastfm_worker else None
|
||||
if not lf_client:
|
||||
return
|
||||
lf_result = _call_source_lookup("Last.fm track", lf_client.get_track_info, artist_name, track_title)
|
||||
if lf_result:
|
||||
lf_url = lf_result.get("url")
|
||||
if lf_url:
|
||||
pp["lastfm_url"] = lf_url
|
||||
lf_toptags = lf_result.get("toptags", {})
|
||||
if isinstance(lf_toptags, dict):
|
||||
tag_list = lf_toptags.get("tag", [])
|
||||
if isinstance(tag_list, list):
|
||||
pp["lastfm_tags"] = [tag.get("name", "") for tag in tag_list if isinstance(tag, dict) and tag.get("name")]
|
||||
elif isinstance(tag_list, dict) and tag_list.get("name"):
|
||||
pp["lastfm_tags"] = [tag_list["name"]]
|
||||
|
||||
|
||||
def _process_genius_source(pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
|
||||
if cfg.get("genius.embed_tags", True) is False:
|
||||
return
|
||||
if not track_title or not artist_name:
|
||||
return
|
||||
|
||||
import core.genius_client as _genius_module
|
||||
|
||||
if time.time() < _genius_module._rate_limit_until:
|
||||
logger.info("Genius rate-limited, skipping (non-blocking)")
|
||||
return
|
||||
genius_worker = getattr(runtime, "genius_worker", None)
|
||||
g_client = genius_worker.client if genius_worker else None
|
||||
if not g_client:
|
||||
return
|
||||
g_result = _call_source_lookup("Genius track", g_client.search_song, artist_name, track_title)
|
||||
if g_result:
|
||||
g_id = g_result.get("id")
|
||||
if g_id:
|
||||
pp["id_tags"]["GENIUS_TRACK_ID"] = str(g_id)
|
||||
g_url = g_result.get("url")
|
||||
if g_url:
|
||||
pp["genius_url"] = g_url
|
||||
|
||||
|
||||
def _process_source_enrichment(source_name: str, pp: dict, metadata: dict, cfg, runtime, track_title: str, artist_name: str) -> None:
|
||||
if source_name == "musicbrainz":
|
||||
_process_musicbrainz_source(pp, metadata, cfg, runtime, track_title, artist_name)
|
||||
elif source_name == "deezer":
|
||||
_process_deezer_source(pp, metadata, cfg, runtime, track_title, artist_name)
|
||||
elif source_name == "audiodb":
|
||||
_process_audiodb_source(pp, metadata, cfg, runtime, track_title, artist_name)
|
||||
elif source_name == "tidal":
|
||||
_process_tidal_source(pp, metadata, cfg, runtime, track_title, artist_name)
|
||||
elif source_name == "qobuz":
|
||||
_process_qobuz_source(pp, metadata, cfg, runtime, track_title, artist_name)
|
||||
elif source_name == "lastfm":
|
||||
_process_lastfm_source(pp, metadata, cfg, runtime, track_title, artist_name)
|
||||
elif source_name == "genius":
|
||||
_process_genius_source(pp, metadata, cfg, runtime, track_title, artist_name)
|
||||
|
||||
|
||||
def _write_embedded_metadata(audio_file, metadata: dict, pp: dict, cfg, symbols):
|
||||
filtered_tags: Dict[str, str] = {}
|
||||
for tag_name, value in pp["id_tags"].items():
|
||||
config_path = SOURCE_TAG_CONFIG.get(tag_name)
|
||||
if config_path and not _tag_enabled(cfg, config_path):
|
||||
continue
|
||||
filtered_tags[tag_name] = value
|
||||
|
||||
written = []
|
||||
release_year = pp["release_year"]
|
||||
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
for tag_name, value in filtered_tags.items():
|
||||
spec = ID3_TAG_MAP.get(tag_name)
|
||||
if spec:
|
||||
frame_type, desc = spec
|
||||
if frame_type == "UFID":
|
||||
audio_file.tags.add(symbols.UFID(owner=desc, data=str(value).encode("ascii")))
|
||||
written.append(f"UFID:{desc}")
|
||||
elif frame_type == "TDOR":
|
||||
audio_file.tags.add(symbols.TDOR(encoding=3, text=[value]))
|
||||
written.append("TDOR")
|
||||
elif frame_type == "TMED":
|
||||
audio_file.tags.add(symbols.TMED(encoding=3, text=[value]))
|
||||
written.append("TMED")
|
||||
else:
|
||||
audio_file.tags.add(symbols.TXXX(encoding=3, desc=desc, text=[value]))
|
||||
written.append(f"TXXX:{desc}")
|
||||
else:
|
||||
audio_file.tags.add(symbols.TXXX(encoding=3, desc=tag_name, text=[str(value)]))
|
||||
written.append(f"TXXX:{tag_name}")
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
for tag_name, value in filtered_tags.items():
|
||||
key = f"----:com.apple.iTunes:{MP4_TAG_MAP.get(tag_name, tag_name)}"
|
||||
audio_file[key] = [symbols.MP4FreeForm(str(value).encode("utf-8"))]
|
||||
written.append(key)
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
for tag_name, value in filtered_tags.items():
|
||||
audio_file[VORBIS_TAG_MAP.get(tag_name, tag_name)] = [str(value)]
|
||||
written.append(VORBIS_TAG_MAP.get(tag_name, tag_name))
|
||||
|
||||
if written:
|
||||
logger.info("Embedded IDs: %s", ", ".join(written))
|
||||
|
||||
if release_year and not metadata.get("date"):
|
||||
metadata["date"] = release_year
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.TDRC(encoding=3, text=[release_year]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
audio_file["date"] = [release_year]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
audio_file["\xa9day"] = [release_year]
|
||||
logger.info("Date tag: %s", release_year)
|
||||
|
||||
if _tag_enabled(cfg, "deezer.tags.bpm") and pp["deezer_bpm"] and pp["deezer_bpm"] > 0:
|
||||
bpm_int = int(pp["deezer_bpm"])
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.TBPM(encoding=3, text=[str(bpm_int)]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
audio_file["BPM"] = [str(bpm_int)]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
audio_file["tmpo"] = [bpm_int]
|
||||
logger.info("BPM: %s", bpm_int)
|
||||
|
||||
if _tag_enabled(cfg, "audiodb.tags.mood") and pp["audiodb_mood"]:
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.TXXX(encoding=3, desc="MOOD", text=[pp["audiodb_mood"]]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
audio_file["MOOD"] = [pp["audiodb_mood"]]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
audio_file["----:com.apple.iTunes:MOOD"] = [symbols.MP4FreeForm(pp["audiodb_mood"].encode("utf-8"))]
|
||||
|
||||
if _tag_enabled(cfg, "audiodb.tags.style") and pp["audiodb_style"]:
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.TXXX(encoding=3, desc="STYLE", text=[pp["audiodb_style"]]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
audio_file["STYLE"] = [pp["audiodb_style"]]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
audio_file["----:com.apple.iTunes:STYLE"] = [symbols.MP4FreeForm(pp["audiodb_style"].encode("utf-8"))]
|
||||
|
||||
if _tag_enabled(cfg, "metadata_enhancement.tags.genre_merge"):
|
||||
enrichment_genres = []
|
||||
if _tag_enabled(cfg, "musicbrainz.tags.genres"):
|
||||
enrichment_genres += pp["mb_genres"]
|
||||
if pp["audiodb_genre"] and _tag_enabled(cfg, "audiodb.tags.genre"):
|
||||
enrichment_genres.append(pp["audiodb_genre"])
|
||||
if _tag_enabled(cfg, "lastfm.tags.genres"):
|
||||
enrichment_genres += pp["lastfm_tags"]
|
||||
if enrichment_genres:
|
||||
from core.genre_filter import filter_genres as _filter_genres
|
||||
|
||||
enrichment_genres = _filter_genres(enrichment_genres, cfg)
|
||||
source_genres = [g.strip() for g in str(metadata.get("genre", "")).split(",") if g.strip()]
|
||||
seen = set()
|
||||
merged = []
|
||||
for genre in source_genres + enrichment_genres:
|
||||
key = genre.strip().lower()
|
||||
if key and key not in seen:
|
||||
seen.add(key)
|
||||
merged.append(genre.strip().title())
|
||||
if len(merged) >= 5:
|
||||
break
|
||||
if merged:
|
||||
genre_string = ", ".join(merged)
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.TCON(encoding=3, text=[genre_string]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
audio_file["GENRE"] = [genre_string]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
audio_file["\xa9gen"] = [genre_string]
|
||||
logger.info("Genres merged: %s", genre_string)
|
||||
|
||||
isrc_candidates = []
|
||||
if pp["isrc"] and _tag_enabled(cfg, "musicbrainz.tags.isrc"):
|
||||
isrc_candidates.append(("MusicBrainz", pp["isrc"]))
|
||||
if pp["deezer_isrc"] and _tag_enabled(cfg, "deezer.tags.isrc"):
|
||||
isrc_candidates.append(("Deezer", pp["deezer_isrc"]))
|
||||
if pp["tidal_isrc"] and _tag_enabled(cfg, "tidal.tags.isrc"):
|
||||
isrc_candidates.append(("Tidal", pp["tidal_isrc"]))
|
||||
if pp["qobuz_isrc"] and _tag_enabled(cfg, "qobuz.tags.isrc"):
|
||||
isrc_candidates.append(("Qobuz", pp["qobuz_isrc"]))
|
||||
if isrc_candidates:
|
||||
isrc_source, final_isrc = isrc_candidates[0]
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.TSRC(encoding=3, text=[final_isrc]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
audio_file["ISRC"] = [final_isrc]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
audio_file["----:com.apple.iTunes:ISRC"] = [symbols.MP4FreeForm(final_isrc.encode("utf-8"))]
|
||||
logger.info("ISRC (%s): %s", isrc_source, final_isrc)
|
||||
|
||||
copyright_candidates = []
|
||||
if pp["tidal_copyright"] and _tag_enabled(cfg, "tidal.tags.copyright"):
|
||||
copyright_candidates.append(("Tidal", pp["tidal_copyright"]))
|
||||
if pp["qobuz_copyright"] and _tag_enabled(cfg, "qobuz.tags.copyright"):
|
||||
copyright_candidates.append(("Qobuz", pp["qobuz_copyright"]))
|
||||
if copyright_candidates:
|
||||
copyright_source, final_copyright = copyright_candidates[0]
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.TCOP(encoding=3, text=[final_copyright]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
audio_file["COPYRIGHT"] = [final_copyright]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
audio_file["cprt"] = [final_copyright]
|
||||
logger.info("Copyright (%s): %s", copyright_source, final_copyright[:60])
|
||||
|
||||
if _tag_enabled(cfg, "qobuz.tags.label") and pp["qobuz_label"]:
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.TPUB(encoding=3, text=[pp["qobuz_label"]]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
audio_file["LABEL"] = [pp["qobuz_label"]]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
audio_file["----:com.apple.iTunes:LABEL"] = [symbols.MP4FreeForm(pp["qobuz_label"].encode("utf-8"))]
|
||||
|
||||
if _tag_enabled(cfg, "lastfm.tags.url") and pp["lastfm_url"]:
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.TXXX(encoding=3, desc="LASTFM_URL", text=[pp["lastfm_url"]]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
audio_file["LASTFM_URL"] = [pp["lastfm_url"]]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
audio_file["----:com.apple.iTunes:LASTFM_URL"] = [symbols.MP4FreeForm(pp["lastfm_url"].encode("utf-8"))]
|
||||
|
||||
if _tag_enabled(cfg, "genius.tags.url") and pp["genius_url"]:
|
||||
if isinstance(audio_file.tags, symbols.ID3):
|
||||
audio_file.tags.add(symbols.TXXX(encoding=3, desc="GENIUS_URL", text=[pp["genius_url"]]))
|
||||
elif is_vorbis_like(audio_file, symbols):
|
||||
audio_file["GENIUS_URL"] = [pp["genius_url"]]
|
||||
elif isinstance(audio_file, symbols.MP4):
|
||||
audio_file["----:com.apple.iTunes:GENIUS_URL"] = [symbols.MP4FreeForm(pp["genius_url"].encode("utf-8"))]
|
||||
|
||||
return release_year
|
||||
|
||||
|
||||
def _update_album_year_in_database(db, metadata: dict, release_year) -> None:
|
||||
if db is None:
|
||||
return
|
||||
|
||||
try:
|
||||
album_name_for_db = metadata.get("album", "")
|
||||
album_artist_for_db = metadata.get("album_artist", "") or metadata.get("artist", "")
|
||||
if album_name_for_db and album_artist_for_db:
|
||||
conn = db._get_connection()
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE albums SET year = ?
|
||||
WHERE (year IS NULL OR year = 0)
|
||||
AND id IN (
|
||||
SELECT al.id FROM albums al
|
||||
JOIN artists ar ON ar.id = al.artist_id
|
||||
WHERE LOWER(al.title) = LOWER(?) AND LOWER(ar.name) = LOWER(?)
|
||||
)
|
||||
""",
|
||||
(int(release_year), album_name_for_db, album_artist_for_db),
|
||||
)
|
||||
if cursor.rowcount > 0:
|
||||
conn.commit()
|
||||
logger.info("Updated album year to %s in database", release_year)
|
||||
else:
|
||||
conn.rollback()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
logger.error("Could not update album year in DB: %s", exc)
|
||||
|
||||
|
||||
def extract_source_metadata(context: dict, artist: dict, album_info: dict) -> dict:
|
||||
if album_info is None:
|
||||
album_info = {}
|
||||
|
||||
cfg = get_config_manager()
|
||||
context = normalize_import_context(context)
|
||||
original_search = get_import_original_search(context)
|
||||
album_ctx = get_import_context_album(context)
|
||||
track_info = get_import_track_info(context)
|
||||
source = get_import_source(context)
|
||||
source_ids = get_import_source_ids(context)
|
||||
|
||||
artist_dict = artist if isinstance(artist, dict) else {
|
||||
"name": extract_artist_name(artist),
|
||||
"id": getattr(artist, "id", ""),
|
||||
"genres": list(getattr(artist, "genres", []) or []),
|
||||
}
|
||||
|
||||
metadata: Dict[str, Any] = {
|
||||
"source": source,
|
||||
"source_track_id": source_ids["track_id"],
|
||||
"source_artist_id": source_ids["artist_id"],
|
||||
"source_album_id": source_ids["album_id"],
|
||||
}
|
||||
|
||||
metadata["title"] = get_import_clean_title(context, album_info=album_info, default=original_search.get("title", ""))
|
||||
if original_search.get("clean_title"):
|
||||
logger.info("Metadata: Using clean title: '%s'", metadata["title"])
|
||||
elif album_info.get("clean_track_name"):
|
||||
logger.info("Metadata: Using album info clean name: '%s'", metadata["title"])
|
||||
else:
|
||||
logger.warning("Metadata: Using original title as fallback: '%s'", metadata["title"])
|
||||
|
||||
artists = original_search.get("artists")
|
||||
if isinstance(artists, list) and artists:
|
||||
all_artists = []
|
||||
for artist_item in artists:
|
||||
if isinstance(artist_item, dict) and artist_item.get("name"):
|
||||
all_artists.append(artist_item["name"])
|
||||
elif isinstance(artist_item, str):
|
||||
all_artists.append(artist_item)
|
||||
else:
|
||||
all_artists.append(str(artist_item))
|
||||
metadata["artist"] = ", ".join(all_artists)
|
||||
logger.info("Metadata: Using all artists: '%s'", metadata["artist"])
|
||||
else:
|
||||
metadata["artist"] = artist_dict.get("name", "") or get_import_clean_artist(context)
|
||||
logger.info("Metadata: Using primary artist: '%s'", metadata["artist"])
|
||||
|
||||
raw_album_artist = artist_dict.get("name", "") or metadata["artist"]
|
||||
track_info_ctx = track_info or {}
|
||||
explicit_artist = track_info_ctx.get("_explicit_artist_context") if isinstance(track_info_ctx, dict) else None
|
||||
album_artists_for_collab = None
|
||||
|
||||
if isinstance(explicit_artist, dict) and explicit_artist.get("name"):
|
||||
raw_album_artist = explicit_artist["name"]
|
||||
album_artists_for_collab = [explicit_artist]
|
||||
elif isinstance(explicit_artist, str) and explicit_artist:
|
||||
raw_album_artist = explicit_artist
|
||||
album_artists_for_collab = [{"name": explicit_artist}]
|
||||
elif album_ctx and isinstance(album_ctx, dict):
|
||||
album_artists = album_ctx.get("artists", [])
|
||||
if album_artists:
|
||||
first_album_artist = album_artists[0]
|
||||
if isinstance(first_album_artist, dict) and first_album_artist.get("name"):
|
||||
raw_album_artist = first_album_artist["name"]
|
||||
elif isinstance(first_album_artist, str) and first_album_artist:
|
||||
raw_album_artist = first_album_artist
|
||||
album_artists_for_collab = album_artists
|
||||
|
||||
collab_mode = cfg.get("file_organization.collab_artist_mode", "first")
|
||||
if collab_mode == "first" and raw_album_artist:
|
||||
context_artists = album_artists_for_collab or original_search.get("artists") or track_info_ctx.get("artists") or []
|
||||
if len(context_artists) > 1:
|
||||
first = context_artists[0]
|
||||
raw_album_artist = first.get("name", first) if isinstance(first, dict) else str(first)
|
||||
elif len(context_artists) == 1 and ("," in raw_album_artist or " & " in raw_album_artist):
|
||||
artist_id = str(artist_dict.get("id", ""))
|
||||
if source == "itunes" and artist_id.isdigit():
|
||||
try:
|
||||
itunes_client = get_itunes_client()
|
||||
if itunes_client and hasattr(itunes_client, "resolve_primary_artist"):
|
||||
resolved = itunes_client.resolve_primary_artist(artist_id)
|
||||
if resolved and resolved != raw_album_artist:
|
||||
raw_album_artist = resolved
|
||||
except Exception:
|
||||
pass
|
||||
metadata["album_artist"] = raw_album_artist
|
||||
|
||||
if album_info.get("is_album"):
|
||||
metadata["album"] = album_info.get("album_name", "Unknown Album")
|
||||
metadata["track_number"] = album_info.get("track_number", 1)
|
||||
metadata["total_tracks"] = album_ctx.get("total_tracks", 1) if album_ctx else 1
|
||||
logger.info("[METADATA] Album track - track_number: %s, album: %s", metadata["track_number"], metadata["album"])
|
||||
else:
|
||||
if album_ctx and album_ctx.get("name"):
|
||||
logger.info("[SAFEGUARD] Using album context name instead of track title for album metadata")
|
||||
metadata["album"] = album_ctx["name"]
|
||||
metadata["track_number"] = album_info.get("track_number", 1) if album_info else 1
|
||||
metadata["total_tracks"] = album_ctx.get("total_tracks", 1)
|
||||
else:
|
||||
metadata["album"] = metadata["title"]
|
||||
metadata["track_number"] = 1
|
||||
metadata["total_tracks"] = 1
|
||||
|
||||
disc_num = original_search.get("disc_number")
|
||||
if disc_num is None and album_info:
|
||||
disc_num = album_info.get("disc_number")
|
||||
metadata["disc_number"] = disc_num if disc_num is not None else 1
|
||||
|
||||
if album_ctx and album_ctx.get("release_date"):
|
||||
metadata["date"] = album_ctx["release_date"][:4]
|
||||
|
||||
genres = artist_dict.get("genres") or []
|
||||
if genres:
|
||||
from core.genre_filter import filter_genres
|
||||
|
||||
filtered = filter_genres(list(genres[:2]), cfg)
|
||||
if filtered:
|
||||
metadata["genre"] = ", ".join(filtered)
|
||||
|
||||
metadata["album_art_url"] = album_info.get("album_image_url") if album_info else None
|
||||
if not metadata["album_art_url"] and album_ctx:
|
||||
album_image = album_ctx.get("image_url")
|
||||
if not album_image and album_ctx.get("images"):
|
||||
first_image = album_ctx["images"][0]
|
||||
album_image = first_image.get("url") if isinstance(first_image, dict) else None
|
||||
metadata["album_art_url"] = album_image
|
||||
|
||||
logger.info(
|
||||
"[Metadata Summary] title='%s' | artist='%s' | album_artist='%s' | album='%s' | track=%s/%s | disc=%s",
|
||||
metadata.get("title"),
|
||||
metadata.get("artist"),
|
||||
metadata.get("album_artist"),
|
||||
metadata.get("album"),
|
||||
metadata.get("track_number"),
|
||||
metadata.get("total_tracks"),
|
||||
metadata.get("disc_number"),
|
||||
)
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
def embed_source_ids(audio_file, metadata: dict, context: dict = None, runtime=None):
|
||||
cfg = get_config_manager()
|
||||
symbols = get_mutagen_symbols()
|
||||
if not symbols:
|
||||
return
|
||||
|
||||
try:
|
||||
context = normalize_import_context(context)
|
||||
source_ids = _collect_source_ids(metadata, cfg)
|
||||
|
||||
track_title = metadata.get("title", "")
|
||||
artist_name = metadata.get("album_artist", "") or metadata.get("artist", "")
|
||||
track_info = get_import_track_info(context)
|
||||
explicit_artist = (track_info or {}).get("_explicit_artist_context") if isinstance(track_info, dict) else None
|
||||
batch_artist_name = None
|
||||
if isinstance(explicit_artist, dict) and explicit_artist.get("name"):
|
||||
batch_artist_name = explicit_artist["name"]
|
||||
elif isinstance(explicit_artist, str) and explicit_artist:
|
||||
batch_artist_name = explicit_artist
|
||||
|
||||
pp = {
|
||||
"id_tags": source_ids,
|
||||
"track_title": track_title,
|
||||
"artist_name": artist_name,
|
||||
"batch_artist_name": batch_artist_name,
|
||||
"metadata": metadata,
|
||||
"recording_mbid": None,
|
||||
"artist_mbid": None,
|
||||
"release_mbid": "",
|
||||
"mb_genres": [],
|
||||
"isrc": None,
|
||||
"deezer_bpm": None,
|
||||
"deezer_isrc": None,
|
||||
"audiodb_mood": None,
|
||||
"audiodb_style": None,
|
||||
"audiodb_genre": None,
|
||||
"tidal_isrc": None,
|
||||
"tidal_copyright": None,
|
||||
"qobuz_isrc": None,
|
||||
"qobuz_copyright": None,
|
||||
"qobuz_label": None,
|
||||
"lastfm_tags": [],
|
||||
"lastfm_url": None,
|
||||
"genius_url": None,
|
||||
"release_year": None,
|
||||
}
|
||||
|
||||
source_order = cfg.get("metadata_enhancement.post_process_order", None)
|
||||
if not isinstance(source_order, list) or not source_order:
|
||||
source_order = DEFAULT_SOURCE_ORDER
|
||||
|
||||
db = get_database()
|
||||
|
||||
for source_name in source_order:
|
||||
_process_source_enrichment(source_name, pp, metadata, cfg, runtime, track_title, artist_name)
|
||||
|
||||
if not pp["id_tags"] and not pp["deezer_bpm"] and not pp["deezer_isrc"] and not pp["audiodb_mood"] and not pp["audiodb_style"]:
|
||||
return
|
||||
|
||||
release_year = _write_embedded_metadata(audio_file, metadata, pp, cfg, symbols)
|
||||
release_id = pp["release_mbid"]
|
||||
if release_id:
|
||||
metadata["musicbrainz_release_id"] = release_id
|
||||
_update_album_year_in_database(db, metadata, release_year)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Error embedding source IDs (non-fatal): %s", exc)
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
"""
|
||||
Metadata Service - Centralized metadata source selection
|
||||
Metadata Service - Centralized metadata source selection and provider access.
|
||||
|
||||
ALL metadata source decisions flow through this module. Other files import
|
||||
get_primary_source() and get_primary_client() instead of reimplementing
|
||||
|
|
@ -100,6 +100,35 @@ def get_source_priority(preferred_source: str):
|
|||
return ordered
|
||||
|
||||
|
||||
def _get_source_chain_for_lookup(options: MetadataLookupOptions) -> List[str]:
|
||||
primary_source = get_primary_source()
|
||||
source_chain = list(get_source_priority(primary_source))
|
||||
override = (options.source_override or '').strip().lower()
|
||||
|
||||
if override:
|
||||
source_chain = [override] + [source for source in source_chain if source != override]
|
||||
|
||||
if not options.allow_fallback:
|
||||
source_chain = source_chain[:1]
|
||||
|
||||
return source_chain
|
||||
|
||||
|
||||
def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any:
|
||||
if value is None:
|
||||
return default
|
||||
|
||||
for name in names:
|
||||
if isinstance(value, dict):
|
||||
if name in value and value[name] is not None:
|
||||
return value[name]
|
||||
else:
|
||||
candidate = getattr(value, name, None)
|
||||
if candidate is not None:
|
||||
return candidate
|
||||
return default
|
||||
|
||||
|
||||
def get_client_for_source(source: str):
|
||||
"""Get the client object for an exact metadata source.
|
||||
|
||||
|
|
@ -235,35 +264,6 @@ def get_artist_albums_for_source(
|
|||
return None
|
||||
|
||||
|
||||
def _get_source_chain_for_lookup(options: MetadataLookupOptions) -> List[str]:
|
||||
primary_source = get_primary_source()
|
||||
source_chain = list(get_source_priority(primary_source))
|
||||
override = (options.source_override or '').strip().lower()
|
||||
|
||||
if override:
|
||||
source_chain = [override] + [source for source in source_chain if source != override]
|
||||
|
||||
if not options.allow_fallback:
|
||||
source_chain = source_chain[:1]
|
||||
|
||||
return source_chain
|
||||
|
||||
|
||||
def _extract_lookup_value(value: Any, *names: str, default: Any = None) -> Any:
|
||||
if value is None:
|
||||
return default
|
||||
|
||||
for name in names:
|
||||
if isinstance(value, dict):
|
||||
if name in value and value[name] is not None:
|
||||
return value[name]
|
||||
else:
|
||||
candidate = getattr(value, name, None)
|
||||
if candidate is not None:
|
||||
return candidate
|
||||
return default
|
||||
|
||||
|
||||
def _normalize_artist_name(value: Any) -> str:
|
||||
return (value or '').strip().casefold()
|
||||
|
||||
|
|
@ -719,6 +719,23 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti
|
|||
if not isinstance(images, list):
|
||||
images = list(images) if images else []
|
||||
|
||||
artists = _normalize_context_artists(_extract_lookup_value(album_data, 'artists', default=[]))
|
||||
if not artists and artist_name:
|
||||
artists = [{'name': artist_name}]
|
||||
|
||||
primary_artist = artists[0] if artists else {}
|
||||
resolved_artist_name = (
|
||||
_extract_lookup_value(primary_artist, 'name', default='')
|
||||
or artist_name
|
||||
or _extract_lookup_value(album_data, 'artist_name', 'artist', default='')
|
||||
or ''
|
||||
)
|
||||
resolved_artist_id = str(
|
||||
_extract_lookup_value(primary_artist, 'id', default='')
|
||||
or _extract_lookup_value(album_data, 'artist_id', default='')
|
||||
or ''
|
||||
).strip()
|
||||
|
||||
image_url = None
|
||||
if images:
|
||||
image_url = _extract_lookup_value(images[0], 'url')
|
||||
|
|
@ -728,12 +745,15 @@ def _build_album_info(album_data: Any, album_id: str, album_name: str = '', arti
|
|||
return {
|
||||
'id': _extract_lookup_value(album_data, 'id', 'album_id', 'collectionId', 'release_id', default=album_id) or album_id,
|
||||
'name': _extract_lookup_value(album_data, 'name', 'title', default=album_name or album_id) or album_name or album_id,
|
||||
'artist': resolved_artist_name or '',
|
||||
'artist_name': resolved_artist_name or '',
|
||||
'artist_id': resolved_artist_id,
|
||||
'artists': artists,
|
||||
'image_url': image_url,
|
||||
'images': images,
|
||||
'release_date': _extract_lookup_value(album_data, 'release_date', default='') or '',
|
||||
'album_type': _extract_lookup_value(album_data, 'album_type', default='album') or 'album',
|
||||
'total_tracks': _extract_lookup_value(album_data, 'total_tracks', 'track_count', default=0) or 0,
|
||||
'artist_name': artist_name or _extract_lookup_value(album_data, 'artist_name', default='') or '',
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -754,6 +774,8 @@ def _build_album_track_entry(track_item: Any, album_info: Dict[str, Any], source
|
|||
'external_urls': _extract_lookup_value(track_item, 'external_urls', default={}) or {},
|
||||
'uri': _extract_lookup_value(track_item, 'uri', default='') or '',
|
||||
'album': album_info,
|
||||
'source': source,
|
||||
'provider': source,
|
||||
'_source': source,
|
||||
}
|
||||
|
||||
|
|
@ -767,6 +789,9 @@ def _build_album_tracks_payload(
|
|||
artist_name: str = '',
|
||||
) -> Dict[str, Any]:
|
||||
album_info = _build_album_info(album_data, album_id, album_name=album_name, artist_name=artist_name)
|
||||
album_info['source'] = source
|
||||
album_info['_source'] = source
|
||||
album_info['provider'] = source
|
||||
track_items = _extract_album_track_items(album_data, tracks_data)
|
||||
tracks = [_build_album_track_entry(track, album_info, source) for track in track_items]
|
||||
|
||||
|
|
@ -778,6 +803,44 @@ def _build_album_tracks_payload(
|
|||
}
|
||||
|
||||
|
||||
def _normalize_context_artists(artists: Any) -> List[Dict[str, Any]]:
|
||||
if not artists:
|
||||
return []
|
||||
|
||||
if isinstance(artists, (str, bytes)):
|
||||
artists = [artists]
|
||||
elif isinstance(artists, dict):
|
||||
artists = [artists]
|
||||
else:
|
||||
try:
|
||||
artists = list(artists)
|
||||
except TypeError:
|
||||
artists = [artists]
|
||||
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
for artist in artists:
|
||||
if isinstance(artist, dict):
|
||||
name = _extract_lookup_value(artist, 'name', 'artist_name', 'title', default='') or ''
|
||||
artist_id = _extract_lookup_value(artist, 'id', 'artist_id', default='') or ''
|
||||
entry: Dict[str, Any] = {}
|
||||
if name:
|
||||
entry['name'] = str(name)
|
||||
if artist_id:
|
||||
entry['id'] = str(artist_id)
|
||||
genres = _extract_lookup_value(artist, 'genres', default=None)
|
||||
if genres is not None:
|
||||
entry['genres'] = genres
|
||||
if entry:
|
||||
normalized.append(entry)
|
||||
continue
|
||||
|
||||
name = str(artist).strip()
|
||||
if name:
|
||||
normalized.append({'name': name})
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def resolve_album_reference(
|
||||
album_id: str,
|
||||
preferred_source: Optional[str] = None,
|
||||
|
|
|
|||
80
core/runtime_state.py
Normal file
80
core/runtime_state.py
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
"""Shared runtime state and tiny helpers for the app."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from functools import wraps
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
matched_context_lock = threading.Lock()
|
||||
matched_downloads_context: Dict[str, Dict[str, Any]] = {}
|
||||
tasks_lock = threading.Lock()
|
||||
download_tasks: Dict[str, Dict[str, Any]] = {}
|
||||
download_batches: Dict[str, Dict[str, Any]] = {}
|
||||
processed_download_ids = set()
|
||||
post_process_locks: Dict[str, threading.Lock] = {}
|
||||
post_process_locks_lock = threading.Lock()
|
||||
|
||||
activity_feed = []
|
||||
activity_feed_lock = threading.Lock()
|
||||
_activity_toast_emitter = None
|
||||
|
||||
|
||||
def caller_must_hold_tasks_lock(func):
|
||||
"""Best-effort guard for helpers that mutate download_tasks in place."""
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
if not tasks_lock.locked():
|
||||
raise RuntimeError(f"{func.__name__}() requires tasks_lock to be held by the caller")
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def set_activity_toast_emitter(emitter) -> None:
|
||||
"""Set the WebSocket-style emitter used by add_activity_item."""
|
||||
global _activity_toast_emitter
|
||||
_activity_toast_emitter = emitter
|
||||
|
||||
|
||||
def add_activity_item(icon, title, subtitle, time_ago="Now", show_toast=True):
|
||||
"""Append an activity item and emit a toast if an emitter is configured."""
|
||||
activity_item = {
|
||||
"icon": icon,
|
||||
"title": title,
|
||||
"subtitle": subtitle,
|
||||
"time": time_ago,
|
||||
"timestamp": time.time(),
|
||||
"show_toast": show_toast,
|
||||
}
|
||||
with activity_feed_lock:
|
||||
activity_feed.append(activity_item)
|
||||
if len(activity_feed) > 20:
|
||||
activity_feed.pop(0)
|
||||
|
||||
if show_toast and _activity_toast_emitter is not None:
|
||||
try:
|
||||
_activity_toast_emitter("dashboard:toast", activity_item)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return activity_item
|
||||
|
||||
|
||||
@caller_must_hold_tasks_lock
|
||||
def mark_task_completed(task_id: str, track_info: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""Mark a download task as completed.
|
||||
|
||||
Callers must already hold `tasks_lock`.
|
||||
"""
|
||||
task = download_tasks.get(task_id)
|
||||
if not task:
|
||||
return False
|
||||
|
||||
task["status"] = "completed"
|
||||
task["stream_processed"] = True
|
||||
task["status_change_time"] = time.time()
|
||||
if track_info is not None:
|
||||
task["track_info"] = track_info
|
||||
return True
|
||||
|
|
@ -8,6 +8,7 @@ import time
|
|||
from pathlib import Path
|
||||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
from core.imports.filename import parse_filename_metadata
|
||||
|
||||
logger = get_logger("soulseek_client")
|
||||
|
||||
|
|
@ -87,60 +88,17 @@ class TrackResult(SearchResult):
|
|||
|
||||
def _parse_filename_metadata(self):
|
||||
"""Extract artist, title, album from filename patterns"""
|
||||
import re
|
||||
import os
|
||||
|
||||
# Get just the filename without extension and path
|
||||
base_name = os.path.splitext(os.path.basename(self.filename))[0]
|
||||
|
||||
# Common patterns for track naming
|
||||
patterns = [
|
||||
r'^(\d+)\s*[-\.]\s*(.+?)\s*[-–]\s*(.+)$', # "01 - Artist - Title" or "01. Artist - Title"
|
||||
r'^(.+?)\s*[-–]\s*(.+)$', # "Artist - Title"
|
||||
r'^(\d+)\s*[-\.]\s*(.+)$', # "01 - Title" or "01. Title"
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
match = re.match(pattern, base_name)
|
||||
if match:
|
||||
groups = match.groups()
|
||||
if len(groups) == 3: # Track number, artist, title
|
||||
try:
|
||||
self.track_number = int(groups[0])
|
||||
self.artist = self.artist or groups[1].strip()
|
||||
self.title = self.title or groups[2].strip()
|
||||
except ValueError:
|
||||
# First group might not be a number
|
||||
self.artist = self.artist or groups[0].strip()
|
||||
self.title = self.title or f"{groups[1]} - {groups[2]}".strip()
|
||||
elif len(groups) == 2:
|
||||
if groups[0].isdigit(): # Track number and title
|
||||
try:
|
||||
self.track_number = int(groups[0])
|
||||
self.title = self.title or groups[1].strip()
|
||||
except ValueError:
|
||||
pass
|
||||
else: # Artist and title
|
||||
self.artist = self.artist or groups[0].strip()
|
||||
self.title = self.title or groups[1].strip()
|
||||
break
|
||||
|
||||
# Fallback: use filename as title if nothing was extracted
|
||||
if not self.title:
|
||||
self.title = base_name
|
||||
|
||||
# Try to extract album from directory path
|
||||
if not self.album and '/' in self.filename:
|
||||
path_parts = self.filename.split('/')
|
||||
if len(path_parts) >= 2:
|
||||
# Look for album-like directory names
|
||||
for part in reversed(path_parts[:-1]): # Exclude filename
|
||||
if part and not part.startswith('@'): # Skip system directories
|
||||
# Clean up common patterns
|
||||
cleaned = re.sub(r'^\d+\s*[-\.]\s*', '', part) # Remove leading numbers
|
||||
if len(cleaned) > 3: # Must be substantial
|
||||
self.album = cleaned
|
||||
break
|
||||
parsed = parse_filename_metadata(self.filename)
|
||||
if not self.artist and parsed.get("artist"):
|
||||
self.artist = parsed["artist"]
|
||||
if not self.title and parsed.get("title"):
|
||||
self.title = parsed["title"]
|
||||
if not self.album and parsed.get("album"):
|
||||
self.album = parsed["album"]
|
||||
if self.track_number is None:
|
||||
track_number = parsed.get("track_number")
|
||||
if track_number is not None:
|
||||
self.track_number = track_number
|
||||
|
||||
@dataclass
|
||||
class AlbumResult:
|
||||
|
|
@ -1829,4 +1787,4 @@ class SoulseekClient:
|
|||
|
||||
def __del__(self):
|
||||
# No persistent session to clean up
|
||||
pass
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -2955,9 +2955,6 @@ class WatchlistScanner:
|
|||
}
|
||||
|
||||
track_data = {
|
||||
'spotify_track_id': track['id'],
|
||||
'spotify_album_id': album_data['id'],
|
||||
'spotify_artist_id': artist.spotify_artist_id,
|
||||
'track_name': track['name'],
|
||||
'artist_name': artist.artist_name,
|
||||
'album_name': album_data.get('name', 'Unknown Album'),
|
||||
|
|
|
|||
189
tests/imports/test_import_album.py
Normal file
189
tests/imports/test_import_album.py
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
import core.imports.album as import_album
|
||||
|
||||
|
||||
class _FakeEngine:
|
||||
def normalize_string(self, text):
|
||||
return str(text or "").strip().lower()
|
||||
|
||||
def similarity_score(self, left, right):
|
||||
return 1.0 if left == right else 0.0
|
||||
|
||||
|
||||
def test_resolve_album_artist_context_uses_provider_genres(monkeypatch):
|
||||
class _FakeClient:
|
||||
def get_artist(self, artist_id):
|
||||
assert artist_id == "artist-1"
|
||||
return {"genres": ["rock", "indie"]}
|
||||
|
||||
monkeypatch.setattr(import_album, "get_client_for_source", lambda source: _FakeClient() if source == "spotify" else None)
|
||||
|
||||
context = import_album.resolve_album_artist_context(
|
||||
{
|
||||
"id": "album-1",
|
||||
"name": "Album One",
|
||||
"artist_id": "artist-1",
|
||||
"artists": [{"name": "Artist One", "id": "artist-1"}],
|
||||
},
|
||||
source="spotify",
|
||||
)
|
||||
|
||||
assert context == {
|
||||
"id": "artist-1",
|
||||
"name": "Artist One",
|
||||
"genres": ["rock", "indie"],
|
||||
"source": "spotify",
|
||||
}
|
||||
|
||||
|
||||
def test_build_album_import_context_is_neutral():
|
||||
context = import_album.build_album_import_context(
|
||||
{
|
||||
"id": "album-1",
|
||||
"name": "Album One",
|
||||
"artist": "Artist One",
|
||||
"artist_id": "artist-1",
|
||||
"artists": [{"name": "Artist One", "id": "artist-1"}],
|
||||
"release_date": "2024-01-01",
|
||||
"total_tracks": 12,
|
||||
"album_type": "album",
|
||||
"image_url": "https://img.example/album.jpg",
|
||||
"source": "deezer",
|
||||
},
|
||||
{
|
||||
"id": "track-1",
|
||||
"name": "Song One",
|
||||
"track_number": 3,
|
||||
"disc_number": 2,
|
||||
"duration_ms": 180000,
|
||||
"artists": [{"name": "Artist One"}],
|
||||
"uri": "deezer:track:track-1",
|
||||
"album": "Album One",
|
||||
"source": "deezer",
|
||||
},
|
||||
artist_context={
|
||||
"id": "artist-1",
|
||||
"name": "Artist One",
|
||||
"genres": ["rock"],
|
||||
"source": "deezer",
|
||||
},
|
||||
total_discs=2,
|
||||
source="deezer",
|
||||
)
|
||||
|
||||
assert context["artist"]["name"] == "Artist One"
|
||||
assert context["artist"]["genres"] == ["rock"]
|
||||
assert context["album"]["name"] == "Album One"
|
||||
assert context["album"]["total_discs"] == 2
|
||||
assert context["track_info"]["name"] == "Song One"
|
||||
assert context["track_info"]["track_number"] == 3
|
||||
assert context["original_search_result"]["clean_title"] == "Song One"
|
||||
assert context["source"] == "deezer"
|
||||
assert "spotify_artist" not in context
|
||||
assert "spotify_album" not in context
|
||||
|
||||
|
||||
def test_build_album_import_match_payload_uses_generic_track_keys(monkeypatch, tmp_path):
|
||||
staging_root = tmp_path / "Staging"
|
||||
staging_root.mkdir()
|
||||
(staging_root / "Song One.flac").write_text("fake")
|
||||
|
||||
monkeypatch.setattr(import_album, "_get_matching_engine", lambda: _FakeEngine())
|
||||
monkeypatch.setattr(
|
||||
import_album,
|
||||
"collect_staging_files",
|
||||
lambda file_paths=None: [
|
||||
{
|
||||
"filename": "Song One.flac",
|
||||
"full_path": str(staging_root / "Song One.flac"),
|
||||
"title": "Song One",
|
||||
"artist": "Artist One",
|
||||
"album": "Album One",
|
||||
"albumartist": "Artist One",
|
||||
"track_number": 1,
|
||||
"disc_number": 1,
|
||||
}
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
import_album,
|
||||
"get_artist_album_tracks",
|
||||
lambda album_id, artist_name="", album_name="", source=None: {
|
||||
"success": True,
|
||||
"album": {
|
||||
"id": album_id,
|
||||
"name": "Album One",
|
||||
"artist": "Artist One",
|
||||
"artist_name": "Artist One",
|
||||
"artist_id": "artist-1",
|
||||
"artists": [{"name": "Artist One", "id": "artist-1"}],
|
||||
"release_date": "2024-01-01",
|
||||
"total_tracks": 1,
|
||||
"total_discs": 1,
|
||||
"album_type": "album",
|
||||
"image_url": "https://img.example/album.jpg",
|
||||
"images": [{"url": "https://img.example/album.jpg"}],
|
||||
"source": "spotify",
|
||||
},
|
||||
"tracks": [
|
||||
{
|
||||
"id": "track-1",
|
||||
"name": "Song One",
|
||||
"track_number": 1,
|
||||
"disc_number": 1,
|
||||
"duration_ms": 180000,
|
||||
"artists": [{"name": "Artist One"}],
|
||||
"uri": "spotify:track:track-1",
|
||||
"album": {
|
||||
"id": album_id,
|
||||
"name": "Album One",
|
||||
"artist": "Artist One",
|
||||
},
|
||||
"source": "spotify",
|
||||
}
|
||||
],
|
||||
"source": "spotify",
|
||||
"source_priority": ["spotify"],
|
||||
"resolved_album_id": album_id,
|
||||
},
|
||||
)
|
||||
|
||||
result = import_album.build_album_import_match_payload(
|
||||
"album-1",
|
||||
album_name="Album One",
|
||||
album_artist="Artist One",
|
||||
source="spotify",
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["album"]["artist"] == "Artist One"
|
||||
assert result["source"] == "spotify"
|
||||
assert result["matches"] == [
|
||||
{
|
||||
"track": {
|
||||
"id": "track-1",
|
||||
"name": "Song One",
|
||||
"track_number": 1,
|
||||
"disc_number": 1,
|
||||
"duration_ms": 180000,
|
||||
"artists": [{"name": "Artist One"}],
|
||||
"uri": "spotify:track:track-1",
|
||||
"album": {
|
||||
"id": "album-1",
|
||||
"name": "Album One",
|
||||
"artist": "Artist One",
|
||||
},
|
||||
"source": "spotify",
|
||||
},
|
||||
"staging_file": {
|
||||
"filename": "Song One.flac",
|
||||
"full_path": str(staging_root / "Song One.flac"),
|
||||
"title": "Song One",
|
||||
"artist": "Artist One",
|
||||
"album": "Album One",
|
||||
"albumartist": "Artist One",
|
||||
"track_number": 1,
|
||||
"disc_number": 1,
|
||||
},
|
||||
"confidence": 1.0,
|
||||
}
|
||||
]
|
||||
311
tests/imports/test_import_context.py
Normal file
311
tests/imports/test_import_context.py
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
import pytest
|
||||
|
||||
from core.imports.context import (
|
||||
build_import_album_info,
|
||||
detect_album_info_web,
|
||||
get_import_clean_album,
|
||||
get_import_clean_artist,
|
||||
get_import_clean_title,
|
||||
get_import_has_clean_metadata,
|
||||
get_import_has_full_metadata,
|
||||
get_import_source,
|
||||
get_import_source_ids,
|
||||
get_library_source_id_columns,
|
||||
get_source_tag_names,
|
||||
normalize_import_context,
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_import_context_promotes_neutral_fields_without_legacy_aliases():
|
||||
context = {
|
||||
"source": "deezer",
|
||||
"spotify_artist": {"name": "Artist One", "id": "artist-1"},
|
||||
"spotify_album": {
|
||||
"name": "Album One",
|
||||
"id": "album-1",
|
||||
"release_date": "2024-01-01",
|
||||
"total_tracks": 12,
|
||||
"album_type": "album",
|
||||
"image_url": "https://img.example/album.jpg",
|
||||
},
|
||||
"track_info": {
|
||||
"name": "Song One",
|
||||
"id": "track-1",
|
||||
"track_number": 3,
|
||||
"disc_number": 2,
|
||||
"artists": [{"name": "Artist One"}],
|
||||
},
|
||||
"original_search_result": {
|
||||
"spotify_clean_title": "Song One",
|
||||
"spotify_clean_album": "Album One",
|
||||
"spotify_clean_artist": "Artist One",
|
||||
},
|
||||
"has_clean_spotify_data": True,
|
||||
"has_full_spotify_metadata": False,
|
||||
}
|
||||
|
||||
normalized = normalize_import_context(context)
|
||||
|
||||
assert normalized["artist"]["name"] == "Artist One"
|
||||
assert normalized["album"]["name"] == "Album One"
|
||||
assert "spotify_artist" not in normalized
|
||||
assert "spotify_album" not in normalized
|
||||
assert normalized["original_search_result"]["clean_title"] == "Song One"
|
||||
assert normalized["original_search_result"]["clean_album"] == "Album One"
|
||||
assert normalized["original_search_result"]["clean_artist"] == "Artist One"
|
||||
assert "spotify_clean_title" not in normalized["original_search_result"]
|
||||
assert "spotify_clean_album" not in normalized["original_search_result"]
|
||||
assert "spotify_clean_artist" not in normalized["original_search_result"]
|
||||
assert get_import_clean_title(normalized) == "Song One"
|
||||
assert get_import_clean_album(normalized) == "Album One"
|
||||
assert get_import_clean_artist(normalized) == "Artist One"
|
||||
assert get_import_source(normalized) == "deezer"
|
||||
assert get_import_has_clean_metadata(normalized) is True
|
||||
assert get_import_has_full_metadata(normalized) is False
|
||||
|
||||
|
||||
def test_normalize_import_context_promotes_legacy_source_alias():
|
||||
context = {
|
||||
"_source": "spotify",
|
||||
"artist": {"name": "Artist One", "id": "artist-1"},
|
||||
"album": {"name": "Album One", "id": "album-1"},
|
||||
"track_info": {"name": "Song One", "id": "track-1"},
|
||||
"original_search_result": {"title": "Song One"},
|
||||
}
|
||||
|
||||
normalized = normalize_import_context(context)
|
||||
|
||||
assert normalized["source"] == "spotify"
|
||||
assert "_source" not in normalized
|
||||
assert get_import_source(normalized) == "spotify"
|
||||
|
||||
|
||||
def test_normalize_import_context_promotes_search_result_when_original_search_missing():
|
||||
context = {
|
||||
"source": "spotify",
|
||||
"track_info": {"name": "Song One", "id": "track-1"},
|
||||
"search_result": {
|
||||
"title": "Song One",
|
||||
"album": "Album One",
|
||||
"artist": "Artist One",
|
||||
"spotify_clean_title": "Song One",
|
||||
"spotify_clean_album": "Album One",
|
||||
"spotify_clean_artist": "Artist One",
|
||||
},
|
||||
}
|
||||
|
||||
normalized = normalize_import_context(context)
|
||||
|
||||
assert normalized["original_search_result"]["clean_title"] == "Song One"
|
||||
assert normalized["original_search_result"]["clean_album"] == "Album One"
|
||||
assert normalized["original_search_result"]["clean_artist"] == "Artist One"
|
||||
assert "spotify_clean_title" not in normalized["original_search_result"]
|
||||
assert "spotify_clean_album" not in normalized["original_search_result"]
|
||||
assert "spotify_clean_artist" not in normalized["original_search_result"]
|
||||
assert get_import_clean_title(normalized) == "Song One"
|
||||
assert get_import_clean_album(normalized) == "Album One"
|
||||
assert get_import_clean_artist(normalized) == "Artist One"
|
||||
|
||||
|
||||
def test_neutral_import_context_helpers_work_without_legacy_aliases():
|
||||
context = {
|
||||
"source": "deezer",
|
||||
"artist": {"name": "Artist One", "id": "artist-1"},
|
||||
"album": {
|
||||
"name": "Album One",
|
||||
"id": "album-1",
|
||||
"release_date": "2024-01-01",
|
||||
"total_tracks": 12,
|
||||
"album_type": "album",
|
||||
"image_url": "https://img.example/album.jpg",
|
||||
},
|
||||
"track_info": {
|
||||
"name": "Song One",
|
||||
"id": "track-1",
|
||||
"track_number": 3,
|
||||
"disc_number": 2,
|
||||
"artists": [{"name": "Artist One"}],
|
||||
},
|
||||
"original_search_result": {
|
||||
"title": "Song One",
|
||||
"artist": "Artist One",
|
||||
"album": "Album One",
|
||||
"clean_title": "Song One",
|
||||
"clean_album": "Album One",
|
||||
"clean_artist": "Artist One",
|
||||
},
|
||||
"has_clean_metadata": True,
|
||||
"has_full_metadata": True,
|
||||
}
|
||||
|
||||
assert get_import_clean_title(context) == "Song One"
|
||||
assert get_import_clean_album(context) == "Album One"
|
||||
assert get_import_clean_artist(context) == "Artist One"
|
||||
assert get_import_source(context) == "deezer"
|
||||
|
||||
normalized = normalize_import_context(context)
|
||||
assert normalized["artist"]["name"] == "Artist One"
|
||||
assert normalized["album"]["name"] == "Album One"
|
||||
assert normalized["original_search_result"]["clean_title"] == "Song One"
|
||||
assert "spotify_artist" not in normalized
|
||||
assert "spotify_album" not in normalized
|
||||
assert get_import_has_clean_metadata(normalized) is True
|
||||
assert get_import_has_full_metadata(normalized) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source,expected_tags,expected_columns",
|
||||
[
|
||||
(
|
||||
"spotify",
|
||||
{"track": "SPOTIFY_TRACK_ID", "artist": "SPOTIFY_ARTIST_ID", "album": "SPOTIFY_ALBUM_ID"},
|
||||
{"artist": "spotify_artist_id", "album": "spotify_album_id", "track": "spotify_track_id"},
|
||||
),
|
||||
(
|
||||
"itunes",
|
||||
{"track": "ITUNES_TRACK_ID", "artist": "ITUNES_ARTIST_ID", "album": "ITUNES_ALBUM_ID"},
|
||||
{"artist": "itunes_artist_id", "album": "itunes_album_id", "track": "itunes_track_id"},
|
||||
),
|
||||
(
|
||||
"deezer",
|
||||
{"track": "DEEZER_TRACK_ID", "artist": "DEEZER_ARTIST_ID", "album": None},
|
||||
{"artist": "deezer_id", "album": "deezer_id", "track": "deezer_id"},
|
||||
),
|
||||
(
|
||||
"hydrabase",
|
||||
{"track": None, "artist": None, "album": None},
|
||||
{"artist": "soul_id", "album": "soul_id", "track": "soul_id", "track_album": "album_soul_id"},
|
||||
),
|
||||
(
|
||||
"discogs",
|
||||
{"track": None, "artist": None, "album": None},
|
||||
{"artist": "discogs_id", "album": "discogs_id", "track": None},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_source_tag_and_library_column_mappings(source, expected_tags, expected_columns):
|
||||
assert get_source_tag_names(source) == expected_tags
|
||||
assert get_library_source_id_columns(source) == expected_columns
|
||||
|
||||
|
||||
def test_get_import_source_ids_prefers_nested_source_specific_ids():
|
||||
context = normalize_import_context(
|
||||
{
|
||||
"source": "deezer",
|
||||
"artist": {"deezer_id": "deezer-artist-1"},
|
||||
"album": {"deezer_id": "deezer-album-1"},
|
||||
"track_info": {"deezer_id": "deezer-track-1"},
|
||||
"original_search_result": {"source_artist_id": "deezer-artist-1"},
|
||||
}
|
||||
)
|
||||
|
||||
assert get_import_source_ids(context) == {
|
||||
"track_id": "deezer-track-1",
|
||||
"artist_id": "deezer-artist-1",
|
||||
"album_id": "deezer-album-1",
|
||||
}
|
||||
|
||||
|
||||
def test_build_import_album_info_uses_normalized_album_context():
|
||||
context = normalize_import_context(
|
||||
{
|
||||
"source": "deezer",
|
||||
"artist": {"name": "Artist One"},
|
||||
"album": {
|
||||
"name": "Album One",
|
||||
"image_url": "https://img.example/album.jpg",
|
||||
"release_date": "2024-05-01",
|
||||
"total_tracks": 8,
|
||||
"album_type": "album",
|
||||
},
|
||||
"track_info": {
|
||||
"name": "Song One",
|
||||
"track_number": 4,
|
||||
"disc_number": 2,
|
||||
"duration_ms": 240000,
|
||||
"artists": [{"name": "Artist One"}],
|
||||
},
|
||||
"original_search_result": {
|
||||
"title": "Song One",
|
||||
"album": "Album One",
|
||||
"clean_title": "Song One",
|
||||
"clean_album": "Album One",
|
||||
"clean_artist": "Artist One",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
album_info = build_import_album_info(context)
|
||||
|
||||
assert album_info["is_album"] is True
|
||||
assert album_info["album_name"] == "Album One"
|
||||
assert album_info["track_number"] == 4
|
||||
assert album_info["disc_number"] == 2
|
||||
assert album_info["clean_track_name"] == "Song One"
|
||||
assert album_info["album_image_url"] == "https://img.example/album.jpg"
|
||||
assert album_info["source"] == "deezer"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"album_name,track_name,artist_name",
|
||||
[
|
||||
("Song One", "Song One", "Artist One"),
|
||||
("Artist One", "Different Song", "Artist One"),
|
||||
],
|
||||
)
|
||||
def test_detect_album_info_web_returns_none_for_ambiguous_album_names(album_name, track_name, artist_name):
|
||||
context = normalize_import_context(
|
||||
{
|
||||
"source": "deezer",
|
||||
"artist": {"name": artist_name},
|
||||
"album": {"name": album_name, "total_tracks": 12, "album_type": "album"},
|
||||
"track_info": {"name": track_name, "track_number": 4, "disc_number": 1},
|
||||
"original_search_result": {
|
||||
"title": track_name,
|
||||
"clean_title": track_name,
|
||||
"clean_album": album_name,
|
||||
"clean_artist": artist_name,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert detect_album_info_web(context) is None
|
||||
|
||||
|
||||
def test_detect_album_info_web_forces_album_when_track_and_artist_differ():
|
||||
context = normalize_import_context(
|
||||
{
|
||||
"source": "deezer",
|
||||
"artist": {"name": "Artist One"},
|
||||
"album": {
|
||||
"name": "Album One",
|
||||
"image_url": "https://img.example/album.jpg",
|
||||
"release_date": "2024-05-01",
|
||||
"total_tracks": 1,
|
||||
"album_type": "album",
|
||||
},
|
||||
"track_info": {
|
||||
"name": "Song One",
|
||||
"track_number": 4,
|
||||
"disc_number": 2,
|
||||
"duration_ms": 240000,
|
||||
"artists": [{"name": "Artist One"}],
|
||||
},
|
||||
"original_search_result": {
|
||||
"title": "Song One",
|
||||
"album": "Album One",
|
||||
"clean_title": "Song One",
|
||||
"clean_album": "Album One",
|
||||
"clean_artist": "Artist One",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
album_info = detect_album_info_web(context)
|
||||
|
||||
assert album_info is not None
|
||||
assert album_info["is_album"] is True
|
||||
assert album_info["confidence"] == 0.5
|
||||
assert album_info["album_name"] == "Album One"
|
||||
assert album_info["track_number"] == 4
|
||||
assert album_info["disc_number"] == 2
|
||||
126
tests/imports/test_import_file_ops.py
Normal file
126
tests/imports/test_import_file_ops.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
import sys
|
||||
import types
|
||||
|
||||
from core.imports.file_ops import (
|
||||
cleanup_empty_directories,
|
||||
safe_move_file,
|
||||
)
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
from core.imports.staging import read_staging_file_metadata
|
||||
|
||||
|
||||
def test_extract_track_number_from_filename_handles_common_patterns():
|
||||
assert extract_track_number_from_filename("01 - Song.mp3") == 1
|
||||
assert extract_track_number_from_filename("1-03 - Song.mp3") == 3
|
||||
assert extract_track_number_from_filename("Artist - Song.mp3") == 1
|
||||
|
||||
|
||||
def test_safe_move_file_replaces_existing_destination(tmp_path):
|
||||
src = tmp_path / "source.flac"
|
||||
dst_dir = tmp_path / "dest"
|
||||
dst_dir.mkdir()
|
||||
dst = dst_dir / "track.flac"
|
||||
|
||||
src.write_text("new")
|
||||
dst.write_text("old")
|
||||
|
||||
safe_move_file(src, dst)
|
||||
|
||||
assert not src.exists()
|
||||
assert dst.read_text() == "new"
|
||||
|
||||
|
||||
def test_cleanup_empty_directories_removes_nested_empty_paths(tmp_path):
|
||||
download_root = tmp_path / "downloads"
|
||||
nested_dir = download_root / "Artist" / "Album"
|
||||
nested_dir.mkdir(parents=True)
|
||||
moved_file_path = nested_dir / "track.flac"
|
||||
|
||||
cleanup_empty_directories(str(download_root), str(moved_file_path))
|
||||
|
||||
assert not nested_dir.exists()
|
||||
assert not (download_root / "Artist").exists()
|
||||
assert download_root.exists()
|
||||
|
||||
|
||||
def test_read_staging_file_metadata_reads_tags(monkeypatch, tmp_path):
|
||||
file_path = tmp_path / "Song One.flac"
|
||||
file_path.write_text("fake")
|
||||
|
||||
class DummyTags:
|
||||
def __init__(self):
|
||||
self.values = {
|
||||
"title": ["Song One"],
|
||||
"artist": ["Artist One"],
|
||||
"albumartist": ["Album Artist"],
|
||||
"album": ["Album One"],
|
||||
"tracknumber": ["03/12"],
|
||||
"discnumber": ["2/3"],
|
||||
}
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self.values.get(key, default)
|
||||
|
||||
fake_mutagen = types.ModuleType("mutagen")
|
||||
fake_mutagen.File = lambda path, easy=True: DummyTags()
|
||||
monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen)
|
||||
|
||||
metadata = read_staging_file_metadata(str(file_path), file_path.name)
|
||||
|
||||
assert metadata == {
|
||||
"title": "Song One",
|
||||
"artist": "Artist One",
|
||||
"albumartist": "Album Artist",
|
||||
"album": "Album One",
|
||||
"track_number": 3,
|
||||
"disc_number": 2,
|
||||
}
|
||||
|
||||
|
||||
def test_read_staging_file_metadata_falls_back_to_filename_track_number(monkeypatch, tmp_path):
|
||||
file_path = tmp_path / "07 - Song Two.flac"
|
||||
file_path.write_text("fake")
|
||||
|
||||
fake_mutagen = types.ModuleType("mutagen")
|
||||
fake_mutagen.File = lambda path, easy=True: None
|
||||
monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen)
|
||||
|
||||
metadata = read_staging_file_metadata(str(file_path), file_path.name)
|
||||
|
||||
assert metadata["title"] == "07 - Song Two"
|
||||
assert metadata["track_number"] == 7
|
||||
assert metadata["disc_number"] == 1
|
||||
|
||||
|
||||
def test_read_staging_file_metadata_uses_filename_fallbacks_when_tags_are_invalid(monkeypatch, tmp_path):
|
||||
file_path = tmp_path / "02 - Song Three.flac"
|
||||
file_path.write_text("fake")
|
||||
|
||||
class DummyTags:
|
||||
def __init__(self):
|
||||
self.values = {
|
||||
"title": [""],
|
||||
"artist": "Artist One",
|
||||
"albumartist": "",
|
||||
"album": ["Album One"],
|
||||
"tracknumber": ["not-a-number"],
|
||||
"discnumber": ["bad/disc"],
|
||||
}
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self.values.get(key, default)
|
||||
|
||||
fake_mutagen = types.ModuleType("mutagen")
|
||||
fake_mutagen.File = lambda path, easy=True: DummyTags()
|
||||
monkeypatch.setitem(sys.modules, "mutagen", fake_mutagen)
|
||||
|
||||
metadata = read_staging_file_metadata(str(file_path), file_path.name)
|
||||
|
||||
assert metadata == {
|
||||
"title": "02 - Song Three",
|
||||
"artist": "Artist One",
|
||||
"albumartist": "Artist One",
|
||||
"album": "Album One",
|
||||
"track_number": 2,
|
||||
"disc_number": 1,
|
||||
}
|
||||
33
tests/imports/test_import_filename.py
Normal file
33
tests/imports/test_import_filename.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import pytest
|
||||
|
||||
from core.imports.filename import parse_filename_metadata
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"filename,expected",
|
||||
[
|
||||
(
|
||||
"01 - Artist One - Title One.mp3",
|
||||
{"artist": "Artist One", "title": "Title One", "album": "", "track_number": 1},
|
||||
),
|
||||
(
|
||||
"Artist Two - Title Two.flac",
|
||||
{"artist": "Artist Two", "title": "Title Two", "album": "", "track_number": None},
|
||||
),
|
||||
(
|
||||
r"Artist Three\Album Three\03 - Title Three.ogg",
|
||||
{"artist": "", "title": "Title Three", "album": "Album Three", "track_number": 3},
|
||||
),
|
||||
(
|
||||
"Loose Song.wav",
|
||||
{"artist": "", "title": "Loose Song", "album": "", "track_number": None},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_parse_filename_metadata_handles_common_patterns(filename, expected):
|
||||
parsed = parse_filename_metadata(filename)
|
||||
|
||||
assert parsed["artist"] == expected["artist"]
|
||||
assert parsed["title"] == expected["title"]
|
||||
assert parsed["album"] == expected["album"]
|
||||
assert parsed["track_number"] == expected["track_number"]
|
||||
47
tests/imports/test_import_guards.py
Normal file
47
tests/imports/test_import_guards.py
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
from types import SimpleNamespace
|
||||
|
||||
from core.imports import guards
|
||||
|
||||
|
||||
class _FakeDB:
|
||||
def __init__(self, quality_profile):
|
||||
self._quality_profile = quality_profile
|
||||
|
||||
def get_quality_profile(self):
|
||||
return self._quality_profile
|
||||
|
||||
|
||||
def test_check_flac_bit_depth_rejects_strict_mismatch(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
guards,
|
||||
"MusicDatabase",
|
||||
lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": False}}}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
guards,
|
||||
"_get_config_manager",
|
||||
lambda: SimpleNamespace(get=lambda _key, default=None: False),
|
||||
)
|
||||
|
||||
context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}}
|
||||
|
||||
assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) == (
|
||||
"FLAC bit depth mismatch: file is 24-bit, preference is 16-bit"
|
||||
)
|
||||
|
||||
|
||||
def test_check_flac_bit_depth_allows_fallback_when_enabled(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
guards,
|
||||
"MusicDatabase",
|
||||
lambda: _FakeDB({"qualities": {"flac": {"bit_depth": "16", "bit_depth_fallback": True}}}),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
guards,
|
||||
"_get_config_manager",
|
||||
lambda: SimpleNamespace(get=lambda _key, default=None: False),
|
||||
)
|
||||
|
||||
context = {"_audio_quality": "FLAC 24bit", "track_info": {"name": "Song One"}}
|
||||
|
||||
assert guards.check_flac_bit_depth("/tmp/Song One.flac", context) is None
|
||||
232
tests/imports/test_import_paths.py
Normal file
232
tests/imports/test_import_paths.py
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
import os
|
||||
|
||||
import core.imports.album_naming as album_naming
|
||||
import core.imports.paths as import_paths
|
||||
|
||||
|
||||
class _Config:
|
||||
def __init__(self, values):
|
||||
self._values = values
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self._values.get(key, default)
|
||||
|
||||
|
||||
def test_sanitize_filename_replaces_illegal_characters():
|
||||
assert import_paths.sanitize_filename("AC/DC: Song?") == "AC_DC_ Song_"
|
||||
assert import_paths.sanitize_filename("AUX.txt").startswith("_")
|
||||
|
||||
|
||||
def test_build_simple_download_destination_uses_album_folder(monkeypatch, tmp_path):
|
||||
config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")})
|
||||
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config)
|
||||
|
||||
destination, album_name, filename = import_paths.build_simple_download_destination(
|
||||
{
|
||||
"search_result": {
|
||||
"filename": "Album Folder/source.flac",
|
||||
"album": "Album Folder",
|
||||
}
|
||||
},
|
||||
str(tmp_path / "source.flac"),
|
||||
)
|
||||
|
||||
assert destination == tmp_path / "Transfer" / "Album Folder" / "source.flac"
|
||||
assert album_name == "Album Folder"
|
||||
assert filename == "source.flac"
|
||||
assert destination.parent.exists()
|
||||
|
||||
|
||||
def test_build_simple_download_destination_falls_back_to_transfer_root(monkeypatch, tmp_path):
|
||||
config = _Config({"soulseek.transfer_path": str(tmp_path / "Transfer")})
|
||||
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config)
|
||||
|
||||
destination, album_name, filename = import_paths.build_simple_download_destination(
|
||||
{
|
||||
"search_result": {
|
||||
"filename": "source.flac",
|
||||
"album": "Unknown Album",
|
||||
}
|
||||
},
|
||||
str(tmp_path / "source.flac"),
|
||||
)
|
||||
|
||||
assert destination == tmp_path / "Transfer" / "source.flac"
|
||||
assert album_name == ""
|
||||
assert filename == "source.flac"
|
||||
assert destination.parent.exists()
|
||||
|
||||
|
||||
def test_get_file_path_from_template_raw_handles_quality_and_disc_placeholders(monkeypatch):
|
||||
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config({}))
|
||||
|
||||
folder_path, filename = import_paths.get_file_path_from_template_raw(
|
||||
"$artist/$album/$discnum - $title [$quality]",
|
||||
{
|
||||
"artist": "Artist One",
|
||||
"album": "Album One",
|
||||
"title": "Song One",
|
||||
"quality": "FLAC 16bit",
|
||||
"disc_number": 3,
|
||||
},
|
||||
)
|
||||
|
||||
assert folder_path == os.path.join("Artist One", "Album One")
|
||||
assert filename == "3 - Song One [FLAC 16bit]"
|
||||
|
||||
|
||||
def test_resolve_album_group_upgrades_standard_to_deluxe():
|
||||
album_naming.clear_album_grouping_cache()
|
||||
|
||||
artist_context = {"name": "Cache Artist"}
|
||||
standard_album = {"album_name": "Cache Album"}
|
||||
deluxe_album = {"album_name": "Cache Album (Deluxe Edition)"}
|
||||
|
||||
assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album"
|
||||
assert import_paths.resolve_album_group(artist_context, deluxe_album) == "Cache Album (Deluxe Edition)"
|
||||
assert import_paths.resolve_album_group(artist_context, standard_album) == "Cache Album (Deluxe Edition)"
|
||||
|
||||
|
||||
def test_build_final_path_for_track_uses_template_and_disc_folder(monkeypatch, tmp_path):
|
||||
config = _Config(
|
||||
{
|
||||
"soulseek.transfer_path": str(tmp_path / "Transfer"),
|
||||
"file_organization.enabled": True,
|
||||
"file_organization.templates": {
|
||||
"album_path": "$albumartist/$albumartist - $album/$track - $title",
|
||||
"single_path": "$artist/$artist - $title",
|
||||
},
|
||||
"file_organization.collab_artist_mode": "first",
|
||||
"file_organization.disc_label": "Disc",
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config)
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_get_album_tracks_for_source(source, album_id):
|
||||
calls.append((source, album_id))
|
||||
return {
|
||||
"items": [
|
||||
{"disc_number": 1},
|
||||
{"disc_number": 2},
|
||||
]
|
||||
}
|
||||
|
||||
monkeypatch.setattr(import_paths, "_get_album_tracks_for_source", fake_get_album_tracks_for_source)
|
||||
|
||||
context = {
|
||||
"artist": {"name": "Artist One"},
|
||||
"album": {
|
||||
"name": "Album One",
|
||||
"id": "album-1",
|
||||
"release_date": "2026-01-01",
|
||||
"total_tracks": 12,
|
||||
"album_type": "album",
|
||||
"artists": [{"name": "Artist One"}],
|
||||
},
|
||||
"track_info": {
|
||||
"name": "Song One",
|
||||
"id": "track-1",
|
||||
"track_number": 4,
|
||||
"disc_number": 1,
|
||||
"artists": [{"name": "Artist One"}],
|
||||
},
|
||||
"original_search_result": {
|
||||
"title": "Song One",
|
||||
"clean_title": "Song One",
|
||||
"clean_album": "Album One",
|
||||
"clean_artist": "Artist One",
|
||||
"artists": [{"name": "Artist One"}],
|
||||
},
|
||||
"source": "deezer",
|
||||
"is_album_download": False,
|
||||
}
|
||||
|
||||
final_path, created = import_paths.build_final_path_for_track(
|
||||
context,
|
||||
{"name": "Artist One"},
|
||||
{
|
||||
"is_album": True,
|
||||
"album_name": "Album One",
|
||||
"track_number": 4,
|
||||
"disc_number": 1,
|
||||
},
|
||||
".flac",
|
||||
)
|
||||
|
||||
assert created is True
|
||||
assert calls == [("deezer", "album-1")]
|
||||
assert final_path == str(
|
||||
tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1" / "04 - Song One.flac"
|
||||
)
|
||||
assert (tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 1").is_dir()
|
||||
|
||||
|
||||
def test_build_final_path_for_track_uses_track_disc_number_without_provider_lookup(monkeypatch, tmp_path):
|
||||
config = _Config(
|
||||
{
|
||||
"soulseek.transfer_path": str(tmp_path / "Transfer"),
|
||||
"file_organization.enabled": True,
|
||||
"file_organization.templates": {
|
||||
"album_path": "$albumartist/$albumartist - $album/$track - $title",
|
||||
"single_path": "$artist/$artist - $title",
|
||||
},
|
||||
"file_organization.collab_artist_mode": "first",
|
||||
"file_organization.disc_label": "Disc",
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: config)
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
import_paths,
|
||||
"_get_album_tracks_for_source",
|
||||
lambda source, album_id: calls.append((source, album_id)) or None,
|
||||
)
|
||||
|
||||
context = {
|
||||
"artist": {"name": "Artist One"},
|
||||
"album": {
|
||||
"name": "Album One",
|
||||
"id": "album-1",
|
||||
"release_date": "2026-01-01",
|
||||
"total_tracks": 12,
|
||||
"album_type": "album",
|
||||
"artists": [{"name": "Artist One"}],
|
||||
},
|
||||
"track_info": {
|
||||
"name": "Song Two",
|
||||
"id": "track-2",
|
||||
"track_number": 4,
|
||||
"disc_number": 2,
|
||||
"artists": [{"name": "Artist One"}],
|
||||
},
|
||||
"original_search_result": {
|
||||
"title": "Song Two",
|
||||
"clean_title": "Song Two",
|
||||
"clean_album": "Album One",
|
||||
"clean_artist": "Artist One",
|
||||
"artists": [{"name": "Artist One"}],
|
||||
},
|
||||
"source": "deezer",
|
||||
"is_album_download": False,
|
||||
}
|
||||
|
||||
final_path, created = import_paths.build_final_path_for_track(
|
||||
context,
|
||||
{"name": "Artist One"},
|
||||
{
|
||||
"is_album": True,
|
||||
"album_name": "Album One",
|
||||
"track_number": 4,
|
||||
"disc_number": 2,
|
||||
},
|
||||
".flac",
|
||||
)
|
||||
|
||||
assert created is True
|
||||
assert calls == []
|
||||
assert final_path == str(
|
||||
tmp_path / "Transfer" / "Artist One" / "Artist One - Album One" / "Disc 2" / "04 - Song Two.flac"
|
||||
)
|
||||
212
tests/imports/test_import_pipeline.py
Normal file
212
tests/imports/test_import_pipeline.py
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
import logging
|
||||
import sys
|
||||
import types
|
||||
|
||||
import core.imports.pipeline as import_pipeline
|
||||
import core.imports.paths as import_paths
|
||||
import core.runtime_state as runtime_state
|
||||
|
||||
|
||||
class _Config:
|
||||
def __init__(self, transfer_path):
|
||||
self.transfer_path = transfer_path
|
||||
|
||||
def get(self, key, default=None):
|
||||
if key == "soulseek.transfer_path":
|
||||
return self.transfer_path
|
||||
if key in {"post_processing.replaygain_enabled", "lossy_copy.enabled", "lossy_copy.delete_original", "import.replace_lower_quality"}:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
class _FakeAcoustidVerifier:
|
||||
def quick_check_available(self):
|
||||
return False, "disabled"
|
||||
|
||||
|
||||
class _ImmediateThread:
|
||||
def __init__(self, target=None, daemon=None):
|
||||
self._target = target
|
||||
|
||||
def start(self):
|
||||
if self._target:
|
||||
self._target()
|
||||
|
||||
|
||||
def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch):
|
||||
transfer_root = tmp_path / "Transfer"
|
||||
transfer_root.mkdir()
|
||||
source_path = tmp_path / "source.flac"
|
||||
source_path.write_bytes(b"audio")
|
||||
|
||||
context_key = "ctx-1"
|
||||
task_id = "task-1"
|
||||
batch_id = "batch-1"
|
||||
context = {
|
||||
"search_result": {
|
||||
"is_simple_download": True,
|
||||
"filename": "Album Folder/source.flac",
|
||||
"album": "Album Folder",
|
||||
},
|
||||
"track_info": {},
|
||||
"original_search_result": {},
|
||||
"is_album_download": False,
|
||||
"task_id": task_id,
|
||||
"batch_id": batch_id,
|
||||
}
|
||||
|
||||
mark_calls = []
|
||||
completion_calls = []
|
||||
scan_calls = []
|
||||
activity_calls = []
|
||||
wishlist_calls = []
|
||||
|
||||
original_matched_context = dict(runtime_state.matched_downloads_context)
|
||||
original_download_tasks = dict(runtime_state.download_tasks)
|
||||
original_download_batches = dict(runtime_state.download_batches)
|
||||
original_processed_ids = set(runtime_state.processed_download_ids)
|
||||
original_post_locks = dict(runtime_state.post_process_locks)
|
||||
|
||||
runtime_state.matched_downloads_context.clear()
|
||||
runtime_state.download_tasks.clear()
|
||||
runtime_state.download_batches.clear()
|
||||
runtime_state.processed_download_ids.clear()
|
||||
runtime_state.post_process_locks.clear()
|
||||
|
||||
runtime = types.SimpleNamespace(
|
||||
automation_engine=None,
|
||||
on_download_completed=lambda batch, task, success: completion_calls.append((batch, task, success)),
|
||||
web_scan_manager=types.SimpleNamespace(request_scan=lambda reason: scan_calls.append(reason)),
|
||||
repair_worker=None,
|
||||
)
|
||||
|
||||
fake_acoustid = types.ModuleType("core.acoustid_verification")
|
||||
fake_acoustid.AcoustIDVerification = _FakeAcoustidVerifier
|
||||
fake_acoustid.VerificationResult = types.SimpleNamespace(FAIL="FAIL")
|
||||
|
||||
monkeypatch.setitem(sys.modules, "core.acoustid_verification", fake_acoustid)
|
||||
monkeypatch.setattr(import_paths, "_get_config_manager", lambda: _Config(str(transfer_root)))
|
||||
monkeypatch.setattr(import_pipeline, "add_activity_item", lambda *args, **kwargs: activity_calls.append((args, kwargs)))
|
||||
monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "record_library_history_download", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "record_download_provenance", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "check_and_remove_from_wishlist", lambda context: wishlist_calls.append(dict(context)))
|
||||
monkeypatch.setattr(import_pipeline, "_mark_task_completed", lambda task, track_info: mark_calls.append((task, track_info)))
|
||||
monkeypatch.setattr(import_pipeline.threading, "Thread", _ImmediateThread)
|
||||
|
||||
runtime_state.matched_downloads_context[context_key] = context
|
||||
runtime_state.download_tasks[task_id] = {"track_info": {}, "status": "running"}
|
||||
|
||||
try:
|
||||
import_pipeline.post_process_matched_download_with_verification(
|
||||
context_key,
|
||||
context,
|
||||
str(source_path),
|
||||
task_id,
|
||||
batch_id,
|
||||
runtime,
|
||||
)
|
||||
|
||||
expected_path = transfer_root / "Album Folder" / "source.flac"
|
||||
assert expected_path.exists()
|
||||
assert not source_path.exists()
|
||||
assert context["_simple_download_completed"] is True
|
||||
assert context["_final_path"] == str(expected_path)
|
||||
assert mark_calls == [(task_id, {})]
|
||||
assert completion_calls == [(batch_id, task_id, True)]
|
||||
assert context_key not in runtime_state.matched_downloads_context
|
||||
assert scan_calls == ["Simple download completed"]
|
||||
assert wishlist_calls and wishlist_calls[0]["search_result"]["is_simple_download"] is True
|
||||
assert activity_calls
|
||||
finally:
|
||||
runtime_state.matched_downloads_context.clear()
|
||||
runtime_state.matched_downloads_context.update(original_matched_context)
|
||||
runtime_state.download_tasks.clear()
|
||||
runtime_state.download_tasks.update(original_download_tasks)
|
||||
runtime_state.download_batches.clear()
|
||||
runtime_state.download_batches.update(original_download_batches)
|
||||
runtime_state.processed_download_ids.clear()
|
||||
runtime_state.processed_download_ids.update(original_processed_ids)
|
||||
runtime_state.post_process_locks.clear()
|
||||
runtime_state.post_process_locks.update(original_post_locks)
|
||||
|
||||
|
||||
def test_post_process_matched_download_forwards_separate_metadata_runtime(tmp_path, monkeypatch):
|
||||
source_path = tmp_path / "source.flac"
|
||||
source_path.write_bytes(b"audio")
|
||||
target_path = tmp_path / "Album Folder" / "track.flac"
|
||||
|
||||
runtime = types.SimpleNamespace(
|
||||
automation_engine=None,
|
||||
on_download_completed=None,
|
||||
web_scan_manager=None,
|
||||
repair_worker=None,
|
||||
)
|
||||
metadata_runtime = types.SimpleNamespace(marker="metadata-runtime")
|
||||
seen = {}
|
||||
|
||||
monkeypatch.setattr(import_pipeline, "config_manager", types.SimpleNamespace(
|
||||
get=lambda key, default=None: {
|
||||
"post_processing.replaygain_enabled": False,
|
||||
"lossy_copy.enabled": False,
|
||||
"lossy_copy.delete_original": False,
|
||||
"import.replace_lower_quality": False,
|
||||
"soulseek.download_path": str(tmp_path / "downloads"),
|
||||
}.get(key, default)
|
||||
))
|
||||
monkeypatch.setattr(import_pipeline, "normalize_import_context", lambda context: context)
|
||||
monkeypatch.setattr(import_pipeline, "get_import_track_info", lambda context: {"_playlist_folder_mode": True, "_playlist_name": "Playlist"})
|
||||
monkeypatch.setattr(import_pipeline, "get_import_original_search", lambda context: {"title": "Track", "album": "Album"})
|
||||
monkeypatch.setattr(import_pipeline, "get_import_context_artist", lambda context: {"name": "Artist"})
|
||||
monkeypatch.setattr(import_pipeline, "get_import_has_clean_metadata", lambda context: True)
|
||||
monkeypatch.setattr(
|
||||
import_pipeline,
|
||||
"build_import_album_info",
|
||||
lambda context, force_album=False: {
|
||||
"is_album": True,
|
||||
"album_name": "Album",
|
||||
"track_number": 1,
|
||||
"disc_number": 1,
|
||||
"clean_track_name": "Track",
|
||||
"source": "spotify",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(import_pipeline, "resolve_album_group", lambda artist_context, album_info, original_album: album_info["album_name"])
|
||||
monkeypatch.setattr(import_pipeline, "get_import_clean_title", lambda *args, **kwargs: "Track")
|
||||
monkeypatch.setattr(import_pipeline, "get_audio_quality_string", lambda file_path: "")
|
||||
monkeypatch.setattr(import_pipeline, "check_flac_bit_depth", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "build_final_path_for_track", lambda *args, **kwargs: (str(target_path), None))
|
||||
|
||||
def _capture_enhance(file_path, context, artist, album_info, runtime=None):
|
||||
seen["runtime"] = runtime
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(import_pipeline, "enhance_file_metadata", _capture_enhance)
|
||||
monkeypatch.setattr(import_pipeline, "safe_move_file", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "download_cover_art", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "generate_lrc_file", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "downsample_hires_flac", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "create_lossy_copy", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "cleanup_empty_directories", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "record_library_history_download", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "record_download_provenance", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "record_soulsync_library_entry", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "check_and_remove_from_wishlist", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(import_pipeline, "record_retag_download", lambda *args, **kwargs: None)
|
||||
|
||||
context = {
|
||||
"track_info": {"_playlist_folder_mode": True, "_playlist_name": "Playlist"},
|
||||
"original_search_result": {"title": "Track", "album": "Album"},
|
||||
"is_album_download": False,
|
||||
}
|
||||
|
||||
import_pipeline.post_process_matched_download(
|
||||
"ctx-1",
|
||||
context,
|
||||
str(source_path),
|
||||
runtime,
|
||||
metadata_runtime=metadata_runtime,
|
||||
)
|
||||
|
||||
assert seen["runtime"] is metadata_runtime
|
||||
217
tests/imports/test_import_resolution_single_track_context.py
Normal file
217
tests/imports/test_import_resolution_single_track_context.py
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
from types import SimpleNamespace
|
||||
|
||||
from core import metadata_service
|
||||
from core.imports import resolution
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, search_results=None, details=None, artist_details=None):
|
||||
self.search_results = search_results or []
|
||||
self.details = details or {}
|
||||
self.artist_details = artist_details or {}
|
||||
self.calls = []
|
||||
|
||||
def search_tracks(self, query, limit=1, allow_fallback=True):
|
||||
self.calls.append(("search_tracks", query, limit, allow_fallback))
|
||||
return self.search_results
|
||||
|
||||
def get_track_details(self, track_id):
|
||||
self.calls.append(("get_track_details", track_id))
|
||||
return self.details.get(str(track_id))
|
||||
|
||||
def get_artist(self, artist_id):
|
||||
self.calls.append(("get_artist", artist_id))
|
||||
return self.artist_details.get(str(artist_id))
|
||||
|
||||
|
||||
def _track_result(track_id="track-1", name="Song One", artist="Artist One"):
|
||||
return SimpleNamespace(
|
||||
id=track_id,
|
||||
name=name,
|
||||
artists=[artist],
|
||||
album="Album One",
|
||||
duration_ms=123000,
|
||||
track_number=1,
|
||||
disc_number=1,
|
||||
image_url="https://img.example/track.jpg",
|
||||
)
|
||||
|
||||
|
||||
def _track_details(source, track_id="track-1", name="Song One", artist_name="Artist One", artist_id="artist-1"):
|
||||
return {
|
||||
"id": track_id,
|
||||
"name": name,
|
||||
"track_number": 7,
|
||||
"disc_number": 1,
|
||||
"duration_ms": 210000,
|
||||
"explicit": True,
|
||||
"uri": f"{source}:track:{track_id}",
|
||||
"artists": [{"name": artist_name, "id": artist_id}],
|
||||
"album": {
|
||||
"id": f"{source}-album-1",
|
||||
"name": "Album One",
|
||||
"release_date": "2024-01-01",
|
||||
"album_type": "album",
|
||||
"total_tracks": 10,
|
||||
"images": [{"url": f"https://img.example/{source}-album.jpg"}],
|
||||
"artists": [{"name": artist_name, "id": artist_id}],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_get_single_track_import_context_uses_primary_source_priority(monkeypatch):
|
||||
deezer_client = FakeClient(
|
||||
search_results=[_track_result(track_id="deezer-track-1")],
|
||||
details={"deezer-track-1": _track_details("deezer", track_id="deezer-track-1", artist_name="Artist One", artist_id="deezer-artist-1")},
|
||||
artist_details={"deezer-artist-1": {"id": "deezer-artist-1", "genres": ["electronic"]}},
|
||||
)
|
||||
spotify_client = FakeClient()
|
||||
|
||||
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
|
||||
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
|
||||
monkeypatch.setattr(
|
||||
metadata_service,
|
||||
"get_client_for_source",
|
||||
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source),
|
||||
)
|
||||
|
||||
result = resolution.get_single_track_import_context("Song One", "Artist One")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["source"] == "deezer"
|
||||
assert result["source_priority"] == ["deezer", "spotify", "itunes"]
|
||||
assert result["context"]["track_info"]["name"] == "Song One"
|
||||
assert result["context"]["track_info"]["album_id"] == "deezer-album-1"
|
||||
assert result["context"]["album"]["image_url"] == "https://img.example/deezer-album.jpg"
|
||||
assert result["context"]["artist"]["genres"] == ["electronic"]
|
||||
assert result["context"]["original_search_result"]["clean_title"] == "Song One"
|
||||
assert deezer_client.calls == [
|
||||
('search_tracks', 'artist:"Artist One" track:"Song One"', 5, True),
|
||||
("get_track_details", "deezer-track-1"),
|
||||
("get_artist", "deezer-artist-1"),
|
||||
]
|
||||
assert spotify_client.calls == []
|
||||
|
||||
|
||||
def test_get_single_track_import_context_falls_back_to_next_source(monkeypatch):
|
||||
deezer_client = FakeClient(search_results=[])
|
||||
spotify_client = FakeClient(
|
||||
search_results=[_track_result(track_id="spotify-track-1")],
|
||||
details={"spotify-track-1": _track_details("spotify", track_id="spotify-track-1", artist_name="Artist Two", artist_id="spotify-artist-1")},
|
||||
artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["indie"]}},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
|
||||
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
|
||||
monkeypatch.setattr(
|
||||
metadata_service,
|
||||
"get_client_for_source",
|
||||
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source),
|
||||
)
|
||||
|
||||
result = resolution.get_single_track_import_context("Song Two", "Artist Two")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["source"] == "spotify"
|
||||
assert result["context"]["track_info"]["id"] == "spotify-track-1"
|
||||
assert result["context"]["album"]["name"] == "Album One"
|
||||
assert deezer_client.calls == [
|
||||
('search_tracks', 'artist:"Artist Two" track:"Song Two"', 5, True),
|
||||
('search_tracks', 'Song Two', 5, True),
|
||||
('search_tracks', 'Artist Two', 5, True),
|
||||
]
|
||||
assert spotify_client.calls == [
|
||||
("search_tracks", "Song Two Artist Two", 5, False),
|
||||
("get_track_details", "spotify-track-1"),
|
||||
("get_artist", "spotify-artist-1"),
|
||||
]
|
||||
|
||||
|
||||
def test_get_single_track_import_context_uses_explicit_override_first(monkeypatch):
|
||||
spotify_client = FakeClient(
|
||||
details={"override-track-1": _track_details("spotify", track_id="override-track-1", artist_name="Override Artist", artist_id="spotify-artist-1")},
|
||||
artist_details={"spotify-artist-1": {"id": "spotify-artist-1", "genres": ["pop"]}},
|
||||
)
|
||||
deezer_client = FakeClient()
|
||||
|
||||
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
|
||||
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
|
||||
monkeypatch.setattr(
|
||||
metadata_service,
|
||||
"get_client_for_source",
|
||||
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": None}.get(source),
|
||||
)
|
||||
|
||||
result = resolution.get_single_track_import_context(
|
||||
"Ignored Title",
|
||||
"Ignored Artist",
|
||||
override_id="override-track-1",
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["source"] == "spotify"
|
||||
assert result["context"]["track_info"]["id"] == "override-track-1"
|
||||
assert result["context"]["artist"]["genres"] == ["pop"]
|
||||
assert spotify_client.calls == [
|
||||
("get_track_details", "override-track-1"),
|
||||
("get_artist", "spotify-artist-1"),
|
||||
]
|
||||
assert deezer_client.calls == []
|
||||
|
||||
|
||||
def test_get_single_track_import_context_uses_explicit_override_source(monkeypatch):
|
||||
itunes_client = FakeClient(
|
||||
details={"override-track-2": _track_details("itunes", track_id="override-track-2", artist_name="Override Artist Two", artist_id="itunes-artist-1")},
|
||||
artist_details={"itunes-artist-1": {"id": "itunes-artist-1", "genres": ["singer-songwriter"]}},
|
||||
)
|
||||
spotify_client = FakeClient()
|
||||
deezer_client = FakeClient()
|
||||
|
||||
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
|
||||
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
|
||||
monkeypatch.setattr(
|
||||
metadata_service,
|
||||
"get_client_for_source",
|
||||
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": itunes_client}.get(source),
|
||||
)
|
||||
|
||||
result = resolution.get_single_track_import_context(
|
||||
"Ignored Title",
|
||||
"Ignored Artist",
|
||||
override_id="override-track-2",
|
||||
override_source="itunes",
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["source"] == "itunes"
|
||||
assert result["context"]["track_info"]["id"] == "override-track-2"
|
||||
assert result["context"]["artist"]["genres"] == ["singer-songwriter"]
|
||||
assert itunes_client.calls == [
|
||||
("get_track_details", "override-track-2"),
|
||||
("get_artist", "itunes-artist-1"),
|
||||
]
|
||||
assert deezer_client.calls == []
|
||||
assert spotify_client.calls == []
|
||||
|
||||
|
||||
def test_get_single_track_import_context_returns_fallback_payload_when_no_source_matches(monkeypatch):
|
||||
deezer_client = FakeClient()
|
||||
spotify_client = FakeClient()
|
||||
itunes_client = FakeClient()
|
||||
|
||||
monkeypatch.setattr(metadata_service, "get_primary_source", lambda: "deezer")
|
||||
monkeypatch.setattr(metadata_service, "get_source_priority", lambda primary: [primary, "spotify", "itunes"])
|
||||
monkeypatch.setattr(
|
||||
metadata_service,
|
||||
"get_client_for_source",
|
||||
lambda source: {"deezer": deezer_client, "spotify": spotify_client, "itunes": itunes_client}.get(source),
|
||||
)
|
||||
|
||||
result = resolution.get_single_track_import_context("Missing Song", "Missing Artist")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["source"] is None
|
||||
assert result["context"]["artist"]["name"] == "Missing Artist"
|
||||
assert result["context"]["track_info"]["name"] == "Missing Song"
|
||||
assert result["context"]["original_search_result"]["clean_title"] == "Missing Song"
|
||||
assert result["context"]["original_search_result"]["clean_artist"] == "Missing Artist"
|
||||
187
tests/imports/test_import_side_effects.py
Normal file
187
tests/imports/test_import_side_effects.py
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
import sqlite3
|
||||
from types import SimpleNamespace
|
||||
|
||||
from core.imports import side_effects
|
||||
|
||||
|
||||
class _FakeDB:
|
||||
def __init__(self, conn):
|
||||
self._conn = conn
|
||||
|
||||
def _get_connection(self):
|
||||
return self._conn
|
||||
|
||||
|
||||
class _FakeWishlistService:
|
||||
def __init__(self, tracks):
|
||||
self.tracks = tracks
|
||||
self.removed = []
|
||||
|
||||
def get_wishlist_tracks_for_download(self, profile_id=1):
|
||||
return list(self.tracks)
|
||||
|
||||
def mark_track_download_result(self, spotify_track_id, success, error_message=None, profile_id=1):
|
||||
self.removed.append((spotify_track_id, success, error_message, profile_id))
|
||||
return True
|
||||
|
||||
|
||||
def _make_soulsync_db():
|
||||
conn = sqlite3.connect(":memory:")
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE artists (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
genres TEXT,
|
||||
thumb_url TEXT,
|
||||
server_source TEXT,
|
||||
created_at TEXT,
|
||||
updated_at TEXT,
|
||||
spotify_artist_id TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE albums (
|
||||
id TEXT PRIMARY KEY,
|
||||
artist_id TEXT,
|
||||
title TEXT,
|
||||
year INTEGER,
|
||||
thumb_url TEXT,
|
||||
genres TEXT,
|
||||
track_count INTEGER,
|
||||
duration INTEGER,
|
||||
server_source TEXT,
|
||||
created_at TEXT,
|
||||
updated_at TEXT,
|
||||
spotify_album_id TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE tracks (
|
||||
id TEXT PRIMARY KEY,
|
||||
album_id TEXT,
|
||||
artist_id TEXT,
|
||||
title TEXT,
|
||||
track_number INTEGER,
|
||||
duration INTEGER,
|
||||
file_path TEXT,
|
||||
bitrate INTEGER,
|
||||
track_artist TEXT,
|
||||
server_source TEXT,
|
||||
created_at TEXT,
|
||||
updated_at TEXT,
|
||||
spotify_track_id TEXT
|
||||
)
|
||||
"""
|
||||
)
|
||||
return conn
|
||||
|
||||
|
||||
def test_record_soulsync_library_entry_writes_artist_album_and_track(tmp_path, monkeypatch):
|
||||
conn = _make_soulsync_db()
|
||||
fake_db = _FakeDB(conn)
|
||||
final_path = tmp_path / "track.flac"
|
||||
final_path.write_bytes(b"audio")
|
||||
|
||||
monkeypatch.setattr(side_effects, "get_database", lambda: fake_db)
|
||||
monkeypatch.setattr(
|
||||
side_effects,
|
||||
"_get_config_manager",
|
||||
lambda: SimpleNamespace(get_active_media_server=lambda: "soulsync"),
|
||||
)
|
||||
|
||||
import core.genre_filter as genre_filter
|
||||
|
||||
monkeypatch.setattr(genre_filter, "filter_genres", lambda genres, _cfg: [genre.upper() for genre in genres])
|
||||
|
||||
context = {
|
||||
"source": "spotify",
|
||||
"artist": {"id": "sp-artist", "name": "Artist One"},
|
||||
"album": {
|
||||
"id": "sp-album",
|
||||
"name": "Album One",
|
||||
"release_date": "2024-02-03",
|
||||
"total_tracks": 12,
|
||||
"image_url": "https://img.example/album.jpg",
|
||||
},
|
||||
"track_info": {
|
||||
"id": "sp-track",
|
||||
"name": "Song One",
|
||||
"track_number": 7,
|
||||
"duration_ms": 210000,
|
||||
"artists": [{"name": "Guest Artist"}],
|
||||
"_source": "spotify",
|
||||
},
|
||||
"original_search_result": {
|
||||
"title": "Song One",
|
||||
"artists": [{"name": "Guest Artist"}],
|
||||
"_source": "spotify",
|
||||
},
|
||||
"_final_processed_path": str(final_path),
|
||||
}
|
||||
|
||||
artist_context = {"name": "Artist One", "genres": ["rock", "indie"]}
|
||||
album_info = {"is_album": True, "album_name": "Album One", "track_number": 7}
|
||||
|
||||
side_effects.record_soulsync_library_entry(context, artist_context, album_info)
|
||||
|
||||
artist_row = conn.execute("SELECT * FROM artists").fetchone()
|
||||
album_row = conn.execute("SELECT * FROM albums").fetchone()
|
||||
track_row = conn.execute("SELECT * FROM tracks").fetchone()
|
||||
|
||||
assert artist_row["name"] == "Artist One"
|
||||
assert artist_row["server_source"] == "soulsync"
|
||||
assert artist_row["spotify_artist_id"] == "sp-artist"
|
||||
assert artist_row["genres"] == '["ROCK", "INDIE"]'
|
||||
|
||||
assert album_row["title"] == "Album One"
|
||||
assert album_row["server_source"] == "soulsync"
|
||||
assert album_row["spotify_album_id"] == "sp-album"
|
||||
assert album_row["year"] == 2024
|
||||
assert album_row["track_count"] == 12
|
||||
assert album_row["duration"] == 210000
|
||||
assert album_row["artist_id"] == artist_row["id"]
|
||||
|
||||
assert track_row["title"] == "Song One"
|
||||
assert track_row["server_source"] == "soulsync"
|
||||
assert track_row["spotify_track_id"] == "sp-track"
|
||||
assert track_row["track_number"] == 7
|
||||
assert track_row["duration"] == 210000
|
||||
assert track_row["track_artist"] == "Guest Artist"
|
||||
assert track_row["album_id"] == album_row["id"]
|
||||
assert track_row["file_path"] == str(final_path)
|
||||
|
||||
|
||||
def test_check_and_remove_from_wishlist_uses_search_result_fallback(monkeypatch):
|
||||
fake_db = SimpleNamespace(get_all_profiles=lambda: [{"id": 1}])
|
||||
wishlist_service = _FakeWishlistService([
|
||||
{
|
||||
"wishlist_id": 11,
|
||||
"spotify_track_id": "sp-track-1",
|
||||
"id": "sp-track-1",
|
||||
"name": "Song One",
|
||||
"artists": [{"name": "Artist One"}],
|
||||
}
|
||||
])
|
||||
|
||||
monkeypatch.setattr(side_effects, "get_database", lambda: fake_db)
|
||||
monkeypatch.setattr(side_effects, "get_wishlist_service", lambda: wishlist_service)
|
||||
|
||||
context = {
|
||||
"search_result": {
|
||||
"title": "Song One",
|
||||
"artist": "Artist One",
|
||||
"album": "Album One",
|
||||
},
|
||||
"track_info": {},
|
||||
"original_search_result": {},
|
||||
}
|
||||
|
||||
side_effects.check_and_remove_from_wishlist(context)
|
||||
|
||||
assert wishlist_service.removed == [("sp-track-1", True, None, 1)]
|
||||
265
tests/imports/test_import_staging.py
Normal file
265
tests/imports/test_import_staging.py
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
from types import SimpleNamespace
|
||||
|
||||
import core.imports.staging as import_staging
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, results=None):
|
||||
self.results = results or []
|
||||
self.calls = []
|
||||
|
||||
def search_albums(self, query, **kwargs):
|
||||
self.calls.append((query, kwargs))
|
||||
return self.results
|
||||
|
||||
def search_tracks(self, query, **kwargs):
|
||||
self.calls.append((query, kwargs))
|
||||
return self.results
|
||||
|
||||
|
||||
def _album_result(album_id, name, artist, release_date="2024-01-01", total_tracks=10, image_url="https://img.example/album.jpg", album_type="album"):
|
||||
return SimpleNamespace(
|
||||
id=album_id,
|
||||
name=name,
|
||||
artists=[artist],
|
||||
release_date=release_date,
|
||||
total_tracks=total_tracks,
|
||||
image_url=image_url,
|
||||
album_type=album_type,
|
||||
)
|
||||
|
||||
|
||||
def test_search_import_albums_prefers_primary_source(monkeypatch):
|
||||
deezer_client = FakeClient([
|
||||
_album_result("deezer-1", "Album One", "Artist One"),
|
||||
])
|
||||
spotify_client = FakeClient([
|
||||
_album_result("spotify-1", "Album One", "Artist One"),
|
||||
])
|
||||
|
||||
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
|
||||
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
|
||||
monkeypatch.setattr(
|
||||
import_staging,
|
||||
"get_client_for_source",
|
||||
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
import_staging,
|
||||
"_search_albums_for_source",
|
||||
lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit),
|
||||
)
|
||||
|
||||
results = import_staging.search_import_albums("Album One", limit=2)
|
||||
|
||||
assert results == [
|
||||
{
|
||||
"id": "deezer-1",
|
||||
"name": "Album One",
|
||||
"artist": "Artist One",
|
||||
"release_date": "2024-01-01",
|
||||
"total_tracks": 10,
|
||||
"image_url": "https://img.example/album.jpg",
|
||||
"album_type": "album",
|
||||
"source": "deezer",
|
||||
}
|
||||
]
|
||||
assert deezer_client.calls == [("Album One", {"limit": 2})]
|
||||
assert spotify_client.calls == []
|
||||
|
||||
|
||||
def test_search_import_albums_falls_back_when_primary_has_no_results(monkeypatch):
|
||||
deezer_client = FakeClient([])
|
||||
spotify_client = FakeClient([
|
||||
_album_result("spotify-1", "Album Two", "Artist Two"),
|
||||
])
|
||||
|
||||
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
|
||||
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
|
||||
monkeypatch.setattr(
|
||||
import_staging,
|
||||
"get_client_for_source",
|
||||
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
import_staging,
|
||||
"_search_albums_for_source",
|
||||
lambda source, client, query, limit=5: client.search_albums(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_albums(query, limit=limit),
|
||||
)
|
||||
|
||||
results = import_staging.search_import_albums("Album Two", limit=2)
|
||||
|
||||
assert results == [
|
||||
{
|
||||
"id": "spotify-1",
|
||||
"name": "Album Two",
|
||||
"artist": "Artist Two",
|
||||
"release_date": "2024-01-01",
|
||||
"total_tracks": 10,
|
||||
"image_url": "https://img.example/album.jpg",
|
||||
"album_type": "album",
|
||||
"source": "spotify",
|
||||
}
|
||||
]
|
||||
assert deezer_client.calls == [("Album Two", {"limit": 2})]
|
||||
assert spotify_client.calls == [("Album Two", {"limit": 2, "allow_fallback": False})]
|
||||
|
||||
|
||||
def test_search_import_tracks_prefers_primary_source(monkeypatch):
|
||||
deezer_client = FakeClient([
|
||||
SimpleNamespace(
|
||||
id="deezer-track-1",
|
||||
name="Song One",
|
||||
artists=[{"name": "Artist One"}],
|
||||
album={"id": "deezer-album-1", "name": "Album One"},
|
||||
duration_ms=210000,
|
||||
image_url="https://img.example/track.jpg",
|
||||
track_number=7,
|
||||
),
|
||||
])
|
||||
spotify_client = FakeClient([
|
||||
SimpleNamespace(
|
||||
id="spotify-track-1",
|
||||
name="Song One",
|
||||
artists=["Artist One"],
|
||||
album="Album One",
|
||||
duration_ms=210000,
|
||||
image_url="https://img.example/track.jpg",
|
||||
track_number=7,
|
||||
),
|
||||
])
|
||||
|
||||
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
|
||||
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
|
||||
monkeypatch.setattr(
|
||||
import_staging,
|
||||
"get_client_for_source",
|
||||
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
import_staging,
|
||||
"_search_tracks_for_source",
|
||||
lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit),
|
||||
)
|
||||
|
||||
results = import_staging.search_import_tracks("Song One", limit=2)
|
||||
|
||||
assert results == [
|
||||
{
|
||||
"id": "deezer-track-1",
|
||||
"name": "Song One",
|
||||
"artist": "Artist One",
|
||||
"album": "Album One",
|
||||
"album_id": "deezer-album-1",
|
||||
"duration_ms": 210000,
|
||||
"image_url": "https://img.example/track.jpg",
|
||||
"track_number": 7,
|
||||
"source": "deezer",
|
||||
}
|
||||
]
|
||||
assert deezer_client.calls == [("Song One", {"limit": 2})]
|
||||
assert spotify_client.calls == []
|
||||
|
||||
|
||||
def test_search_import_tracks_falls_back_when_primary_has_no_results(monkeypatch):
|
||||
deezer_client = FakeClient([])
|
||||
spotify_client = FakeClient([
|
||||
SimpleNamespace(
|
||||
id="spotify-track-1",
|
||||
name="Song Two",
|
||||
artists=["Artist Two"],
|
||||
album="Album Two",
|
||||
duration_ms=180000,
|
||||
image_url="",
|
||||
track_number=3,
|
||||
),
|
||||
])
|
||||
|
||||
monkeypatch.setattr(import_staging, "get_primary_source", lambda: "deezer")
|
||||
monkeypatch.setattr(import_staging, "get_source_priority", lambda primary: [primary, "spotify"])
|
||||
monkeypatch.setattr(
|
||||
import_staging,
|
||||
"get_client_for_source",
|
||||
lambda source: {"deezer": deezer_client, "spotify": spotify_client}.get(source),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
import_staging,
|
||||
"_search_tracks_for_source",
|
||||
lambda source, client, query, limit=5: client.search_tracks(query, limit=limit, allow_fallback=False) if source == "spotify" else client.search_tracks(query, limit=limit),
|
||||
)
|
||||
|
||||
results = import_staging.search_import_tracks("Song Two", limit=2)
|
||||
|
||||
assert results == [
|
||||
{
|
||||
"id": "spotify-track-1",
|
||||
"name": "Song Two",
|
||||
"artist": "Artist Two",
|
||||
"album": "Album Two",
|
||||
"album_id": "",
|
||||
"duration_ms": 180000,
|
||||
"image_url": "",
|
||||
"track_number": 3,
|
||||
"source": "spotify",
|
||||
}
|
||||
]
|
||||
assert deezer_client.calls == [("Song Two", {"limit": 2})]
|
||||
assert spotify_client.calls == [("Song Two", {"limit": 2, "allow_fallback": False})]
|
||||
|
||||
|
||||
def test_build_import_suggestions_background_uses_collected_queries(monkeypatch, tmp_path):
|
||||
staging_root = tmp_path / "Staging"
|
||||
staging_root.mkdir()
|
||||
|
||||
cache = import_staging.get_import_suggestions_cache()
|
||||
original_cache = dict(cache)
|
||||
cache["suggestions"] = []
|
||||
cache["building"] = False
|
||||
cache["built"] = False
|
||||
|
||||
monkeypatch.setattr(import_staging, "get_staging_path", lambda: str(staging_root))
|
||||
monkeypatch.setattr(import_staging, "_collect_import_suggestion_queries", lambda staging_path: ["Album One", "Folder Hint"])
|
||||
monkeypatch.setattr(
|
||||
import_staging,
|
||||
"search_import_albums",
|
||||
lambda query, limit=2: [
|
||||
{
|
||||
"id": query.lower().replace(" ", "-"),
|
||||
"name": query,
|
||||
"artist": f"{query} Artist",
|
||||
"release_date": "2024-02-01",
|
||||
"total_tracks": 12,
|
||||
"image_url": "",
|
||||
"album_type": "album",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
try:
|
||||
import_staging._build_import_suggestions_background()
|
||||
|
||||
assert cache["built"] is True
|
||||
assert cache["building"] is False
|
||||
assert cache["suggestions"] == [
|
||||
{
|
||||
"id": "album-one",
|
||||
"name": "Album One",
|
||||
"artist": "Album One Artist",
|
||||
"release_date": "2024-02-01",
|
||||
"total_tracks": 12,
|
||||
"image_url": "",
|
||||
"album_type": "album",
|
||||
},
|
||||
{
|
||||
"id": "folder-hint",
|
||||
"name": "Folder Hint",
|
||||
"artist": "Folder Hint Artist",
|
||||
"release_date": "2024-02-01",
|
||||
"total_tracks": 12,
|
||||
"image_url": "",
|
||||
"album_type": "album",
|
||||
},
|
||||
]
|
||||
finally:
|
||||
cache.clear()
|
||||
cache.update(original_cache)
|
||||
595
tests/metadata/test_metadata_enrichment.py
Normal file
595
tests/metadata/test_metadata_enrichment.py
Normal file
|
|
@ -0,0 +1,595 @@
|
|||
from collections import OrderedDict
|
||||
import types
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from core.metadata import enrichment as me
|
||||
from core.metadata import artwork as ma
|
||||
from core.metadata import source as ms
|
||||
|
||||
|
||||
class _Config:
|
||||
def __init__(self, values=None):
|
||||
self.values = values or {}
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self.values.get(key, default)
|
||||
|
||||
|
||||
class _FakeTag:
|
||||
def __init__(self, kind, **kwargs):
|
||||
self.kind = kind
|
||||
self.kwargs = kwargs
|
||||
|
||||
|
||||
class _FakeID3Tags:
|
||||
def __init__(self):
|
||||
self.added = []
|
||||
|
||||
def add(self, frame):
|
||||
self.added.append(frame)
|
||||
|
||||
def clear(self):
|
||||
self.added.clear()
|
||||
|
||||
def getall(self, _key):
|
||||
return []
|
||||
|
||||
def keys(self):
|
||||
return [frame.kind for frame in self.added]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.added)
|
||||
|
||||
|
||||
class _FakeAudio:
|
||||
def __init__(self):
|
||||
self.tags = _FakeID3Tags()
|
||||
self.save_calls = []
|
||||
self.clear_pictures_calls = 0
|
||||
|
||||
def clear_pictures(self):
|
||||
self.clear_pictures_calls += 1
|
||||
|
||||
def add_tags(self):
|
||||
self.tags = _FakeID3Tags()
|
||||
|
||||
def save(self, **kwargs):
|
||||
self.save_calls.append(kwargs)
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload, content_type="image/jpeg"):
|
||||
self._payload = payload
|
||||
self._content_type = content_type
|
||||
|
||||
def read(self):
|
||||
return self._payload
|
||||
|
||||
def info(self):
|
||||
return types.SimpleNamespace(get_content_type=lambda: self._content_type)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class _FileDB:
|
||||
def __init__(self, db_path):
|
||||
self.db_path = db_path
|
||||
|
||||
def _get_connection(self):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
def _fake_symbols(audio):
|
||||
def _tag_factory(kind):
|
||||
return lambda **kwargs: _FakeTag(kind, **kwargs)
|
||||
|
||||
return types.SimpleNamespace(
|
||||
File=lambda _path: audio,
|
||||
APEv2=type("FakeAPEv2", (), {}),
|
||||
APENoHeaderError=Exception,
|
||||
FLAC=type("FakeFLAC", (), {}),
|
||||
Picture=type("FakePicture", (), {"__init__": lambda self: None}),
|
||||
ID3=_FakeID3Tags,
|
||||
APIC=_tag_factory("APIC"),
|
||||
TBPM=_tag_factory("TBPM"),
|
||||
TCOP=_tag_factory("TCOP"),
|
||||
TDOR=_tag_factory("TDOR"),
|
||||
TDRC=_tag_factory("TDRC"),
|
||||
TCON=_tag_factory("TCON"),
|
||||
TIT2=_tag_factory("TIT2"),
|
||||
TALB=_tag_factory("TALB"),
|
||||
TPE1=_tag_factory("TPE1"),
|
||||
TPE2=_tag_factory("TPE2"),
|
||||
TPOS=_tag_factory("TPOS"),
|
||||
TPUB=_tag_factory("TPUB"),
|
||||
TRCK=_tag_factory("TRCK"),
|
||||
TSRC=_tag_factory("TSRC"),
|
||||
TXXX=_tag_factory("TXXX"),
|
||||
UFID=_tag_factory("UFID"),
|
||||
TMED=_tag_factory("TMED"),
|
||||
MP4=type("FakeMP4", (), {}),
|
||||
MP4Cover=types.SimpleNamespace(FORMAT_JPEG=1, FORMAT_PNG=2, __call__=lambda *args, **kwargs: ("cover", args, kwargs)),
|
||||
MP4FreeForm=lambda data: ("freeform", data),
|
||||
OggVorbis=type("FakeOggVorbis", (), {}),
|
||||
OggOpus=None,
|
||||
)
|
||||
|
||||
|
||||
def test_extract_source_metadata_keeps_neutral_fields_and_skips_itunes_fallback_for_non_itunes_sources(monkeypatch):
|
||||
monkeypatch.setattr(ms, "get_config_manager", lambda: _Config({"file_organization.collab_artist_mode": "first"}))
|
||||
monkeypatch.setattr(ms, "get_itunes_client", lambda: (_ for _ in ()).throw(AssertionError("itunes fallback should not run for non-itunes sources")))
|
||||
|
||||
context = {
|
||||
"source": "spotify",
|
||||
"artist": {"name": "Artist One & Artist Two", "id": "123", "genres": ["rock", "indie"]},
|
||||
"album": {
|
||||
"name": "Album One",
|
||||
"total_tracks": 12,
|
||||
"release_date": "2024-01-02",
|
||||
"images": [{"url": "https://img.example/album.jpg"}],
|
||||
},
|
||||
"track_info": {
|
||||
"artists": [{"name": "Artist One"}],
|
||||
"_source": "spotify",
|
||||
"track_number": 3,
|
||||
"disc_number": 2,
|
||||
"total_tracks": 12,
|
||||
},
|
||||
"original_search_result": {
|
||||
"title": "Song One",
|
||||
"artists": [{"name": "Artist One"}],
|
||||
"clean_title": "Song One",
|
||||
"clean_album": "Album One",
|
||||
"clean_artist": "Artist One",
|
||||
"disc_number": 2,
|
||||
"duration_ms": 180000,
|
||||
},
|
||||
}
|
||||
|
||||
metadata = me.extract_source_metadata(
|
||||
context,
|
||||
context["artist"],
|
||||
{"is_album": True, "album_name": "Album One", "track_number": 3, "disc_number": 2, "album_image_url": "https://img.example/album.jpg"},
|
||||
)
|
||||
|
||||
assert metadata["source"] == "spotify"
|
||||
assert metadata["title"] == "Song One"
|
||||
assert metadata["artist"] == "Artist One"
|
||||
assert metadata["album_artist"] == "Artist One & Artist Two"
|
||||
assert metadata["album"] == "Album One"
|
||||
assert metadata["track_number"] == 3
|
||||
assert metadata["total_tracks"] == 12
|
||||
assert metadata["disc_number"] == 2
|
||||
assert metadata["album_art_url"] == "https://img.example/album.jpg"
|
||||
|
||||
|
||||
def test_embed_source_ids_uses_current_source_ids_and_legacy_fallback(monkeypatch):
|
||||
audio = _FakeAudio()
|
||||
symbols = _fake_symbols(audio)
|
||||
monkeypatch.setattr(ms, "get_config_manager", lambda: _Config())
|
||||
monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols)
|
||||
monkeypatch.setattr(ms, "get_database", lambda: None)
|
||||
|
||||
current_metadata = {
|
||||
"source": "deezer",
|
||||
"source_track_id": "dz-track",
|
||||
"source_artist_id": "dz-artist",
|
||||
"source_album_id": "dz-album",
|
||||
"title": "Song One",
|
||||
"artist": "Artist One",
|
||||
"album_artist": "Artist One",
|
||||
"album": "Album One",
|
||||
}
|
||||
me.embed_source_ids(audio, current_metadata, context={"track_info": {}, "original_search_result": {}})
|
||||
|
||||
current_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"]
|
||||
assert "DEEZER_TRACK_ID" in current_descs
|
||||
assert "DEEZER_ARTIST_ID" in current_descs
|
||||
|
||||
audio = _FakeAudio()
|
||||
symbols = _fake_symbols(audio)
|
||||
monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols)
|
||||
|
||||
legacy_metadata = {
|
||||
"source": "",
|
||||
"spotify_track_id": "sp-track",
|
||||
"spotify_artist_id": "sp-artist",
|
||||
"spotify_album_id": "sp-album",
|
||||
"itunes_track_id": "it-track",
|
||||
"itunes_artist_id": "it-artist",
|
||||
"itunes_album_id": "it-album",
|
||||
"title": "Song One",
|
||||
"artist": "Artist One",
|
||||
"album_artist": "Artist One",
|
||||
"album": "Album One",
|
||||
}
|
||||
me.embed_source_ids(audio, legacy_metadata, context={"track_info": {}, "original_search_result": {}})
|
||||
|
||||
legacy_descs = [frame.kwargs.get("desc") for frame in audio.tags.added if frame.kind == "TXXX"]
|
||||
assert "SPOTIFY_TRACK_ID" in legacy_descs
|
||||
assert "SPOTIFY_ARTIST_ID" in legacy_descs
|
||||
assert "SPOTIFY_ALBUM_ID" in legacy_descs
|
||||
assert "ITUNES_TRACK_ID" in legacy_descs
|
||||
assert "ITUNES_ARTIST_ID" in legacy_descs
|
||||
assert "ITUNES_ALBUM_ID" in legacy_descs
|
||||
|
||||
|
||||
def test_embed_source_ids_skips_disabled_source_specific_tags(monkeypatch):
|
||||
audio = _FakeAudio()
|
||||
symbols = _fake_symbols(audio)
|
||||
|
||||
monkeypatch.setattr(
|
||||
ms,
|
||||
"get_config_manager",
|
||||
lambda: _Config({"deezer.embed_tags": False}),
|
||||
)
|
||||
monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols)
|
||||
monkeypatch.setattr(ms, "get_database", lambda: None)
|
||||
|
||||
metadata = {
|
||||
"source": "deezer",
|
||||
"source_track_id": "dz-track",
|
||||
"source_artist_id": "dz-artist",
|
||||
"source_album_id": "dz-album",
|
||||
"title": "Song One",
|
||||
"artist": "Artist One",
|
||||
"album_artist": "Artist One",
|
||||
"album": "Album One",
|
||||
}
|
||||
|
||||
me.embed_source_ids(audio, metadata, context={"track_info": {}, "original_search_result": {}})
|
||||
|
||||
assert audio.tags.added == []
|
||||
|
||||
|
||||
def test_embed_source_ids_writes_musicbrainz_release_year_and_updates_album_year(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "music.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute("CREATE TABLE artists (id TEXT PRIMARY KEY, name TEXT)")
|
||||
conn.execute("CREATE TABLE albums (id TEXT PRIMARY KEY, artist_id TEXT, title TEXT, year INTEGER)")
|
||||
conn.execute("INSERT INTO artists (id, name) VALUES (?, ?)", ("artist-1", "Artist One"))
|
||||
conn.execute(
|
||||
"INSERT INTO albums (id, artist_id, title, year) VALUES (?, ?, ?, ?)",
|
||||
("album-1", "artist-1", "Album One", None),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
class _FakeMBClient:
|
||||
def get_recording(self, mbid, includes=None):
|
||||
return {
|
||||
"isrcs": ["ISRC-123"],
|
||||
"genres": [{"name": "Post Rock", "count": 10}],
|
||||
}
|
||||
|
||||
def get_release(self, mbid, includes=None):
|
||||
return {
|
||||
"release-group": {
|
||||
"id": "rg-1",
|
||||
"primary-type": "album",
|
||||
"first-release-date": "2021-09-17",
|
||||
},
|
||||
"artist-credit": [{"artist": {"id": "artist-mb-1"}}],
|
||||
"status": "Official",
|
||||
"country": "US",
|
||||
"barcode": "1234567890",
|
||||
"media": [{"format": "CD", "tracks": [{"position": 1, "id": "reltrack-1", "recording": {"id": "rec-1"}}]}],
|
||||
"label-info": [{"catalog-number": "CAT-1"}],
|
||||
"text-representation": {"script": "Latn"},
|
||||
"asin": "ASIN1",
|
||||
}
|
||||
|
||||
class _FakeMBService:
|
||||
def __init__(self):
|
||||
self.mb_client = _FakeMBClient()
|
||||
|
||||
def match_recording(self, title, artist):
|
||||
return {"mbid": "rec-mbid"}
|
||||
|
||||
def match_artist(self, artist):
|
||||
return {"mbid": "artist-mbid"}
|
||||
|
||||
def match_release(self, album, artist):
|
||||
return {"mbid": "release-mbid"}
|
||||
|
||||
audio = _FakeAudio()
|
||||
symbols = _fake_symbols(audio)
|
||||
runtime = types.SimpleNamespace(mb_worker=types.SimpleNamespace(mb_service=_FakeMBService()))
|
||||
|
||||
monkeypatch.setattr(
|
||||
ms,
|
||||
"get_config_manager",
|
||||
lambda: _Config(
|
||||
{
|
||||
"metadata_enhancement.enabled": True,
|
||||
"metadata_enhancement.embed_album_art": False,
|
||||
"metadata_enhancement.tags.write_multi_artist": False,
|
||||
"musicbrainz.embed_tags": True,
|
||||
}
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols)
|
||||
monkeypatch.setattr(ms, "get_database", lambda: _FileDB(str(db_path)))
|
||||
|
||||
metadata = {
|
||||
"source": "musicbrainz",
|
||||
"title": "Song One",
|
||||
"artist": "Artist One",
|
||||
"album_artist": "Artist One",
|
||||
"album": "Album One",
|
||||
"track_number": 1,
|
||||
"total_tracks": 12,
|
||||
"disc_number": 1,
|
||||
}
|
||||
|
||||
me.embed_source_ids(audio, metadata, context={"track_info": {}, "original_search_result": {}}, runtime=runtime)
|
||||
|
||||
assert metadata["musicbrainz_release_id"] == "release-mbid"
|
||||
assert metadata["date"] == "2021"
|
||||
assert any(frame.kind == "TDRC" for frame in audio.tags.added)
|
||||
assert any(frame.kind == "TSRC" for frame in audio.tags.added)
|
||||
|
||||
check = sqlite3.connect(db_path)
|
||||
check.row_factory = sqlite3.Row
|
||||
row = check.execute(
|
||||
"""
|
||||
SELECT albums.year
|
||||
FROM albums
|
||||
JOIN artists ON artists.id = albums.artist_id
|
||||
WHERE albums.title = ? AND artists.name = ?
|
||||
""",
|
||||
("Album One", "Artist One"),
|
||||
).fetchone()
|
||||
check.close()
|
||||
|
||||
assert row["year"] == 2021
|
||||
|
||||
|
||||
def test_musicbrainz_release_lookup_failure_does_not_poison_cache(monkeypatch):
|
||||
class _FakeMBClient:
|
||||
def get_release(self, mbid, includes=None):
|
||||
return {}
|
||||
|
||||
class _FakeMBService:
|
||||
def __init__(self):
|
||||
self.release_calls = 0
|
||||
self.mb_client = _FakeMBClient()
|
||||
|
||||
def match_recording(self, title, artist):
|
||||
return None
|
||||
|
||||
def match_artist(self, artist):
|
||||
return {"mbid": "artist-mbid"}
|
||||
|
||||
def match_release(self, album, artist):
|
||||
self.release_calls += 1
|
||||
if self.release_calls == 1:
|
||||
raise requests.RequestException("temporary MusicBrainz outage")
|
||||
return {"mbid": "release-mbid"}
|
||||
|
||||
monkeypatch.setattr(ms, "get_config_manager", lambda: _Config({"musicbrainz.embed_tags": True}))
|
||||
monkeypatch.setattr(ms, "mb_release_cache", {})
|
||||
monkeypatch.setattr(ms, "mb_release_detail_cache", {})
|
||||
|
||||
service = _FakeMBService()
|
||||
runtime = types.SimpleNamespace(mb_worker=types.SimpleNamespace(mb_service=service))
|
||||
pp = {
|
||||
"id_tags": {},
|
||||
"track_title": "Song One",
|
||||
"artist_name": "Artist One",
|
||||
"batch_artist_name": "Artist One",
|
||||
"metadata": {"album": "Album One"},
|
||||
"recording_mbid": None,
|
||||
"artist_mbid": None,
|
||||
"release_mbid": "",
|
||||
"mb_genres": [],
|
||||
"isrc": None,
|
||||
"deezer_bpm": None,
|
||||
"deezer_isrc": None,
|
||||
"audiodb_mood": None,
|
||||
"audiodb_style": None,
|
||||
"audiodb_genre": None,
|
||||
"tidal_isrc": None,
|
||||
"tidal_copyright": None,
|
||||
"qobuz_isrc": None,
|
||||
"qobuz_copyright": None,
|
||||
"qobuz_label": None,
|
||||
"lastfm_tags": [],
|
||||
"lastfm_url": None,
|
||||
"genius_url": None,
|
||||
"release_year": None,
|
||||
}
|
||||
metadata = {"album": "Album One", "artist": "Artist One"}
|
||||
|
||||
ms._process_musicbrainz_source(pp, metadata, _Config({"musicbrainz.embed_tags": True}), runtime, "Song One", "Artist One")
|
||||
assert service.release_calls == 1
|
||||
assert ms.mb_release_cache == {}
|
||||
|
||||
poisoned_norm_key = (ms.normalize_album_cache_key("Album One"), "artist one")
|
||||
poisoned_exact_key = ("album one", "artist one")
|
||||
ms.mb_release_cache[poisoned_norm_key] = ""
|
||||
ms.mb_release_cache[poisoned_exact_key] = ""
|
||||
|
||||
ms._process_musicbrainz_source(pp, metadata, _Config({"musicbrainz.embed_tags": True}), runtime, "Song One", "Artist One")
|
||||
assert service.release_calls == 2
|
||||
|
||||
|
||||
def test_source_processors_do_not_swallow_programmer_errors():
|
||||
class _BoomClient:
|
||||
def search_track(self, artist_name, track_title):
|
||||
raise ValueError("boom")
|
||||
|
||||
runtime = types.SimpleNamespace(deezer_worker=types.SimpleNamespace(client=_BoomClient()))
|
||||
pp = {
|
||||
"id_tags": {},
|
||||
"batch_artist_name": "Artist One",
|
||||
"release_year": None,
|
||||
}
|
||||
metadata = {"album": "Album One"}
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
ms._process_deezer_source(pp, metadata, _Config({"deezer.embed_tags": True}), runtime, "Song One", "Artist One")
|
||||
|
||||
|
||||
def test_musicbrainz_caches_evict_oldest_entries():
|
||||
release_cache = OrderedDict()
|
||||
detail_cache = OrderedDict()
|
||||
|
||||
ms._bounded_cache_set(release_cache, ("album-1", "artist"), "release-1", 2)
|
||||
ms._bounded_cache_set(release_cache, ("album-2", "artist"), "release-2", 2)
|
||||
assert list(release_cache.keys()) == [("album-1", "artist"), ("album-2", "artist")]
|
||||
|
||||
assert ms._bounded_cache_get(release_cache, ("album-1", "artist")) == "release-1"
|
||||
ms._bounded_cache_set(release_cache, ("album-3", "artist"), "release-3", 2)
|
||||
assert list(release_cache.keys()) == [("album-1", "artist"), ("album-3", "artist")]
|
||||
|
||||
ms._bounded_cache_set(detail_cache, "release-1", {"title": "One"}, 1)
|
||||
ms._bounded_cache_set(detail_cache, "release-2", {"title": "Two"}, 1)
|
||||
assert list(detail_cache.keys()) == ["release-2"]
|
||||
|
||||
|
||||
def test_enhance_file_metadata_forwards_runtime_to_source_embedding(monkeypatch):
|
||||
audio = _FakeAudio()
|
||||
symbols = _fake_symbols(audio)
|
||||
seen = {}
|
||||
runtime = types.SimpleNamespace(marker="runtime")
|
||||
|
||||
monkeypatch.setattr(me, "get_config_manager", lambda: _Config(
|
||||
{
|
||||
"metadata_enhancement.enabled": True,
|
||||
"metadata_enhancement.embed_album_art": False,
|
||||
"metadata_enhancement.tags.write_multi_artist": False,
|
||||
}
|
||||
))
|
||||
monkeypatch.setattr(me, "get_mutagen_symbols", lambda: symbols)
|
||||
monkeypatch.setattr(me, "strip_all_non_audio_tags", lambda file_path: {"apev2_stripped": False, "apev2_tag_count": 0})
|
||||
monkeypatch.setattr(
|
||||
me,
|
||||
"extract_source_metadata",
|
||||
lambda context, artist, album_info: {
|
||||
"source": "deezer",
|
||||
"source_track_id": "dz-track",
|
||||
"source_artist_id": "dz-artist",
|
||||
"source_album_id": "dz-album",
|
||||
"title": "Song One",
|
||||
"artist": "Artist One",
|
||||
"album_artist": "Artist One",
|
||||
"album": "Album One",
|
||||
"track_number": 3,
|
||||
"total_tracks": 12,
|
||||
"disc_number": 2,
|
||||
"date": "2024",
|
||||
"genre": "Rock",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
me,
|
||||
"embed_source_ids",
|
||||
lambda audio_file, metadata, context, runtime=None: seen.setdefault("runtime", runtime),
|
||||
)
|
||||
monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(me, "verify_metadata_written", lambda file_path: True)
|
||||
|
||||
result = me.enhance_file_metadata(
|
||||
"song.flac",
|
||||
{"_audio_quality": ""},
|
||||
{"name": "Artist One"},
|
||||
{},
|
||||
runtime=runtime,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert seen["runtime"] is runtime
|
||||
|
||||
|
||||
def test_enhance_file_metadata_writes_tags_and_propagates_release_id(monkeypatch):
|
||||
audio = _FakeAudio()
|
||||
symbols = _fake_symbols(audio)
|
||||
strip_calls = []
|
||||
verify_calls = []
|
||||
|
||||
monkeypatch.setattr(me, "get_config_manager", lambda: _Config(
|
||||
{
|
||||
"metadata_enhancement.enabled": True,
|
||||
"metadata_enhancement.embed_album_art": False,
|
||||
"metadata_enhancement.tags.write_multi_artist": False,
|
||||
}
|
||||
))
|
||||
monkeypatch.setattr(ms, "get_config_manager", lambda: _Config())
|
||||
monkeypatch.setattr(me, "get_mutagen_symbols", lambda: symbols)
|
||||
monkeypatch.setattr(ms, "get_mutagen_symbols", lambda: symbols)
|
||||
monkeypatch.setattr(ms, "get_database", lambda: None)
|
||||
monkeypatch.setattr(me, "strip_all_non_audio_tags", lambda file_path: strip_calls.append(file_path) or {"apev2_stripped": False, "apev2_tag_count": 0})
|
||||
monkeypatch.setattr(
|
||||
me,
|
||||
"extract_source_metadata",
|
||||
lambda context, artist, album_info: {
|
||||
"source": "deezer",
|
||||
"source_track_id": "dz-track",
|
||||
"source_artist_id": "dz-artist",
|
||||
"source_album_id": "dz-album",
|
||||
"title": "Song One",
|
||||
"artist": "Artist One",
|
||||
"album_artist": "Artist One",
|
||||
"album": "Album One",
|
||||
"track_number": 3,
|
||||
"total_tracks": 12,
|
||||
"disc_number": 2,
|
||||
"date": "2024",
|
||||
"genre": "Rock",
|
||||
"musicbrainz_release_id": "mb-release-1",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(me, "embed_album_art_metadata", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(me, "verify_metadata_written", lambda file_path: verify_calls.append(file_path) or True)
|
||||
|
||||
album_info = {}
|
||||
result = me.enhance_file_metadata(
|
||||
"song.flac",
|
||||
{"_audio_quality": ""},
|
||||
{"name": "Artist One"},
|
||||
album_info,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
assert strip_calls == ["song.flac"]
|
||||
assert verify_calls == ["song.flac"]
|
||||
assert audio.clear_pictures_calls == 1
|
||||
assert len(audio.save_calls) == 2
|
||||
assert album_info["musicbrainz_release_id"] == "mb-release-1"
|
||||
assert any(frame.kind == "TIT2" for frame in audio.tags.added)
|
||||
assert any(frame.kind == "TPE1" for frame in audio.tags.added)
|
||||
assert any(frame.kind == "TXXX" and frame.kwargs.get("desc") == "DEEZER_TRACK_ID" for frame in audio.tags.added)
|
||||
|
||||
|
||||
def test_download_cover_art_uses_album_context_image_url(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(ma, "get_config_manager", lambda: _Config(
|
||||
{
|
||||
"metadata_enhancement.cover_art_download": True,
|
||||
"metadata_enhancement.prefer_caa_art": False,
|
||||
}
|
||||
))
|
||||
|
||||
monkeypatch.setattr(ma.urllib.request, "urlopen", lambda *args, **kwargs: _FakeResponse(b"cover-bytes"))
|
||||
|
||||
target_dir = tmp_path / "Album One"
|
||||
target_dir.mkdir()
|
||||
|
||||
ma.download_cover_art(
|
||||
{},
|
||||
str(target_dir),
|
||||
{"album": {"image_url": "https://img.example/album.jpg"}},
|
||||
)
|
||||
|
||||
cover_path = target_dir / "cover.jpg"
|
||||
assert cover_path.exists()
|
||||
assert cover_path.read_bytes() == b"cover-bytes"
|
||||
56
tests/metadata/test_runtime_bundle.py
Normal file
56
tests/metadata/test_runtime_bundle.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import types
|
||||
|
||||
from core.imports.pipeline import build_import_pipeline_runtime
|
||||
from core.metadata.enrichment import build_metadata_enrichment_runtime
|
||||
|
||||
|
||||
def test_build_import_pipeline_runtime_exposes_expected_contract():
|
||||
import_fields = {
|
||||
"automation_engine": object(),
|
||||
"on_download_completed": object(),
|
||||
"web_scan_manager": object(),
|
||||
"repair_worker": object(),
|
||||
}
|
||||
runtime = build_import_pipeline_runtime(**import_fields)
|
||||
|
||||
assert isinstance(runtime, types.SimpleNamespace)
|
||||
for name, value in import_fields.items():
|
||||
assert hasattr(runtime, name)
|
||||
assert getattr(runtime, name) is value
|
||||
|
||||
for name in (
|
||||
"mb_worker",
|
||||
"deezer_worker",
|
||||
"audiodb_worker",
|
||||
"tidal_client",
|
||||
"qobuz_enrichment_worker",
|
||||
"lastfm_worker",
|
||||
"genius_worker",
|
||||
"spotify_enrichment_worker",
|
||||
"itunes_enrichment_worker",
|
||||
):
|
||||
assert not hasattr(runtime, name)
|
||||
|
||||
|
||||
def test_build_metadata_enrichment_runtime_exposes_expected_contract():
|
||||
metadata_fields = {
|
||||
"mb_worker": object(),
|
||||
"deezer_worker": object(),
|
||||
"audiodb_worker": object(),
|
||||
"tidal_client": object(),
|
||||
"qobuz_enrichment_worker": object(),
|
||||
"lastfm_worker": object(),
|
||||
"genius_worker": object(),
|
||||
"spotify_enrichment_worker": object(),
|
||||
"itunes_enrichment_worker": object(),
|
||||
}
|
||||
|
||||
runtime = build_metadata_enrichment_runtime(**metadata_fields)
|
||||
|
||||
assert isinstance(runtime, types.SimpleNamespace)
|
||||
for name, value in metadata_fields.items():
|
||||
assert hasattr(runtime, name)
|
||||
assert getattr(runtime, name) is value
|
||||
|
||||
for name in ("automation_engine", "on_download_completed", "web_scan_manager", "repair_worker"):
|
||||
assert not hasattr(runtime, name)
|
||||
|
|
@ -2,6 +2,14 @@ import sys
|
|||
import types
|
||||
|
||||
|
||||
class _DummyConfigManager:
|
||||
def get(self, key, default=None):
|
||||
return default
|
||||
|
||||
def get_active_media_server(self):
|
||||
return "plex"
|
||||
|
||||
|
||||
if "spotipy" not in sys.modules:
|
||||
spotipy = types.ModuleType("spotipy")
|
||||
|
||||
|
|
@ -26,13 +34,6 @@ if "config.settings" not in sys.modules:
|
|||
config_pkg = types.ModuleType("config")
|
||||
settings_mod = types.ModuleType("config.settings")
|
||||
|
||||
class _DummyConfigManager:
|
||||
def get(self, key, default=None):
|
||||
return default
|
||||
|
||||
def get_active_media_server(self):
|
||||
return "plex"
|
||||
|
||||
settings_mod.config_manager = _DummyConfigManager()
|
||||
config_pkg.settings = settings_mod
|
||||
sys.modules["config"] = config_pkg
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ and `status` query params and includes a `total` count.
|
|||
"""
|
||||
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
|
|
@ -45,6 +44,7 @@ _install_flask_limiter_stub()
|
|||
from flask import Flask, Blueprint # noqa: E402
|
||||
|
||||
from api import downloads as downloads_mod # noqa: E402
|
||||
import core.runtime_state as runtime_state # noqa: E402
|
||||
|
||||
|
||||
def _make_task(status="downloading", when=None):
|
||||
|
|
@ -63,11 +63,10 @@ def _make_task(status="downloading", when=None):
|
|||
|
||||
def _make_app_with_tasks(tasks_dict):
|
||||
"""Create a minimal Flask app with the downloads blueprint mounted and
|
||||
a fake `web_server` module exposing the given download_tasks dict."""
|
||||
fake_ws = types.ModuleType("web_server")
|
||||
fake_ws.download_tasks = tasks_dict
|
||||
fake_ws.tasks_lock = threading.RLock()
|
||||
sys.modules["web_server"] = fake_ws
|
||||
the shared import runtime state populated with the given download_tasks dict."""
|
||||
original_tasks = dict(runtime_state.download_tasks)
|
||||
runtime_state.download_tasks.clear()
|
||||
runtime_state.download_tasks.update(tasks_dict)
|
||||
|
||||
# Bypass API key auth for tests.
|
||||
def _passthrough(f):
|
||||
|
|
@ -83,6 +82,7 @@ def _make_app_with_tasks(tasks_dict):
|
|||
downloads_mod.register_routes(bp)
|
||||
|
||||
app.register_blueprint(bp)
|
||||
app._original_download_tasks = original_tasks
|
||||
return app
|
||||
|
||||
|
||||
|
|
@ -96,8 +96,12 @@ def client():
|
|||
for i in range(25)
|
||||
}
|
||||
app = _make_app_with_tasks(tasks)
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
try:
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
finally:
|
||||
runtime_state.download_tasks.clear()
|
||||
runtime_state.download_tasks.update(app._original_download_tasks)
|
||||
|
||||
|
||||
def test_default_limit_applied(client):
|
||||
|
|
|
|||
32
tests/test_runtime_state.py
Normal file
32
tests/test_runtime_state.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
import pytest
|
||||
|
||||
import core.runtime_state as runtime_state
|
||||
|
||||
|
||||
def test_mark_task_completed_requires_tasks_lock():
|
||||
original_tasks = dict(runtime_state.download_tasks)
|
||||
runtime_state.download_tasks.clear()
|
||||
runtime_state.download_tasks["task-1"] = {"status": "running", "stream_processed": False}
|
||||
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="tasks_lock"):
|
||||
runtime_state.mark_task_completed("task-1")
|
||||
finally:
|
||||
runtime_state.download_tasks.clear()
|
||||
runtime_state.download_tasks.update(original_tasks)
|
||||
|
||||
|
||||
def test_mark_task_completed_succeeds_when_lock_held():
|
||||
original_tasks = dict(runtime_state.download_tasks)
|
||||
runtime_state.download_tasks.clear()
|
||||
runtime_state.download_tasks["task-1"] = {"status": "running", "stream_processed": False}
|
||||
|
||||
try:
|
||||
with runtime_state.tasks_lock:
|
||||
assert runtime_state.mark_task_completed("task-1", {"name": "Song One"}) is True
|
||||
assert runtime_state.download_tasks["task-1"]["status"] == "completed"
|
||||
assert runtime_state.download_tasks["task-1"]["stream_processed"] is True
|
||||
assert runtime_state.download_tasks["task-1"]["track_info"] == {"name": "Song One"}
|
||||
finally:
|
||||
runtime_state.download_tasks.clear()
|
||||
runtime_state.download_tasks.update(original_tasks)
|
||||
|
|
@ -2,10 +2,8 @@ import sys
|
|||
import types
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
_RECENT_RELEASE_DATE = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
if "spotipy" not in sys.modules:
|
||||
spotipy = types.ModuleType("spotipy")
|
||||
|
||||
|
|
@ -1045,6 +1043,10 @@ def test_update_discovery_pool_incremental_uses_source_priority(monkeypatch):
|
|||
|
||||
assert scanner.database.discovery_pool_calls
|
||||
assert scanner.database.discovery_pool_calls[0][1] == "deezer"
|
||||
assert scanner.database.discovery_pool_calls[0][0]["deezer_track_id"] == "dz-track-1"
|
||||
assert scanner.database.discovery_pool_calls[0][0]["deezer_album_id"] == "dz-release-1"
|
||||
assert scanner.database.discovery_pool_calls[0][0]["deezer_artist_id"] == "dz-artist"
|
||||
assert scanner.database.discovery_pool_calls[0][0].get("spotify_track_id") is None
|
||||
assert deezer_client.search_calls == [("Incremental Artist", 1, {})]
|
||||
assert deezer_client.album_calls
|
||||
assert spotify_client.search_calls == []
|
||||
|
|
|
|||
5498
web_server.py
5498
web_server.py
File diff suppressed because it is too large
Load diff
|
|
@ -1214,6 +1214,7 @@ function importPageRenderMatchList() {
|
|||
// Build rows
|
||||
let matchedCount = 0;
|
||||
const rows = data.matches.map((m, idx) => {
|
||||
const trackInfo = _importPageGetTrackDisplayInfo(m, idx);
|
||||
let file = null;
|
||||
let confidence = m.confidence;
|
||||
let isOverride = false;
|
||||
|
|
@ -1253,8 +1254,8 @@ function importPageRenderMatchList() {
|
|||
<div class="import-page-match-row ${file ? 'matched' : ''}"
|
||||
ondragover="importPageHandleDragOver(event)" ondragleave="this.classList.remove('drag-over')" ondrop="importPageHandleDrop(event, ${idx})"
|
||||
onclick="importPageTapAssign(${idx})">
|
||||
<span class="import-page-match-num">${m.spotify_track.track_number}</span>
|
||||
<span class="import-page-match-track">${_esc(m.spotify_track.name)}</span>
|
||||
<span class="import-page-match-num">${trackInfo.displayTrackNumber}</span>
|
||||
<span class="import-page-match-track">${_esc(trackInfo.name)}</span>
|
||||
<span class="import-page-match-file ${file ? 'has-file' : ''}">
|
||||
${file
|
||||
? `<span class="import-page-match-file-name">${_esc(file.filename)}</span>
|
||||
|
|
@ -1304,6 +1305,21 @@ function importPageRenderMatchList() {
|
|||
processBtn.textContent = `Process ${matchedCount} Track${matchedCount !== 1 ? 's' : ''}`;
|
||||
}
|
||||
|
||||
function _importPageGetTrackDisplayInfo(item, index) {
|
||||
const track = item?.track || item?.spotify_track || {};
|
||||
const rawTrackNumber = track.track_number ?? track.trackNumber ?? null;
|
||||
const trackNumber = rawTrackNumber === null || rawTrackNumber === undefined || rawTrackNumber === ''
|
||||
? null
|
||||
: String(rawTrackNumber).split('/')[0].trim();
|
||||
|
||||
return {
|
||||
track,
|
||||
name: track.name || track.title || `Track ${index + 1}`,
|
||||
trackNumber,
|
||||
displayTrackNumber: trackNumber || String(index + 1),
|
||||
};
|
||||
}
|
||||
|
||||
// --- Album Tab: Drag and Drop ---
|
||||
|
||||
function importPageStartDrag(event, stagingFileIndex) {
|
||||
|
|
@ -1622,7 +1638,7 @@ function importPageProcessSingles() {
|
|||
const f = importPageState.stagingFiles[i];
|
||||
const manualMatch = importPageState.singlesManualMatches[i];
|
||||
if (manualMatch) {
|
||||
return { ...f, spotify_override: manualMatch };
|
||||
return { ...f, manual_match: manualMatch };
|
||||
}
|
||||
return f;
|
||||
});
|
||||
|
|
@ -1669,7 +1685,7 @@ function _importQueueAdd(job) {
|
|||
async function _importQueueRunJob(entry, job) {
|
||||
for (let i = 0; i < job.items.length; i++) {
|
||||
const itemName = job.type === 'album'
|
||||
? (job.items[i].spotify_track?.name || `Track ${i + 1}`)
|
||||
? _importPageGetTrackDisplayInfo(job.items[i], i).name
|
||||
: (job.items[i].title || job.items[i].filename || `File ${i + 1}`);
|
||||
|
||||
// Update status with current track info
|
||||
|
|
@ -7608,4 +7624,3 @@ window.updateEnhanceSelectedCount = updateEnhanceSelectedCount;
|
|||
window.submitEnhanceQuality = submitEnhanceQuality;
|
||||
|
||||
// ===== END ENHANCE QUALITY MODAL =====
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue